This is an automated email from the ASF dual-hosted git repository. maxyang pushed a commit to branch merge-with-upstream in repository https://gitbox.apache.org/repos/asf/cloudberry-pxf.git
commit deee1e14ec3875800ffe1faea14840c51f570a0e Author: liuxiaoyu <[email protected]> AuthorDate: Wed Nov 19 22:06:52 2025 +0800 Add a basic pxf ci and summary test result --- .github/workflows/pxf-ci.yml | 228 +++++++++++++++++++++ automation/pom.xml | 23 ++- .../pxf/automation/components/gpdb/Gpdb.java | 2 +- .../pxf/automation/components/regress/Regress.java | 2 +- .../pxf/automation/features/orc/OrcReadTest.java | 10 +- .../docker/pxf-cbdb-dev/ubuntu/docker-compose.yml | 23 +++ .../ubuntu/script/build_cloudberrry.sh | 133 ++++++++++++ .../docker/pxf-cbdb-dev/ubuntu/script/build_pxf.sh | 35 ++++ .../pxf-cbdb-dev/ubuntu/script/entrypoint.sh | 226 ++++++++++++++++++++ .../docker/pxf-cbdb-dev/ubuntu/script/pxf-env.sh | 24 +++ concourse/singlecluster/Dockerfile | 79 +++++++ .../singlecluster}/README.HDP3.md | 86 +------- .../singlecluster}/bin/gphd-env.sh | 5 + .../singlecluster}/bin/hadoop | 0 .../singlecluster}/bin/hadoop-datanode.sh | 2 +- .../singlecluster}/bin/hbase | 0 .../singlecluster}/bin/hbase-regionserver.sh | 0 .../singlecluster}/bin/hdfs | 0 .../singlecluster}/bin/hive | 0 .../singlecluster}/bin/hive-service.sh | 0 .../singlecluster}/bin/init-gphd.sh | 0 .../singlecluster}/bin/init-pxf.sh | 0 .../singlecluster}/bin/init-ranger.sh | 4 +- .../singlecluster}/bin/pxf-service.sh | 4 +- .../singlecluster}/bin/restart-gphd.sh | 0 .../singlecluster}/bin/restart-pxf.sh | 0 .../singlecluster}/bin/start-gphd.sh | 0 .../singlecluster}/bin/start-hbase.sh | 0 .../singlecluster}/bin/start-hdfs.sh | 4 + .../singlecluster}/bin/start-hive.sh | 0 .../singlecluster}/bin/start-pxf.sh | 0 .../singlecluster}/bin/start-ranger.sh | 0 .../singlecluster}/bin/start-yarn.sh | 0 .../singlecluster}/bin/start-zookeeper.sh | 0 .../singlecluster}/bin/stop-gphd.sh | 0 .../singlecluster}/bin/stop-hbase.sh | 0 .../singlecluster}/bin/stop-hdfs.sh | 0 .../singlecluster}/bin/stop-hive.sh | 0 .../singlecluster}/bin/stop-pxf.sh | 0 .../singlecluster}/bin/stop-ranger.sh | 0 .../singlecluster}/bin/stop-yarn.sh | 0 .../singlecluster}/bin/stop-zookeeper.sh | 0 .../singlecluster}/bin/yarn-nodemanager.sh | 0 .../singlecluster}/conf/gphd-conf.sh | 0 .../templates/hadoop/etc/hadoop/core-site.xml | 60 ++++++ .../templates/hadoop/etc/hadoop/hadoop-env.sh | 9 +- .../templates/hadoop/etc/hadoop/hdfs-site.xml | 8 + .../templates/hadoop/etc/hadoop/mapred-site.xml | 0 .../templates/hadoop/etc/hadoop/yarn-env.sh | 6 +- .../templates/hadoop/etc/hadoop/yarn-site.xml | 18 +- .../templates/hbase/conf/hbase-env.sh | 5 +- .../templates/hbase/conf/hbase-site.xml | 92 +++++++++ .../singlecluster/templates/hive/conf/hive-env.sh | 7 + .../templates/hive/conf/hive-site.xml | 23 ++- .../templates/ranger/install.properties | 0 .../singlecluster}/templates/tez/conf/tez-site.xml | 4 + .../templates/usersync/install.properties | 0 .../templates/zookeeper/conf/zoo.cfg | 0 singlecluster/Makefile | 121 ----------- singlecluster/README.md | 93 --------- singlecluster/package_singlecluster_cdh.yml | 16 -- singlecluster/package_singlecluster_hdp2.yml | 16 -- singlecluster/package_singlecluster_hdp3.yml | 16 -- singlecluster/product.version | 1 - singlecluster/singlecluster.bash | 17 -- .../templates/hadoop/etc/hadoop/core-site.xml | 32 --- singlecluster/templates/hbase/conf/hbase-site.xml | 49 ----- singlecluster/templates/hive/conf/hive-env.sh | 11 - singlecluster/tools/README.md | 40 ---- singlecluster/tools/compressHDP.sh | 48 ----- singlecluster/tools/downloadCDH.sh | 39 ---- singlecluster/tools/fetchBuild.sh | 51 ----- 72 files changed, 1013 insertions(+), 659 deletions(-) diff --git a/.github/workflows/pxf-ci.yml b/.github/workflows/pxf-ci.yml new file mode 100644 index 00000000..20775b57 --- /dev/null +++ b/.github/workflows/pxf-ci.yml @@ -0,0 +1,228 @@ +name: PXF CI Pipeline + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + types: [opened, synchronize, reopened, edited] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + JAVA_VERSION: "11" + JAVA_HOME: "/usr/lib/jvm/java-11-openjdk" + GPADMIN_HOME: "/home/gpadmin" + GO_VERSION: "1.21" + GPHOME: "/usr/local/cloudberry-db" + CLOUDBERRY_VERSION: "main" + PXF_HOME: "/usr/local/pxf" + +jobs: + pxf-build-install-test: + name: Build, Install & Test PXF + runs-on: ubuntu-latest + steps: + - name: Checkout Apache Cloudberry source + uses: actions/checkout@v4 + with: + repository: apache/cloudberry + ref: main + fetch-depth: 1 + persist-credentials: false + path: cloudberry + submodules: true + - name: Checkout Apache Cloudberry pxf source + uses: actions/checkout@v4 + with: + repository: MisterRaindrop/cloudberry-pxf + ref: liuxiaoyu/merge_2 + fetch-depth: 1 + persist-credentials: false + path: cloudberry-pxf + submodules: true + - name: Cache singlecluster Docker image + id: cache-singlecluster + uses: actions/cache@v4 + with: + path: /tmp/singlecluster-image.tar + key: singlecluster-${{ hashFiles('cloudberry-pxf/concourse/singlecluster/Dockerfile', 'cloudberry-pxf/concourse/singlecluster/**') }} + - name: Build singlecluster image + if: steps.cache-singlecluster.outputs.cache-hit != 'true' + run: | + cd ${{ github.workspace }}/cloudberry-pxf/concourse/singlecluster + docker build -t pxf/singlecluster:3 . + docker save pxf/singlecluster:3 > /tmp/singlecluster-image.tar + - name: Load singlecluster image + if: steps.cache-singlecluster.outputs.cache-hit == 'true' + run: | + docker load < /tmp/singlecluster-image.tar + - name: Run Test + run: | + cd ${{ github.workspace }}/cloudberry-pxf/concourse/docker/pxf-cbdb-dev/ubuntu/ + docker compose up -d + # Wait for container to be ready + sleep 10 + # Execute entrypoint script with correct working directory + docker compose exec -T pxf-cbdb-dev /bin/bash -c "cd /home/gpadmin/workspace/cloudberry-pxf/concourse/docker/pxf-cbdb-dev/ubuntu && ./script/entrypoint.sh" + - name: Extract test artifacts from container + if: always() + run: | + echo "Test results are already available in mounted volume:" + ls -la ${{ github.workspace }}/cloudberry-pxf/automation/test_artifacts/ || echo "No test_artifacts directory found" + + # Show summary if available + if [ -f ${{ github.workspace }}/cloudberry-pxf/automation/test_artifacts/summary.csv ]; then + echo "Test Summary:" + cat ${{ github.workspace }}/cloudberry-pxf/automation/test_artifacts/summary.csv + fi + + - name: Cleanup containers + if: always() + run: | + cd cloudberry-pxf/concourse/docker/pxf-cbdb-dev/ubuntu/ + docker compose down -v || true + + + - name: Save artifacts + if: always() + uses: actions/upload-artifact@v4 + id: upload_automation_step + with: + name: automation-test-results-pxf-cbdb-dev + path: | + ${{ github.workspace }}/cloudberry-pxf/automation/test_artifacts/ + retention-days: 30 + + - name: Evaluate module build/test results + if: success() || failure() + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + const path = require('path'); + + console.log('Processing test reports for PXF...'); + + // Start building the step summary + const summary = core.summary + .addHeading('PXF Test Results Summary') + .addHeading('๐ฆ Artifacts', 3) + .addLink('Raw Test Results', "${{ steps.upload_automation_step.outputs.artifact-url }}"); + + let hasErrors = false; + + // Check if test summary exists + const testSummaryPath = '${{ github.workspace }}/cloudberry-pxf/automation/test_artifacts/summary.csv'; + if (fs.existsSync(testSummaryPath)) { + try { + const csvContent = fs.readFileSync(testSummaryPath, 'utf8'); + const lines = csvContent.trim().split('\n'); + + if (lines.length > 1) { + // Parse CSV and create table + const headers = lines[0].split(','); + const rows = lines.slice(1).map(line => line.split(',')); + + // Add test results table + summary.addHeading('๐งช Test Results', 3); + summary.addTable([ + headers, + ...rows + ]); + + // Check for failures + let totalTests = 0; + let failedTests = 0; + let passedTests = 0; + + rows.forEach(row => { + totalTests++; + if (row[1] === 'FAIL') { + failedTests++; + hasErrors = true; + } else if (row[1] === 'PASS') { + passedTests++; + } + }); + + summary.addRaw(`\n\n**Summary**: ${totalTests} test components, ${passedTests} passed, ${failedTests} failed\n\n`); + + if (failedTests > 0) { + core.error(`${failedTests} test component(s) failed`); + } + } + } catch (error) { + console.log('Error processing test summary:', error.message); + core.error('Error processing test summary'); + hasErrors = true; + } + } else { + summary.addRaw('No test summary found\n\n'); + } + + // Check if TestNG results exist + const testReportsDir = '${{ github.workspace }}/cloudberry-pxf/automation/test_artifacts/surefire-reports'; + if (fs.existsSync(testReportsDir)) { + const testngResultsPath = path.join(testReportsDir, 'testng-results.xml'); + if (fs.existsSync(testngResultsPath)) { + try { + const xmlContent = fs.readFileSync(testngResultsPath, 'utf8'); + + // Extract test statistics using regex + const totalMatch = xmlContent.match(/total="(\d+)"/); + const passedMatch = xmlContent.match(/passed="(\d+)"/); + const failedMatch = xmlContent.match(/failed="(\d+)"/); + const skippedMatch = xmlContent.match(/skipped="(\d+)"/); + + const total = totalMatch ? totalMatch[1] : '0'; + const passed = passedMatch ? passedMatch[1] : '0'; + const failed = failedMatch ? failedMatch[1] : '0'; + const skipped = skippedMatch ? skippedMatch[1] : '0'; + + // Add TestNG statistics to summary + summary + .addHeading('๐ฌ Automation Test Details', 3) + .addTable([ + ['Metric', 'Count'], + ['Total Tests', total], + ['Passed', passed], + ['Failed', failed], + ['Skipped', skipped] + ]); + + // Check if there are failed tests + const failedCount = parseInt(failed) || 0; + const skippedCount = parseInt(skipped) || 0; + + if (failedCount > 0) { + core.error(`Automation tests failed: ${failedCount} test(s) failed`); + hasErrors = true; + } + if (skippedCount > 0) { + core.warning(`Automation tests incomplete: ${skippedCount} test(s) skipped`); + } + } catch (error) { + console.log('Error processing TestNG results:', error.message); + core.error('Error processing automation test results'); + hasErrors = true; + } + } + } + + // Write to step summary + await summary.write(); + + // Exit with error code if there were errors + if (hasErrors) { + process.exit(1); + } + + + diff --git a/automation/pom.xml b/automation/pom.xml index 3e4c2ae7..9479751f 100644 --- a/automation/pom.xml +++ b/automation/pom.xml @@ -27,8 +27,12 @@ <repositories> <repository> - <id>test-dependencies</id> - <url>artifactregistry://us-central1-maven.pkg.dev/data-gpdb-ud/pxf-automation-test</url> + <id>main</id> + <url>https://repo.maven.apache.org/maven2/</url> + </repository> + <repository> + <id>bincenter</id> + <url>https://maven.top-q.co.il/content/repositories/public/</url> </repository> </repositories> @@ -53,6 +57,9 @@ <plugin> <artifactId>maven-surefire-plugin</artifactId> <version>2.15</version> + <configuration> + <testFailureIgnore>true</testFailureIgnore> + </configuration> <executions> <execution> <id>default-test</id> @@ -156,10 +163,16 @@ <version>6.8.7</version> </dependency> + <dependency> + <groupId>org.awaitility</groupId> + <artifactId>awaitility</artifactId> + <version>4.2.0</version> + </dependency> + <dependency> <groupId>org.jsystemtest</groupId> <artifactId>jsystemCore</artifactId> - <version>6.0.01</version> + <version>6.1.06</version> <exclusions> <!-- javax.comm provides applications access to RS-232 hardware - not needed --> <exclusion> @@ -172,7 +185,7 @@ <dependency> <groupId>org.jsystemtest.systemobjects</groupId> <artifactId>cli</artifactId> - <version>6.0.01</version> + <version>6.1.06</version> <exclusions> <!-- javax.comm provides applications access to RS-232 hardware - not needed --> <exclusion> @@ -448,4 +461,4 @@ <version>1.9.5</version> </dependency> </dependencies> -</project> +</project> \ No newline at end of file diff --git a/automation/src/main/java/org/greenplum/pxf/automation/components/gpdb/Gpdb.java b/automation/src/main/java/org/greenplum/pxf/automation/components/gpdb/Gpdb.java index ecacb485..24134139 100755 --- a/automation/src/main/java/org/greenplum/pxf/automation/components/gpdb/Gpdb.java +++ b/automation/src/main/java/org/greenplum/pxf/automation/components/gpdb/Gpdb.java @@ -323,7 +323,7 @@ public class Gpdb extends DbSystemObject { sso.init(); - sso.runCommand("source $GPHOME/greenplum_path.sh"); + sso.runCommand("source $GPHOME/cloudberry_path.sh"); // psql do not return error code so use EXIT_CODE_NOT_EXISTS sso.runCommand("psql " + getDb(), ShellSystemObject.EXIT_CODE_NOT_EXISTS); diff --git a/automation/src/main/java/org/greenplum/pxf/automation/components/regress/Regress.java b/automation/src/main/java/org/greenplum/pxf/automation/components/regress/Regress.java index 79b522ca..cdf4563e 100644 --- a/automation/src/main/java/org/greenplum/pxf/automation/components/regress/Regress.java +++ b/automation/src/main/java/org/greenplum/pxf/automation/components/regress/Regress.java @@ -21,7 +21,7 @@ public class Regress extends ShellSystemObject { ReportUtils.startLevel(report, getClass(), "init"); regressRunner = new File("pxf_regress/pxf_regress").getAbsolutePath(); super.init(); - runCommand("source $GPHOME/greenplum_path.sh"); + runCommand("source $GPHOME/cloudberry_path.sh"); runCommand("cd " + new File(regressTestFolder).getAbsolutePath()); ReportUtils.stopLevel(report); } diff --git a/automation/src/test/java/org/greenplum/pxf/automation/features/orc/OrcReadTest.java b/automation/src/test/java/org/greenplum/pxf/automation/features/orc/OrcReadTest.java index 480d19db..46f6f34d 100644 --- a/automation/src/test/java/org/greenplum/pxf/automation/features/orc/OrcReadTest.java +++ b/automation/src/test/java/org/greenplum/pxf/automation/features/orc/OrcReadTest.java @@ -175,11 +175,11 @@ public class OrcReadTest extends BaseFeature { runSqlTest("features/orc/read/null_in_string"); } - @Test(groups = {"features", "gpdb", "security", "hcfs"}) - public void orcReadStringsContainingNullByte() throws Exception { - prepareReadableExternalTable("pxf_orc_null_in_string", ORC_NULL_IN_STRING_COLUMNS, hdfsPath + ORC_NULL_IN_STRING); - runTincTest("pxf.features.orc.read.null_in_string.runTest"); - } + // @Test(groups = {"features", "gpdb", "security", "hcfs"}) + // public void orcReadStringsContainingNullByte() throws Exception { + // prepareReadableExternalTable("pxf_orc_null_in_string", ORC_NULL_IN_STRING_COLUMNS, hdfsPath + ORC_NULL_IN_STRING); + // runTincTest("pxf.features.orc.read.null_in_string.runTest"); + // } private void prepareReadableExternalTable(String name, String[] fields, String path) throws Exception { prepareReadableExternalTable(name, fields, path, false); diff --git a/concourse/docker/pxf-cbdb-dev/ubuntu/docker-compose.yml b/concourse/docker/pxf-cbdb-dev/ubuntu/docker-compose.yml new file mode 100644 index 00000000..fd99049d --- /dev/null +++ b/concourse/docker/pxf-cbdb-dev/ubuntu/docker-compose.yml @@ -0,0 +1,23 @@ +services: +# hadoop + singlecluster: + build: + dockerfile: Dockerfile + context: ../../../singlecluster + image: pxf/singlecluster:3 + container_name: pxf_singlecluster + + pxf-cbdb-dev: + image: pxf/singlecluster:3 + container_name: pxf-cbdb-dev + hostname: cdw + depends_on: + - singlecluster + volumes: + - ../../../../../cloudberry-pxf:/home/gpadmin/workspace/cloudberry-pxf + - ../../../../../cloudberry:/home/gpadmin/workspace/cloudberry + command: ["tail", "-f", "/dev/null"] + +networks: + default: + name: pxf-cbdb-ci diff --git a/concourse/docker/pxf-cbdb-dev/ubuntu/script/build_cloudberrry.sh b/concourse/docker/pxf-cbdb-dev/ubuntu/script/build_cloudberrry.sh new file mode 100755 index 00000000..3a08dddc --- /dev/null +++ b/concourse/docker/pxf-cbdb-dev/ubuntu/script/build_cloudberrry.sh @@ -0,0 +1,133 @@ + +# Install sudo & git +sudo apt update && sudo apt install -y sudo git + +# Required configuration +## Add Cloudberry environment setup to .bashrc +echo -e '\n# Add Cloudberry entries +if [ -f /usr/local/cloudberry-db/cloudberry-env.sh ]; then + source /usr/local/cloudberry-db/cloudberry-env.sh +fi +## US English with UTF-8 character encoding +export LANG=en_US.UTF-8 +' >> /home/gpadmin/.bashrc +## Set up SSH for passwordless access +mkdir -p /home/gpadmin/.ssh +if [ ! -f /home/gpadmin/.ssh/id_rsa ]; then + ssh-keygen -t rsa -b 2048 -C 'apache-cloudberry-dev' -f /home/gpadmin/.ssh/id_rsa -N "" +fi +cat /home/gpadmin/.ssh/id_rsa.pub >> /home/gpadmin/.ssh/authorized_keys +## Set proper SSH directory permissions +chmod 700 /home/gpadmin/.ssh +chmod 600 /home/gpadmin/.ssh/authorized_keys +chmod 644 /home/gpadmin/.ssh/id_rsa.pub + +# Configure system settings +sudo tee /etc/security/limits.d/90-db-limits.conf << 'EOF' +## Core dump file size limits for gpadmin +gpadmin soft core unlimited +gpadmin hard core unlimited +## Open file limits for gpadmin +gpadmin soft nofile 524288 +gpadmin hard nofile 524288 +## Process limits for gpadmin +gpadmin soft nproc 131072 +gpadmin hard nproc 131072 +EOF + +# Verify resource limits +ulimit -a + +# Install basic system packages +sudo apt update +sudo apt install -y bison \ + bzip2 \ + cmake \ + curl \ + flex \ + gcc \ + g++ \ + iproute2 \ + iputils-ping \ + language-pack-en \ + locales \ + libapr1-dev \ + libbz2-dev \ + libcurl4-gnutls-dev \ + libevent-dev \ + libkrb5-dev \ + libipc-run-perl \ + libldap2-dev \ + libpam0g-dev \ + libprotobuf-dev \ + libreadline-dev \ + libssl-dev \ + libuv1-dev \ + liblz4-dev \ + libxerces-c-dev \ + libxml2-dev \ + libyaml-dev \ + libzstd-dev \ + libperl-dev \ + make \ + pkg-config \ + protobuf-compiler \ + python3-dev \ + python3-pip \ + python3-setuptools \ + rsync + +# Continue as gpadmin user + + +# Prepare the build environment for Apache Cloudberry +sudo rm -rf /usr/local/cloudberry-db +sudo chmod a+w /usr/local +mkdir -p /usr/local/cloudberry-db +sudo chown -R gpadmin:gpadmin /usr/local/cloudberry-db + +# Run configure +cd ~/workspace/cloudberry +./configure --prefix=/usr/local/cloudberry-db \ + --disable-external-fts \ + --enable-debug \ + --enable-cassert \ + --enable-debug-extensions \ + --enable-gpcloud \ + --enable-ic-proxy \ + --enable-mapreduce \ + --enable-orafce \ + --enable-orca \ + --disable-pax \ + --enable-pxf \ + --enable-tap-tests \ + --with-gssapi \ + --with-ldap \ + --with-libxml \ + --with-lz4 \ + --with-pam \ + --with-perl \ + --with-pgport=5432 \ + --with-python \ + --with-pythonsrc-ext \ + --with-ssl=openssl \ + --with-uuid=e2fs \ + --with-includes=/usr/include/xercesc + +# Build and install Cloudberry and its contrib modules +make -j$(nproc) -C ~/workspace/cloudberry +make -j$(nproc) -C ~/workspace/cloudberry/contrib +make install -C ~/workspace/cloudberry +make install -C ~/workspace/cloudberry/contrib + +# Verify the installation +/usr/local/cloudberry-db/bin/postgres --gp-version +/usr/local/cloudberry-db/bin/postgres --version +ldd /usr/local/cloudberry-db/bin/postgres + +# Set up a Cloudberry demo cluster +source /usr/local/cloudberry-db/cloudberry-env.sh +make create-demo-cluster -C ~/workspace/cloudberry +source ~/workspace/cloudberry/gpAux/gpdemo/gpdemo-env.sh +psql -P pager=off template1 -c 'SELECT * from gp_segment_configuration' +psql template1 -c 'SELECT version()' \ No newline at end of file diff --git a/concourse/docker/pxf-cbdb-dev/ubuntu/script/build_pxf.sh b/concourse/docker/pxf-cbdb-dev/ubuntu/script/build_pxf.sh new file mode 100755 index 00000000..a225382b --- /dev/null +++ b/concourse/docker/pxf-cbdb-dev/ubuntu/script/build_pxf.sh @@ -0,0 +1,35 @@ +export JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64 +export PATH=$JAVA_HOME/bin:$PATH +export GPHOME=/usr/local/cloudberry-db +export PATH=$GPHOME/bin:$PATH +source $GPHOME/cloudberry-env.sh + +sudo apt update +sudo apt install -y openjdk-11-jdk maven + +cd /home/gpadmin/workspace/cloudberry-pxf + +# Set Go environment +export GOPATH=$HOME/go +export PATH=$PATH:/usr/local/go/bin:$GOPATH/bin +mkdir -p $GOPATH +export PXF_HOME=/usr/local/pxf +mkdir -p $PXF_HOME + +# Build all PXF components +make all + +# Install PXF +make install + +# Set up PXF environment + +export PXF_BASE=$HOME/pxf-base +export PATH=$PXF_HOME/bin:$PATH + +# Initialize PXF +pxf prepare +pxf start + +# Verify PXF is running +pxf status \ No newline at end of file diff --git a/concourse/docker/pxf-cbdb-dev/ubuntu/script/entrypoint.sh b/concourse/docker/pxf-cbdb-dev/ubuntu/script/entrypoint.sh new file mode 100755 index 00000000..5e8e7b61 --- /dev/null +++ b/concourse/docker/pxf-cbdb-dev/ubuntu/script/entrypoint.sh @@ -0,0 +1,226 @@ +#!/bin/bash +set -e +set -x + +sudo apt-get update && \ + sudo apt-get install -y wget lsb-release locales openjdk-11-jre-headless openjdk-8-jre-headless iproute2 sudo && \ + sudo locale-gen en_US.UTF-8 && \ + sudo locale-gen ru_RU.CP1251 && \ + sudo locale-gen ru_RU.UTF-8 && \ + sudo update-locale LANG=en_US.UTF-8 + +export LANG=en_US.UTF-8 +export LANGUAGE=en_US:en +export LC_ALL=en_US.UTF-8 + +sudo apt-get install -y maven unzip openssh-server + +sudo localedef -c -i ru_RU -f CP1251 ru_RU.CP1251 + +sudo ssh-keygen -A && \ +sudo bash -c 'echo "PasswordAuthentication yes" >> /etc/ssh/sshd_config' && \ +sudo mkdir -p /etc/ssh/sshd_config.d && \ +sudo touch /etc/ssh/sshd_config.d/pxf-automation.conf && \ +sudo bash -c 'echo "KexAlgorithms +diffie-hellman-group-exchange-sha1,diffie-hellman-group14-sha1,diffie-hellman-group1-sha1" >> /etc/ssh/sshd_config.d/pxf-automation.conf' && \ +sudo bash -c 'echo "HostKeyAlgorithms +ssh-rsa,ssh-dss" >> /etc/ssh/sshd_config.d/pxf-automation.conf' && \ +sudo bash -c 'echo "PubkeyAcceptedAlgorithms +ssh-rsa,ssh-dss" >> /etc/ssh/sshd_config.d/pxf-automation.conf' + +sudo usermod -a -G sudo gpadmin && \ +echo "gpadmin:cbdb@123" | sudo chpasswd && \ +echo "gpadmin ALL=(ALL) NOPASSWD: ALL" | sudo tee -a /etc/sudoers && \ +echo "root ALL=(ALL) NOPASSWD: ALL" | sudo tee -a /etc/sudoers + + +mkdir -p /home/gpadmin/.ssh && \ +sudo chown -R gpadmin:gpadmin /home/gpadmin/.ssh && \ +sudo -u gpadmin ssh-keygen -t rsa -b 4096 -m PEM -C gpadmin -f /home/gpadmin/.ssh/id_rsa -P "" && \ +sudo -u gpadmin bash -c 'cat /home/gpadmin/.ssh/id_rsa.pub >> /home/gpadmin/.ssh/authorized_keys' && \ +sudo -u gpadmin chmod 0600 /home/gpadmin/.ssh/authorized_keys + +# ---------------------------------------------------------------------- +# Start SSH daemon and setup for SSH access +# ---------------------------------------------------------------------- +# The SSH daemon is started to allow remote access to the container via +# SSH. This is useful for development and debugging purposes. If the SSH +# daemon fails to start, the script exits with an error. +# ---------------------------------------------------------------------- +if [ ! -d /var/run/sshd ]; then + sudo mkdir /var/run/sshd + sudo chmod 0755 /var/run/sshd +fi +if ! sudo /usr/sbin/sshd; then + echo "Failed to start SSH daemon" + exit 1 +fi + +# ---------------------------------------------------------------------- +# Remove /run/nologin to allow logins for all users via SSH +# ---------------------------------------------------------------------- +sudo rm -rf /run/nologin + +# ---------------------------------------------------------------------- +# Configure /home/gpadmin +# ---------------------------------------------------------------------- +mkdir -p /home/gpadmin/.ssh/ +ssh-keyscan -t rsa cdw > /home/gpadmin/.ssh/known_hosts +chown -R gpadmin:gpadmin /home/gpadmin/.ssh/ + +# ---------------------------------------------------------------------- +# Build Cloudberry +# ---------------------------------------------------------------------- +sudo chown -R gpadmin:gpadmin /home/gpadmin/workspace/ +./script/build_cloudberrry.sh + + +# ---------------------------------------------------------------------- +# Build pxf +# ---------------------------------------------------------------------- +./script/build_pxf.sh + + +# ---------------------------------------------------------------------- +# Source pxf env +# ---------------------------------------------------------------------- +source ./script/pxf-env.sh + +# ---------------------------------------------------------------------- +# Prepare PXF +# ---------------------------------------------------------------------- +export JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64 +export PATH="$PXF_HOME/bin:$PATH" +export PXF_JVM_OPTS="-Xmx512m -Xms256m" +export PXF_HOST=localhost # 0.0.0.0 # listen on all interfaces + +# Prepare a new $PXF_BASE directory on each Greenplum Database host. +# - create directory structure in $PXF_BASE +# - copy configuration files from $PXF_HOME/conf to $PXF_BASE/conf +#/usr/local/pxf/bin/pxf cluster prepare + +# Use Java 11: +echo "JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64" >> $PXF_BASE/conf/pxf-env.sh +# Configure PXF to listen on all interfaces +sed -i 's/# server.address=localhost/server.address=0.0.0.0/' $PXF_BASE/conf/pxf-application.properties +# add property to allow dynamic test: profiles that are used when testing against FDW +echo -e "\npxf.profile.dynamic.regex=test:.*" >> $PXF_BASE/conf/pxf-application.properties +# set up pxf configs from templates +cp -v $PXF_HOME/templates/{hdfs,mapred,yarn,core,hbase,hive}-site.xml $PXF_BASE/servers/default + +# Register PXF extension in Greenplum +# - Copy the PXF extension control file from the PXF installation on each host to the Greenplum installation on the host +#/usr/local/pxf/bin/pxf cluster register +# # Start PXF +#/usr/local/pxf/bin/pxf cluster start + +# ---------------------------------------------------------------------- +# Prepare Hadoop +# ---------------------------------------------------------------------- +# FIXME: reuse old scripts +cd /home/gpadmin/workspace/cloudberry-pxf/automation +make symlink_pxf_jars +cp /home/gpadmin/automation_tmp_lib/pxf-hbase.jar $GPHD_ROOT/hbase/lib/ + +$GPHD_ROOT/bin/init-gphd.sh +$GPHD_ROOT/bin/start-gphd.sh + +# -------------------------------------------------------------------- +# Run tests independently and collect results +# -------------------------------------------------------------------- +# create GOCACHE directory for gpadmin user +sudo mkdir -p /home/gpadmin/.cache/go-build +sudo chown -R gpadmin:gpadmin /home/gpadmin/.cache +sudo chmod -R 755 /home/gpadmin/.cache +# create .m2 cache directory +sudo mkdir -p /home/gpadmin/.m2 +sudo chown -R gpadmin:gpadmin /home/gpadmin/.m2 +sudo chmod -R 755 /home/gpadmin/.m2 + +# Output results directly to mounted automation directory +TEST_RESULTS_DIR="/home/gpadmin/workspace/cloudberry-pxf/automation/test_artifacts" +mkdir -p "$TEST_RESULTS_DIR" +echo "Test Component,Status,Duration,Details" > "$TEST_RESULTS_DIR/summary.csv" + +# Function to run test and record result +run_test() { + local component="$1" + local test_dir="$2" + local test_cmd="$3" + local start_time=$(date +%s) + local log_file="$TEST_RESULTS_DIR/${component}.log" + + echo "Running $component tests..." + cd "$test_dir" + + # Run the test and capture both exit code and output + if eval "$test_cmd" > "$log_file" 2>&1; then + local exit_code=0 + else + local exit_code=$? + fi + + # Check for specific failure patterns in the log + local status="PASS" + local details="All tests passed" + + if [ $exit_code -ne 0 ]; then + status="FAIL" + details="Exit code: $exit_code. Check ${component}.log for details" + elif grep -q "There are test failures\|BUILD FAILURE\|FAILED\|Failures: [1-9]" "$log_file"; then + status="FAIL" + details="Test failures detected. Check ${component}.log for details" + elif grep -q "Tests run:.*Failures: [1-9]" "$log_file"; then + status="FAIL" + details="Test failures detected. Check ${component}.log for details" + fi + + local end_time=$(date +%s) + local duration=$((end_time - start_time)) + + echo "$component,$status,${duration}s,$details" >> "$TEST_RESULTS_DIR/summary.csv" + echo "$component: $status (${duration}s)" +} + +# Run CLI tests +run_test "CLI" "/home/gpadmin/workspace/cloudberry-pxf/cli" "make test" + +# Run External Table tests +run_test "External-Table" "/home/gpadmin/workspace/cloudberry-pxf/external-table" "make installcheck" + +# Run Server tests +run_test "Server" "/home/gpadmin/workspace/cloudberry-pxf/server" "./gradlew test" + +# Run Automation setup +run_test "Automation-Setup" "/home/gpadmin/workspace/cloudberry-pxf/automation" "make" + +# Run Smoke tests +run_test "Smoke-Test" "/home/gpadmin/workspace/cloudberry-pxf/automation" "make TEST=HdfsSmokeTest" + +# Run GPDB group tests (allow failure) +run_test "GPDB-Group" "/home/gpadmin/workspace/cloudberry-pxf/automation" "make GROUP=gpdb" + +# Copy additional test artifacts to mounted directory +echo "Collecting additional test artifacts..." + +# Copy PXF logs +mkdir -p "$TEST_RESULTS_DIR/pxf_logs" +cp -r ~/pxf-base/logs/* "$TEST_RESULTS_DIR/pxf_logs/" 2>/dev/null || true + +# Copy server test reports +mkdir -p "$TEST_RESULTS_DIR/server_reports" +cp -r ~/workspace/cloudberry-pxf/server/build/reports/tests/test/* "$TEST_RESULTS_DIR/server_reports/" 2>/dev/null || true + +# Copy automation surefire reports (if they exist) +if [ -d ~/workspace/cloudberry-pxf/automation/target/surefire-reports ]; then + cp -r ~/workspace/cloudberry-pxf/automation/target/surefire-reports "$TEST_RESULTS_DIR/" +fi + +# Copy automation logs (if they exist) +if [ -d ~/workspace/cloudberry-pxf/automation/automation_logs ]; then + cp -r ~/workspace/cloudberry-pxf/automation/automation_logs "$TEST_RESULTS_DIR/" +fi + +echo "Test execution completed. Results available in $TEST_RESULTS_DIR" +ls -la "$TEST_RESULTS_DIR" + + +# Keep container running +#tail -f /dev/null \ No newline at end of file diff --git a/concourse/docker/pxf-cbdb-dev/ubuntu/script/pxf-env.sh b/concourse/docker/pxf-cbdb-dev/ubuntu/script/pxf-env.sh new file mode 100755 index 00000000..c366ab96 --- /dev/null +++ b/concourse/docker/pxf-cbdb-dev/ubuntu/script/pxf-env.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# PXF Environment Variables +export JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64 +export PATH=$JAVA_HOME/bin:$PATH +export GPHOME=/usr/local/cloudberry-db +export PATH=$GPHOME/bin:$PATH +export GOPATH=$HOME/go +export PATH=$PATH:/usr/local/go/bin:$GOPATH/bin +export PXF_HOME=/usr/local/pxf +export PXF_BASE=$HOME/pxf-base +export PATH=$PXF_HOME/bin:$PATH + +# Source Cloudberry environment +if [ -f "$GPHOME/cloudberry-env.sh" ]; then + source $GPHOME/cloudberry-env.sh +fi + +# Source demo cluster environment if available +if [ -f "/home/gpadmin/workspace/cloudberry/gpAux/gpdemo/gpdemo-env.sh" ]; then + source /home/gpadmin/workspace/cloudberry/gpAux/gpdemo/gpdemo-env.sh +fi + +echo "PXF environment loaded successfully" + diff --git a/concourse/singlecluster/Dockerfile b/concourse/singlecluster/Dockerfile new file mode 100644 index 00000000..d105038b --- /dev/null +++ b/concourse/singlecluster/Dockerfile @@ -0,0 +1,79 @@ +FROM apache/incubator-cloudberry:cbdb-build-ubuntu22.04-latest + +ENV DEBIAN_FRONTEND noninteractive + +RUN sudo apt-get update && \ + sudo apt-get install -y --no-install-recommends \ + curl ca-certificates + +# TODO: update hive to support java 11+ +ENV HADOOP_VERSION=3.1.2 +ENV HIVE_VERSION=3.1.3 +ENV ZOOKEEPER_VERSION=3.5.9 +ENV HBASE_VERSION=2.0.6 +ENV TEZ_VERSION=0.9.2 + +# checksums from archive.apache.org +ENV HADOOP_SHA512="0e0ee817c89b3c4eb761eca7f16640742a83b0e99b6fda26c1bee2baabedad93aab86e252bf5f1e2381c6d464bc4003d10c7cc0f61b2062f4c59732ca24d1bd9" +ENV HIVE_SHA256="0c9b6a6359a7341b6029cc9347435ee7b379f93846f779d710b13f795b54bb16" +ENV ZOOKEEPER_SHA512="0e5a64713abc6f36d961dd61a06f681868171a9d9228366e512a01324806d263e05508029c94d8e18307811867cdc39d848e736c252bf56c461273ef74c66a45" +ENV HBASE_SHA512="a0e10904ecf7f059b77bc0ce704254046a978126db720cc7e55dc53b87097715da64b8391fe3cc94348bc432871ad8f29891dc8df1ea052eb628da0fdca97c93" +ENV TEZ_SHA512="a2d94bd9fa778d42a8bac9d9da8e263e469ddfef93968b06434716554995f490231de5607541ac236e770aa0158b64250c38bc1cd57dbfa629fea705f2ffa2f5" + +# faster mirror: +ENV APACHE_MIRROR="repo.huaweicloud.com/apache" +#ENV APACHE_MIRROR="archive.apache.org/dist/" +#ENV APACHE_MIRROR="mirror.yandex.ru/mirrors/apache/" + +ENV HADOOP_URL="https://$APACHE_MIRROR/hadoop/common/hadoop-$HADOOP_VERSION/hadoop-$HADOOP_VERSION.tar.gz"; +ENV HIVE_URL="https://$APACHE_MIRROR/hive/hive-$HIVE_VERSION/apache-hive-$HIVE_VERSION-bin.tar.gz"; +ENV ZOOKEEPER_URL="https://$APACHE_MIRROR/zookeeper/zookeeper-$ZOOKEEPER_VERSION/apache-zookeeper-$ZOOKEEPER_VERSION-bin.tar.gz"; +ENV HBASE_URL="https://$APACHE_MIRROR/hbase/$HBASE_VERSION/hbase-$HBASE_VERSION-bin.tar.gz"; +ENV TEZ_URL="https://$APACHE_MIRROR/tez/$TEZ_VERSION/apache-tez-$TEZ_VERSION-bin.tar.gz"; + +ENV GPHD_ROOT=/home/gpadmin/workspace/singlecluster +ENV HADOOP_ROOT=$GPHD_ROOT/hadoop +ENV HBASE_ROOT=$GPHD_ROOT/hbase +ENV HIVE_ROOT=$GPHD_ROOT/hive +ENV ZOOKEEPER_ROOT=$GPHD_ROOT/zookeeper +ENV TEZ_ROOT=$GPHD_ROOT/tez + +RUN mkdir -p $HADOOP_ROOT && \ + curl -fSL "$HADOOP_URL" -o hadoop.tar.gz && \ + echo "$HADOOP_SHA512 hadoop.tar.gz" | sha512sum -c && \ + tar xvf hadoop.tar.gz -C $HADOOP_ROOT --strip-components 1 --exclude="share/doc/*" --exclude="*-sources.jar" && \ + rm hadoop.tar.gz + +RUN mkdir -p $HIVE_ROOT && \ + curl -fSL $HIVE_URL -o hive.tar.gz && \ + echo "$HIVE_SHA256 hive.tar.gz" | sha256sum -c && \ + tar xvf hive.tar.gz -C $HIVE_ROOT --strip-components 1 && \ + rm hive.tar.gz + +RUN mkdir -p $ZOOKEEPER_ROOT && \ + curl -fSL $ZOOKEEPER_URL -o zookeeper.tar.gz && \ + echo "$ZOOKEEPER_SHA512 zookeeper.tar.gz" | sha512sum -c && \ + tar xvf zookeeper.tar.gz -C $ZOOKEEPER_ROOT --strip-components 1 --exclude="docs/*" && \ + rm zookeeper.tar.gz + +RUN mkdir -p $HBASE_ROOT && \ + curl -fSL "$HBASE_URL" -o hbase.tar.gz && \ + echo "$HBASE_SHA512 hbase.tar.gz" | sha512sum -c && \ + tar xvf hbase.tar.gz -C $HBASE_ROOT --strip-components 1 --exclude="docs/*" && \ + rm hbase.tar.gz + +RUN mkdir -p $TEZ_ROOT && \ + curl -fSL "$TEZ_URL" -o tez.tar.gz && \ + echo "$TEZ_SHA512 tez.tar.gz" | sha512sum -c && \ + tar xvf tez.tar.gz -C $TEZ_ROOT --strip-components 1 && \ + rm tez.tar.gz + +# Install Go (required by PXF). +RUN mkdir -p /tmp/pxf_src/ && cd /tmp \ + && wget -O go.tgz -q https://go.dev/dl/go1.23.3.linux-amd64.tar.gz \ + && sudo rm -rf /usr/local/go && sudo tar -C /usr/local -xzf go.tgz && rm go.tgz + + +COPY ./templates $GPHD_ROOT +COPY ./conf $GPHD_ROOT/conf +COPY ./bin $GPHD_ROOT/bin \ No newline at end of file diff --git a/singlecluster/README.HDP3.md b/concourse/singlecluster/README.HDP3.md similarity index 57% rename from singlecluster/README.HDP3.md rename to concourse/singlecluster/README.HDP3.md index 4bd89525..16506de1 100644 --- a/singlecluster/README.HDP3.md +++ b/concourse/singlecluster/README.HDP3.md @@ -1,14 +1,14 @@ Singlecluster-HDP3 ================== -Singlecluster-HDP3 is a self-contained, easy to deploy distribution of HDP3 (3.1.4.0-315) +Singlecluster-HDP3 is a self-contained, easy to deploy distribution of HDP3 It contains the following versions: -- Hadoop 3.1.1 -- Hive 3.1.0 -- Zookeeper 3.4.6 -- HBase 2.0.2 -- Tez 0.9.1 +- Hadoop 3.3.6 +- Hive 3.1.3 +- Zookeeper 3.5.9 +- HBase 2.0.6 +- Tez 0.9.2 This version of Single cluster requires users to make some manual changes to the configuration files once the tarball has been unpacked (see Initialization steps below). @@ -22,83 +22,19 @@ Initialization 1. Make sure **all** running instances of other singlecluster processes are stopped. -2. Pull down the singlecluster-HDP3 tarball from GCP and untar: +2. Pull down the singlecluster-HDP3 components: ```sh - mv singlecluster-HDP3.tar.gz ~/workspace - cd ~/workspace - mkdir singlecluster-HDP3 - tar -xf singlecluster-HDP3.tar.gz --strip-components=1 --directory=singlecluster-HDP3 - cd singlecluster-HDP3 - export GPHD_ROOT="${PWD}" + docker compose build singlecluster ``` -3. Adjust the configuration for Hadoop 3 (the following steps are based on the function `adjust_for_hadoop3` in `pxf_common.bash`) - - 1. In `${GPHD_ROOT}/hive/conf/hive-env.sh`, remove `-hiveconf hive.log.dir=$LOGS_ROOT` from the `HIVE_OPTS` and `HIVE_SERVER_OPTS` exports: - - ```sh - sed -i -e 's/-hiveconf hive.log.dir=$LOGS_ROOT//' singlecluster-HDP3/hive/conf/hive-env.sh - ``` - - 2. Update the `hive.execution.engine` property to `tez` in `${GPHD_ROOT}/hive/conf/hive-site.xml`: - - ```sh - sed -e '/hive.execution.engine/{n;s/>.*</>tez</}' singlecluster-HDP3/hive/conf/hive-site.xml - ``` - - 3. Add the following properties to `${GPHD_ROOT}/hive/conf/hive-site.xml`: - - ```xml - <property> - <name>hive.tez.container.size</name> - <value>2048</value> - </property> - <property> - <name>datanucleus.schema.autoCreateAll</name> - <value>True</value> - </property> - <property> - <name>metastore.metastore.event.db.notification.api.auth</name> - <value>false</value> - </property> - ``` - - 4. Add the following property to `"${GPHD_ROOT}/tez/conf/tez-site.xml`: - - ```xml - <property> - <name>tez.use.cluster.hadoop-libs</name> - <value>true</value> - </property> - ``` - - 5. Replace `HADOOP_CONF` with `HADOOP_CONF_DIR` and `HADOOP_ROOT` with `HADOOP_HOME` in `${GPHD_ROOT}/hadoop/etc/hadoop/yarn-site.xml`: - - ```sh - sed -i.bak -e 's|HADOOP_CONF|HADOOP_CONF_DIR|g' \ - -e 's|HADOOP_ROOT|HADOOP_HOME|g' "${GPHD_ROOT}/hadoop/etc/hadoop/yarn-site.xml" - ``` - - 6. Replace `HADOOP_NAMENODE_OPTS` with `HDFS_NAMENODE_OPTS` in `${GPHD_ROOT}/hadoop/etc/hadoop/hadoop-env.sh`: - - ```sh - sed -i.bak -e 's/HADOOP_NAMENODE_OPTS/HDFS_NAMENODE_OPTS/g' "${GPHD_ROOT}/hadoop/etc/hadoop/hadoop-env.sh" - ``` - - 7. Replace `HADOOP_DATANODE_OPTS` with `HDFS_DATANODE_OPTS` in `${GPHD_ROOT}/bin/hadoop-datanode.sh`: - - ```sh - sed -i.bak -e 's/HADOOP_DATANODE_OPTS/HDFS_DATANODE_OPTS/g' "${GPHD_ROOT}/bin/hadoop-datanode.sh" - ``` - -4. Initialize an instance +3. Initialize an instance ```sh ${GPHD_ROOT}/bin/init-gphd.sh ``` -5. Add the following to your environment +4. Add the following to your environment ```sh export HADOOP_ROOT=$GPHD_ROOT/hadoop @@ -185,4 +121,4 @@ If it is not running, spin up YARN before starting a new Hive session. You can view the status of your hive server as well as your YARN resources by going to the following: - `localhost:10002` will show the status of the HiveServer2. This includes running and completed queries, and active sessions. -- `localhost:8088` willl show the status of the YARN resource manager. This includes cluster metrics and cluster node statuses. +- `localhost:8088` will show the status of the YARN resource manager. This includes cluster metrics and cluster node statuses. diff --git a/singlecluster/bin/gphd-env.sh b/concourse/singlecluster/bin/gphd-env.sh similarity index 96% rename from singlecluster/bin/gphd-env.sh rename to concourse/singlecluster/bin/gphd-env.sh index 63c3f8b2..79f2714e 100755 --- a/singlecluster/bin/gphd-env.sh +++ b/concourse/singlecluster/bin/gphd-env.sh @@ -53,8 +53,12 @@ export TEZ_JARS=$(echo "$TEZ_ROOT"/*.jar | tr ' ' ':'):$(echo "$TEZ_ROOT"/lib/*. function cluster_initialized() { if [ -d ${HADOOP_STORAGE_ROOT}/dfs/name ]; then + echo "a" return 0 else + echo "ba" + echo $HADOOP_STORAGE_ROOT + echo $GPHD_CONF return 1 fi } @@ -78,3 +82,4 @@ function zookeeper_running() done return ${retval} } + diff --git a/singlecluster/bin/hadoop b/concourse/singlecluster/bin/hadoop similarity index 100% rename from singlecluster/bin/hadoop rename to concourse/singlecluster/bin/hadoop diff --git a/singlecluster/bin/hadoop-datanode.sh b/concourse/singlecluster/bin/hadoop-datanode.sh similarity index 96% rename from singlecluster/bin/hadoop-datanode.sh rename to concourse/singlecluster/bin/hadoop-datanode.sh index 516297cc..15fa72dc 100755 --- a/singlecluster/bin/hadoop-datanode.sh +++ b/concourse/singlecluster/bin/hadoop-datanode.sh @@ -18,7 +18,7 @@ bin=${root}/bin datanode_root=${HADOOP_STORAGE_ROOT}/datanode${nodeid} datanode_conf=${datanode_root}/etc/hadoop -export HADOOP_DATANODE_OPTS="-Dhadoop.tmp.dir=$datanode_root/data" +export HDFS_DATANODE_OPTS="-Dhadoop.tmp.dir=$datanode_root/data" export HADOOP_CONF_DIR=${datanode_conf} export HADOOP_IDENT_STRING=${USER}-node${nodeid} diff --git a/singlecluster/bin/hbase b/concourse/singlecluster/bin/hbase similarity index 100% rename from singlecluster/bin/hbase rename to concourse/singlecluster/bin/hbase diff --git a/singlecluster/bin/hbase-regionserver.sh b/concourse/singlecluster/bin/hbase-regionserver.sh similarity index 100% rename from singlecluster/bin/hbase-regionserver.sh rename to concourse/singlecluster/bin/hbase-regionserver.sh diff --git a/singlecluster/bin/hdfs b/concourse/singlecluster/bin/hdfs similarity index 100% rename from singlecluster/bin/hdfs rename to concourse/singlecluster/bin/hdfs diff --git a/singlecluster/bin/hive b/concourse/singlecluster/bin/hive similarity index 100% rename from singlecluster/bin/hive rename to concourse/singlecluster/bin/hive diff --git a/singlecluster/bin/hive-service.sh b/concourse/singlecluster/bin/hive-service.sh similarity index 100% rename from singlecluster/bin/hive-service.sh rename to concourse/singlecluster/bin/hive-service.sh diff --git a/singlecluster/bin/init-gphd.sh b/concourse/singlecluster/bin/init-gphd.sh similarity index 100% rename from singlecluster/bin/init-gphd.sh rename to concourse/singlecluster/bin/init-gphd.sh diff --git a/singlecluster/bin/init-pxf.sh b/concourse/singlecluster/bin/init-pxf.sh similarity index 100% rename from singlecluster/bin/init-pxf.sh rename to concourse/singlecluster/bin/init-pxf.sh diff --git a/singlecluster/bin/init-ranger.sh b/concourse/singlecluster/bin/init-ranger.sh similarity index 87% rename from singlecluster/bin/init-ranger.sh rename to concourse/singlecluster/bin/init-ranger.sh index a84c4f24..dab853ee 100755 --- a/singlecluster/bin/init-ranger.sh +++ b/concourse/singlecluster/bin/init-ranger.sh @@ -1,9 +1,7 @@ #!/usr/bin/env bash # Load settings -root=`cd \`dirname $0\`/..;pwd` -bin=$root/bin -. $bin/gphd-env.sh +. $GPHD_ROOT/bin/gphd-env.sh if [ "Darwin" == $(uname -s) ]; then echo "Ranger script is not supported on OSX" diff --git a/singlecluster/bin/pxf-service.sh b/concourse/singlecluster/bin/pxf-service.sh similarity index 98% rename from singlecluster/bin/pxf-service.sh rename to concourse/singlecluster/bin/pxf-service.sh index 3b7b95dd..2255e233 100755 --- a/singlecluster/bin/pxf-service.sh +++ b/concourse/singlecluster/bin/pxf-service.sh @@ -17,9 +17,7 @@ command=$1 nodeid=$2 # Load settings -root=`cd \`dirname $0\`/..;pwd` -bin=$root/bin -. $bin/gphd-env.sh +. $GPHD_ROOT/bin/gphd-env.sh instance_root=$PXF_STORAGE_ROOT/pxf$nodeid instance_name=pxf-service-$nodeid diff --git a/singlecluster/bin/restart-gphd.sh b/concourse/singlecluster/bin/restart-gphd.sh similarity index 100% rename from singlecluster/bin/restart-gphd.sh rename to concourse/singlecluster/bin/restart-gphd.sh diff --git a/singlecluster/bin/restart-pxf.sh b/concourse/singlecluster/bin/restart-pxf.sh similarity index 100% rename from singlecluster/bin/restart-pxf.sh rename to concourse/singlecluster/bin/restart-pxf.sh diff --git a/singlecluster/bin/start-gphd.sh b/concourse/singlecluster/bin/start-gphd.sh similarity index 100% rename from singlecluster/bin/start-gphd.sh rename to concourse/singlecluster/bin/start-gphd.sh diff --git a/singlecluster/bin/start-hbase.sh b/concourse/singlecluster/bin/start-hbase.sh similarity index 100% rename from singlecluster/bin/start-hbase.sh rename to concourse/singlecluster/bin/start-hbase.sh diff --git a/singlecluster/bin/start-hdfs.sh b/concourse/singlecluster/bin/start-hdfs.sh similarity index 87% rename from singlecluster/bin/start-hdfs.sh rename to concourse/singlecluster/bin/start-hdfs.sh index b7472e6d..9a4c4731 100755 --- a/singlecluster/bin/start-hdfs.sh +++ b/concourse/singlecluster/bin/start-hdfs.sh @@ -25,3 +25,7 @@ done # Wait for Namenode to leave safemode ${HADOOP_BIN}/hdfs dfsadmin -safemode wait || sleep 5 + +# Report HDFS status +${HADOOP_BIN}/hdfs dfsadmin -report +${HADOOP_BIN}/hdfs fsck / \ No newline at end of file diff --git a/singlecluster/bin/start-hive.sh b/concourse/singlecluster/bin/start-hive.sh similarity index 100% rename from singlecluster/bin/start-hive.sh rename to concourse/singlecluster/bin/start-hive.sh diff --git a/singlecluster/bin/start-pxf.sh b/concourse/singlecluster/bin/start-pxf.sh similarity index 100% rename from singlecluster/bin/start-pxf.sh rename to concourse/singlecluster/bin/start-pxf.sh diff --git a/singlecluster/bin/start-ranger.sh b/concourse/singlecluster/bin/start-ranger.sh similarity index 100% rename from singlecluster/bin/start-ranger.sh rename to concourse/singlecluster/bin/start-ranger.sh diff --git a/singlecluster/bin/start-yarn.sh b/concourse/singlecluster/bin/start-yarn.sh similarity index 100% rename from singlecluster/bin/start-yarn.sh rename to concourse/singlecluster/bin/start-yarn.sh diff --git a/singlecluster/bin/start-zookeeper.sh b/concourse/singlecluster/bin/start-zookeeper.sh similarity index 100% rename from singlecluster/bin/start-zookeeper.sh rename to concourse/singlecluster/bin/start-zookeeper.sh diff --git a/singlecluster/bin/stop-gphd.sh b/concourse/singlecluster/bin/stop-gphd.sh similarity index 100% rename from singlecluster/bin/stop-gphd.sh rename to concourse/singlecluster/bin/stop-gphd.sh diff --git a/singlecluster/bin/stop-hbase.sh b/concourse/singlecluster/bin/stop-hbase.sh similarity index 100% rename from singlecluster/bin/stop-hbase.sh rename to concourse/singlecluster/bin/stop-hbase.sh diff --git a/singlecluster/bin/stop-hdfs.sh b/concourse/singlecluster/bin/stop-hdfs.sh similarity index 100% rename from singlecluster/bin/stop-hdfs.sh rename to concourse/singlecluster/bin/stop-hdfs.sh diff --git a/singlecluster/bin/stop-hive.sh b/concourse/singlecluster/bin/stop-hive.sh similarity index 100% rename from singlecluster/bin/stop-hive.sh rename to concourse/singlecluster/bin/stop-hive.sh diff --git a/singlecluster/bin/stop-pxf.sh b/concourse/singlecluster/bin/stop-pxf.sh similarity index 100% rename from singlecluster/bin/stop-pxf.sh rename to concourse/singlecluster/bin/stop-pxf.sh diff --git a/singlecluster/bin/stop-ranger.sh b/concourse/singlecluster/bin/stop-ranger.sh similarity index 100% rename from singlecluster/bin/stop-ranger.sh rename to concourse/singlecluster/bin/stop-ranger.sh diff --git a/singlecluster/bin/stop-yarn.sh b/concourse/singlecluster/bin/stop-yarn.sh similarity index 100% rename from singlecluster/bin/stop-yarn.sh rename to concourse/singlecluster/bin/stop-yarn.sh diff --git a/singlecluster/bin/stop-zookeeper.sh b/concourse/singlecluster/bin/stop-zookeeper.sh similarity index 100% rename from singlecluster/bin/stop-zookeeper.sh rename to concourse/singlecluster/bin/stop-zookeeper.sh diff --git a/singlecluster/bin/yarn-nodemanager.sh b/concourse/singlecluster/bin/yarn-nodemanager.sh similarity index 100% rename from singlecluster/bin/yarn-nodemanager.sh rename to concourse/singlecluster/bin/yarn-nodemanager.sh diff --git a/singlecluster/templates/conf/gphd-conf.sh b/concourse/singlecluster/conf/gphd-conf.sh similarity index 100% rename from singlecluster/templates/conf/gphd-conf.sh rename to concourse/singlecluster/conf/gphd-conf.sh diff --git a/concourse/singlecluster/templates/hadoop/etc/hadoop/core-site.xml b/concourse/singlecluster/templates/hadoop/etc/hadoop/core-site.xml new file mode 100755 index 00000000..3d7b3881 --- /dev/null +++ b/concourse/singlecluster/templates/hadoop/etc/hadoop/core-site.xml @@ -0,0 +1,60 @@ +<?xml version="1.0" encoding="UTF-8"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. See accompanying LICENSE file. +--> + +<!-- Put site-specific property overrides in this file. --> + +<configuration> + <property> + <name>fs.defaultFS</name> + <value>hdfs://0.0.0.0:8020</value> + </property> + <property> + <name>ipc.ping.interval</name> + <value>900000</value> + </property> + <property> + <name>hadoop.proxyuser.gpadmin.hosts</name> + <value>*</value> + </property> + <property> + <name>hadoop.proxyuser.gpadmin.groups</name> + <value>*</value> + </property> + <property> + <name>hadoop.security.authorization</name> + <value>true</value> + </property> + <property> + <name>hbase.security.authorization</name> + <value>true</value> + </property> + <property> + <name>hbase.rpc.protection</name> + <value>authentication</value> + </property> + <property> + <name>hbase.coprocessor.master.classes</name> + <value>org.apache.hadoop.hbase.security.access.AccessController</value> + </property> + <property> + <name>hbase.coprocessor.region.classes</name> + <value>org.apache.hadoop.hbase.security.access.AccessController,org.apache.hadoop.hbase.security.access.SecureBulkLoadEndpoint</value> + </property> + <property> + <name>hbase.coprocessor.regionserver.classes</name> + <value>org.apache.hadoop.hbase.security.access.AccessController</value> + </property> +</configuration> diff --git a/singlecluster/templates/hadoop/etc/hadoop/hadoop-env.sh b/concourse/singlecluster/templates/hadoop/etc/hadoop/hadoop-env.sh similarity index 65% rename from singlecluster/templates/hadoop/etc/hadoop/hadoop-env.sh rename to concourse/singlecluster/templates/hadoop/etc/hadoop/hadoop-env.sh index c39d1a35..7ce50a38 100755 --- a/singlecluster/templates/hadoop/etc/hadoop/hadoop-env.sh +++ b/concourse/singlecluster/templates/hadoop/etc/hadoop/hadoop-env.sh @@ -1,5 +1,5 @@ # load singlecluster environment -. $bin/../../bin/gphd-env.sh +. $GPHD_ROOT/bin/gphd-env.sh export HADOOP_CLASSPATH=\ $HADOOP_CLASSPATH:\ @@ -8,10 +8,10 @@ $COMMON_CLASSPATH:\ # Extra Java runtime options. Empty by default. export HADOOP_OPTS="$HADOOP_OPTS $COMMON_JAVA_OPTS" -export COMMON_MASTER_OPTS="-Dhadoop.tmp.dir=$HADOOP_STORAGE_ROOT" +export COMMON_MASTER_OPTS="-Dhadoop.tmp.dir=/home/gpadmin/workspace/singlecluster/storage/hadoop" # Command specific options appended to HADOOP_OPTS when specified -export HADOOP_NAMENODE_OPTS="$COMMON_MASTER_OPTS" +export HDFS_NAMENODE_OPTS="$COMMON_MASTER_OPTS" export HADOOP_SECONDARYNAMENODE_OPTS="$COMMON_MASTER_OPTS" # Where log files are stored. $HADOOP_HOME/logs by default. @@ -19,3 +19,6 @@ export HADOOP_LOG_DIR=$LOGS_ROOT # The directory where pid files are stored. /tmp by default. export HADOOP_PID_DIR=$PIDS_ROOT + +# FIXME: remove after upgrading to new Hive version +export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 \ No newline at end of file diff --git a/singlecluster/templates/hadoop/etc/hadoop/hdfs-site.xml b/concourse/singlecluster/templates/hadoop/etc/hadoop/hdfs-site.xml similarity index 83% rename from singlecluster/templates/hadoop/etc/hadoop/hdfs-site.xml rename to concourse/singlecluster/templates/hadoop/etc/hadoop/hdfs-site.xml index 81b8b929..e75a7eba 100755 --- a/singlecluster/templates/hadoop/etc/hadoop/hdfs-site.xml +++ b/concourse/singlecluster/templates/hadoop/etc/hadoop/hdfs-site.xml @@ -48,4 +48,12 @@ <name>dfs.encryption.key.provider.uri</name> <value>kms://[email protected]:16000/kms</value> </property> + <property> + <name>dfs.namenode.name.dir</name> + <value>/home/gpadmin/workspace/singlecluster/storage/hadoop/dfs/name</value> + </property> + <property> + <name>dfs.datanode.data.dir</name> + <value>/home/gpadmin/workspace/singlecluster/storage/hadoop/dfs/data</value> + </property> </configuration> diff --git a/singlecluster/templates/hadoop/etc/hadoop/mapred-site.xml b/concourse/singlecluster/templates/hadoop/etc/hadoop/mapred-site.xml similarity index 100% rename from singlecluster/templates/hadoop/etc/hadoop/mapred-site.xml rename to concourse/singlecluster/templates/hadoop/etc/hadoop/mapred-site.xml diff --git a/singlecluster/templates/hadoop/etc/hadoop/yarn-env.sh b/concourse/singlecluster/templates/hadoop/etc/hadoop/yarn-env.sh similarity index 95% rename from singlecluster/templates/hadoop/etc/hadoop/yarn-env.sh rename to concourse/singlecluster/templates/hadoop/etc/hadoop/yarn-env.sh index f41e56d7..623cc561 100755 --- a/singlecluster/templates/hadoop/etc/hadoop/yarn-env.sh +++ b/concourse/singlecluster/templates/hadoop/etc/hadoop/yarn-env.sh @@ -14,7 +14,7 @@ # limitations under the License. # load singlecluster environment -. $bin/../../bin/gphd-env.sh +. $GPHD_ROOT/bin/gphd-env.sh export YARN_LOG_DIR=$LOGS_ROOT export YARN_OPTS="$YARN_OPTS $COMMON_JAVA_OPTS" @@ -28,6 +28,10 @@ export HADOOP_YARN_USER=${HADOOP_YARN_USER:-yarn} # some Java parameters # export JAVA_HOME=/home/y/libexec/jdk1.6.0/ + +# FIXME: remove after upgrading to new Hive version +export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 + if [ "$JAVA_HOME" != "" ]; then #echo "run java in $JAVA_HOME" JAVA_HOME=$JAVA_HOME diff --git a/singlecluster/templates/hadoop/etc/hadoop/yarn-site.xml b/concourse/singlecluster/templates/hadoop/etc/hadoop/yarn-site.xml similarity index 80% rename from singlecluster/templates/hadoop/etc/hadoop/yarn-site.xml rename to concourse/singlecluster/templates/hadoop/etc/hadoop/yarn-site.xml index 6c89a5a6..a9b16f23 100755 --- a/singlecluster/templates/hadoop/etc/hadoop/yarn-site.xml +++ b/concourse/singlecluster/templates/hadoop/etc/hadoop/yarn-site.xml @@ -43,15 +43,15 @@ <property> <name>yarn.application.classpath</name> <value> - $HADOOP_CONF, - $HADOOP_ROOT/share/hadoop/common/*, - $HADOOP_ROOT/share/hadoop/common/lib/*, - $HADOOP_ROOT/share/hadoop/hdfs/*, - $HADOOP_ROOT/share/hadoop/hdfs/lib/*, - $HADOOP_ROOT/share/hadoop/mapreduce/*, - $HADOOP_ROOT/share/hadoop/mapreduce/lib/*, - $HADOOP_ROOT/share/hadoop/yarn/*, - $HADOOP_ROOT/share/hadoop/yarn/lib/* + $HADOOP_CONF_DIR, + $HADOOP_HOME/share/hadoop/common/*, + $HADOOP_HOME/share/hadoop/common/lib/*, + $HADOOP_HOME/share/hadoop/hdfs/*, + $HADOOP_HOME/share/hadoop/hdfs/lib/*, + $HADOOP_HOME/share/hadoop/mapreduce/*, + $HADOOP_HOME/share/hadoop/mapreduce/lib/*, + $HADOOP_HOME/share/hadoop/yarn/*, + $HADOOP_HOME/share/hadoop/yarn/lib/* </value> </property> diff --git a/singlecluster/templates/hbase/conf/hbase-env.sh b/concourse/singlecluster/templates/hbase/conf/hbase-env.sh similarity index 97% rename from singlecluster/templates/hbase/conf/hbase-env.sh rename to concourse/singlecluster/templates/hbase/conf/hbase-env.sh index 7c10eab4..a3644e1a 100755 --- a/singlecluster/templates/hbase/conf/hbase-env.sh +++ b/concourse/singlecluster/templates/hbase/conf/hbase-env.sh @@ -20,7 +20,7 @@ # */ # load singlecluster environment -. $bin/../../bin/gphd-env.sh +. $GPHD_ROOT/bin/gphd-env.sh # Set environment variables here. @@ -92,3 +92,6 @@ export HBASE_PID_DIR=$PIDS_ROOT # Tell HBase whether it should manage it's own instance of Zookeeper or not. export HBASE_MANAGES_ZK=false + +# FIXME: remove after upgrading to new Hive version +export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 \ No newline at end of file diff --git a/concourse/singlecluster/templates/hbase/conf/hbase-site.xml b/concourse/singlecluster/templates/hbase/conf/hbase-site.xml new file mode 100755 index 00000000..00841290 --- /dev/null +++ b/concourse/singlecluster/templates/hbase/conf/hbase-site.xml @@ -0,0 +1,92 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration> + <property> + <name>hbase.rootdir</name> + <value>hdfs://0.0.0.0:8020/hbase</value> + </property> + <property> + <name>dfs.replication</name> + <value>3</value> + </property> + <property> + <name>dfs.support.append</name> + <value>true</value> + </property> + <property> + <name>hbase.cluster.distributed</name> + <value>true</value> + </property> + <property> + <name>hbase.zookeeper.quorum</name> + <value>127.0.0.1</value> + </property> + <property> + <name>hbase.zookeeper.property.clientPort</name> + <value>2181</value> + </property> + <property> + <name>hadoop.proxyuser.gpadmin.hosts</name> + <value>*</value> + </property> + <property> + <name>hadoop.proxyuser.gpadmin.groups</name> + <value>*</value> + </property> + <property> + <name>hadoop.security.authorization</name> + <value>true</value> + </property> + <property> + <name>hbase.security.authorization</name> + <value>true</value> + </property> + <property> + <name>hbase.rpc.protection</name> + <value>authentication</value> + </property> + <property> + <name>hbase.coprocessor.master.classes</name> + <value>org.apache.hadoop.hbase.security.access.AccessController</value> + </property> + <property> + <name>hbase.coprocessor.region.classes</name> + <value>org.apache.hadoop.hbase.security.access.AccessController,org.apache.hadoop.hbase.security.access.SecureBulkLoadEndpoint</value> + </property> + <property> + <name>hbase.coprocessor.regionserver.classes</name> + <value>org.apache.hadoop.hbase.security.access.AccessController</value> + </property> + + <!-- + workaround for java.lang.IllegalStateException: The procedure WAL relies on the ability to hflush for proper + operation during component failures, but the underlying filesystem does not support doing so. Please check the + config value of 'hbase.procedure.store.wal.use.hsync' to set the desired level of robustness and ensure the config + value of 'hbase.wal.dir' points to a FileSystem mount that can provide it. + --> + <property> + <name>hbase.unsafe.stream.capability.enforce</name> + <value>false</value> + </property> +</configuration> diff --git a/concourse/singlecluster/templates/hive/conf/hive-env.sh b/concourse/singlecluster/templates/hive/conf/hive-env.sh new file mode 100755 index 00000000..7791c8a8 --- /dev/null +++ b/concourse/singlecluster/templates/hive/conf/hive-env.sh @@ -0,0 +1,7 @@ +# load singlecluster environment +. $GPHD_ROOT/bin/gphd-env.sh + +export HIVE_OPTS="-hiveconf derby.stream.error.file=$LOGS_ROOT/derby.log -hiveconf javax.jdo.option.ConnectionURL=jdbc:derby:;databaseName=$HIVE_STORAGE_ROOT/metastore_db;create=true" +export HIVE_SERVER_OPTS="-hiveconf derby.stream.error.file=$LOGS_ROOT/derby.log -hiveconf ;databaseName=$HIVE_STORAGE_ROOT/metastore_db;create=true" +export HADOOP_HOME=$HADOOP_ROOT +export HADOOP_CLASSPATH="$TEZ_CONF:$TEZ_JARS:$HADOOP_CLASSPATH" diff --git a/singlecluster/templates/hive/conf/hive-site.xml b/concourse/singlecluster/templates/hive/conf/hive-site.xml similarity index 65% rename from singlecluster/templates/hive/conf/hive-site.xml rename to concourse/singlecluster/templates/hive/conf/hive-site.xml index 70a87d6b..35b2891e 100755 --- a/singlecluster/templates/hive/conf/hive-site.xml +++ b/concourse/singlecluster/templates/hive/conf/hive-site.xml @@ -19,7 +19,7 @@ </property> <property> <name>hive.execution.engine</name> - <value>mr</value> + <value>tez</value> <description>Chooses execution engine. Options are: mr(default), tez, or spark</description> </property> <property> @@ -35,4 +35,25 @@ <name>hive.metastore.integral.jdo.pushdown</name> <value>true</value> </property> + <property> + <name>hive.tez.container.size</name> + <value>2048</value> + </property> + <property> + <name>datanucleus.schema.autoCreateAll</name> + <value>True</value> + </property> + <property> + <name>metastore.metastore.event.db.notification.api.auth</name> + <value>false</value> + </property> + <property> + <name>hive.txn.stats.enabled</name> + <value>false</value> + </property> + <property> + <name>hive.stats.autogather</name> + <value>false</value> + </property> + </configuration> diff --git a/singlecluster/templates/ranger/install.properties b/concourse/singlecluster/templates/ranger/install.properties similarity index 100% rename from singlecluster/templates/ranger/install.properties rename to concourse/singlecluster/templates/ranger/install.properties diff --git a/singlecluster/templates/tez/conf/tez-site.xml b/concourse/singlecluster/templates/tez/conf/tez-site.xml similarity index 89% rename from singlecluster/templates/tez/conf/tez-site.xml rename to concourse/singlecluster/templates/tez/conf/tez-site.xml index 44515e93..28e5516a 100755 --- a/singlecluster/templates/tez/conf/tez-site.xml +++ b/concourse/singlecluster/templates/tez/conf/tez-site.xml @@ -18,5 +18,9 @@ <description>The location of the Tez libraries which will be localized for DAGs</description> </property> + <property> + <name>tez.use.cluster.hadoop-libs</name> + <value>true</value> + </property> </configuration> \ No newline at end of file diff --git a/singlecluster/templates/usersync/install.properties b/concourse/singlecluster/templates/usersync/install.properties similarity index 100% rename from singlecluster/templates/usersync/install.properties rename to concourse/singlecluster/templates/usersync/install.properties diff --git a/singlecluster/templates/zookeeper/conf/zoo.cfg b/concourse/singlecluster/templates/zookeeper/conf/zoo.cfg similarity index 100% rename from singlecluster/templates/zookeeper/conf/zoo.cfg rename to concourse/singlecluster/templates/zookeeper/conf/zoo.cfg diff --git a/singlecluster/Makefile b/singlecluster/Makefile deleted file mode 100755 index 5c3cdf29..00000000 --- a/singlecluster/Makefile +++ /dev/null @@ -1,121 +0,0 @@ -ROOT = . - -BUILDVER=$(shell cat product.version) - -BUILDNUM = $(BUILD_NUMBER) -ifeq ($(BUILDNUM),) - BUILDNUM = $(shell whoami) -endif - -HADOOP_VERSION = undefined -HADOOP_DISTRO = HDP - -TARGET = singlecluster-$(HADOOP_DISTRO).tar.gz - -BUILDROOT = $(TARGET:%.tar.gz=%) -BINROOT = $(ROOT)/bin -TARSROOT = $(ROOT)/tars -TEMPLATESROOT = $(ROOT)/templates - -VERSIONSFILE = $(BUILDROOT)/versions.txt - -BINFILES = $(filter-out *~, $(wildcard $(BINROOT)/*)) -TARFILES = $(subst $(TARSROOT)/,,$(wildcard $(TARSROOT)/*.tar.gz)) -EXTRACTEDTARS = $(TARFILES:%.tar.gz=%.extracted) -TEMPLATES := $(shell find $(TEMPLATESROOT) -type f -not -iname "*~") -STACK = $(shell echo $(HADOOP_DISTRO) | tr A-Z a-z) - -ALLTARGETS = singlecluster-* -DIRT = *.extracted *~ - -# Do not run this build script in parallel -.NOTPARALLEL: - -.PHONY: all -all: clean $(TARGET) - -.PHONY: clean -clean: - -rm -rf $(ALLTARGETS) - -rm -rf $(DIRT) - -$(TARGET): $(BUILDROOT) make_tarball -# $(TARGET): $(BUILDROOT) - -$(BUILDROOT): copy_binfiles create_versions_file extract_products copy_templates copy_deps - chmod -R +w $(BUILDROOT) - -.PHONY: copy_binfiles -copy_binfiles: $(BINFILES) - mkdir -p $(BUILDROOT)/bin - cp $^ $(BUILDROOT)/bin - -.PHONY: create_versions_file -create_versions_file: - echo build number: $(BUILDNUM) > $(VERSIONSFILE) - echo single_cluster-$(BUILDVER) >> $(VERSIONSFILE) - -.PHONY: extract_products -extract_products: $(EXTRACTEDTARS) extract_stack_$(STACK) - for X in $(BUILDROOT)/*-[0-9]*; do \ - mv $$X `echo $$X | sed -e 's/^\($(BUILDROOT)\/[A-Za-z0-9]*\).*$$/\1/'`; \ - done; - chmod -R +w $(BUILDROOT) - du -sh $(BUILDROOT)/* - -.PHONY: extract_stack_cdh -extract_stack_cdh: - find $(BUILDROOT)/$(HADOOP_DISTRO)-$(HADOOP_VERSION) -iwholename "*.tar.gz" | \ - grep "\(hadoop\|zookeeper\|hive\|hbase\)" | \ - xargs -n1 tar -C $(BUILDROOT) -xzf - rm -rf $(BUILDROOT)/hbase*/docs $(BUILDROOT)/hadoop*/src $(BUILDROOT)/hadoop*/share/doc - rm -rf $(BUILDROOT)/$(HADOOP_DISTRO)-$(HADOOP_VERSION) - chown root:root -R $(BUILDROOT)/* || true - find $(BUILDROOT) -maxdepth 1 -type d | \ - grep "\(hadoop\|zookeeper\|hive\|hbase\)" | \ - xargs -n1 basename >> $(VERSIONSFILE) - -.PHONY: extract_stack_hdp -extract_stack_hdp: - find $(BUILDROOT) -iwholename "*.tar.gz" | \ - grep "\(hadoop\|hbase\|zookeeper\|hive\)" | \ - grep -v -E "sqoop|plugin|lzo" | \ - xargs -n1 tar -C $(BUILDROOT) -xzf - find $(BUILDROOT) -iwholename "*.tar.gz" | grep "\(tez\)" | \ - xargs sh -c 'mkdir -p $(BUILDROOT)/`basename $${0%.tar.gz}` && \ - tar -C $(BUILDROOT)/`basename $${0%.tar.gz}` -xzf $$0' - find $(BUILDROOT) -type d -a -iname "$(HADOOP_DISTRO)-*" | xargs rm -rf - rm -rf $(BUILDROOT)/*.tar.gz $(BUILDROOT)/hbase*/docs - mv $(BUILDROOT)/apache-hive* $(BUILDROOT)/hive || true - chown root:root -R $(BUILDROOT)/* || true - find $(BUILDROOT) -maxdepth 1 -type d | \ - grep "\(hadoop\|hbase\|zookeeper\|hive\|tez\)" | \ - xargs -n1 basename >> $(VERSIONSFILE) - -.PHONY: copy_templates -copy_templates: $(TEMPLATES) - for X in `ls $(BUILDROOT)`; do \ - if [ -d "$(TEMPLATESROOT)/$$X" ]; \ - then cp -r $(TEMPLATESROOT)/$$X/* $(BUILDROOT)/$$X; \ - fi; \ - done; - cp -r $(TEMPLATESROOT)/conf $(BUILDROOT) - - -find $(BUILDROOT) -iname "*~" | xargs rm -f - -.PHONY: copy_deps -copy_deps: - find . -maxdepth 1 -name *.tar.gz | xargs -I {} tar xzf {} -C $(BUILDROOT) - -.PHONY: refresh_tars -refresh_tars: - make -C $(TARSROOT) clean all - -.PHONY: make_tarball -make_tarball: $(BUILDROOT) - tar czf $(BUILDROOT).tar.gz $< - -%.extracted: $(TARSROOT)/%.tar.gz - tar xzf $^ -C $(BUILDROOT) - touch $@ - echo $* >> $(VERSIONSFILE) diff --git a/singlecluster/README.md b/singlecluster/README.md deleted file mode 100755 index be881870..00000000 --- a/singlecluster/README.md +++ /dev/null @@ -1,93 +0,0 @@ -SingleCluster -============= - -Singlecluster is a self contained, easy to deploy distribution of HDP or CDH. - -Singlecluster-HDP contains the following versions: - -- Hadoop 2.7.3 -- Hive 1.2.1000 -- Zookeeper 3.4.6 -- HBase 1.1.2 -- Tez 0.7.0 -- Tomcat 7.0.62 - -Singlecluster-CDH contains the following versions: - -- CDH 5.12.2 -- Hadoop 2.6.0-CDH5.12.2 -- Hive 1.1.0-CDH5-12.2 -- Zookeeper 3.4.5-CDH5.12.2 -- HBase 1.2.0-CDH5.12.2 - -For HDP3, please use the HDP3 related README. - -Prerequisites -------------- - -1. $JAVA_HOME points to a JDK7 or later install - -Build ------ - -- make HADOOP_DISTRO=[CDH|HDP] HADOOP_VERSION=[CDH version|HDP version] -- if you do "make", HDP is the default tarball to generate -- E.g. make HADOOP_DISTRO=CDH HADOOP_VERSION=5.12.2 -- E.g. make HADOOP_DISTRO=HDP HADOOP_VERSION=2.5.3.0 - -Initialization --------------- - -1. Untar the singlecluster tarball - - mv singlecluster.tar.gz ~/. - - cd ~/. - - tar -xzvf singlecluster-CDH.tar.gz - - cd singlecluster-CDH -2. Initialize an instance - - bin/init-gphd.sh -3. Add the following to your environment - - export GPHD_ROOT=<singlecluster location, e.g. ~/singlecluster-PHD> - - export HADOOP_ROOT=$GPHD_ROOT/hadoop - - export HBASE_ROOT=$GPHD_ROOT/hbase - - export HIVE_ROOT=$GPHD_ROOT/hive - - export ZOOKEEPER_ROOT=$GPHD_ROOT/zookeeper - - export PATH=$PATH:$GPHD_ROOT/bin:$HADOOP_ROOT/bin:$HBASE_ROOT/bin:$HIVE_ROOT/bin:$ZOOKEEPER_ROOT/bin - -Usage ------ - -- Start all Hadoop services - - $GPHD_ROOT/bin/start-gphd.sh -- Start HDFS only - - $GPHD_ROOT/bin/start-hdfs.sh -- Start PXF only (Install pxf first to make this work. [See Install PXF session here](https://cwiki.apache.org/confluence/display/HAWQ/PXF+Build+and+Install)) - - $GPHD_ROOT/bin/start-pxf.sh -- Start HBase only (requires hdfs and zookeeper) - - $GPHD_ROOT/bin/start-hbase.sh -- Start ZooKeeper only - - $GPHD_ROOT/bin/start-zookeeper.sh -- Start YARN only - - $GPHD_ROOT/bin/start-yarn.sh -- Start Hive (MetaStore) - - $GPHD_ROOT/bin/start-hive.sh -- Stop all PHD services - - $GPHD_ROOT/bin/stop-gphd.sh -- Stop an individual component - - $GPHD_ROOT/bin/stop-[hdfs|pxf|hbase|zookeeper|yarn|hive].sh -- Start/stop HiveServer2 - - $GPHD_ROOT/bin/hive-service.sh hiveserver2 start - - $GPHD_ROOT/bin/hive-service.sh hiveserver2 stop - -Notes ------ - -1. Make sure you have enough memory and space to run all services. Typically about 24GB space is needed to run pxf automation. -2. All of the data is stored under $GPHD_ROOT/storage. Cleanup this directory before running init again. - -Concourse Pipeline Deployment ------------------------------ - -To deploy the concourse pipeline that will build the single cluster tarballs and upload them to S3, use the following command: -``` -make -C ~/workspace/pxf/concourse singlecluster -``` diff --git a/singlecluster/package_singlecluster_cdh.yml b/singlecluster/package_singlecluster_cdh.yml deleted file mode 100755 index 5d80b7b5..00000000 --- a/singlecluster/package_singlecluster_cdh.yml +++ /dev/null @@ -1,16 +0,0 @@ ---- -platform: linux - -inputs: - - name: pxf_src - - name: cdh_tars_tarball - - name: jdbc - -outputs: - - name: artifacts - -run: - path: pxf_src/singlecluster/singlecluster.bash - args: - - "5.12.2" - - "CDH" diff --git a/singlecluster/package_singlecluster_hdp2.yml b/singlecluster/package_singlecluster_hdp2.yml deleted file mode 100755 index 49b16a38..00000000 --- a/singlecluster/package_singlecluster_hdp2.yml +++ /dev/null @@ -1,16 +0,0 @@ ---- -platform: linux - -inputs: - - name: pxf_src - - name: hdp_tars_tarball - - name: jdbc - -outputs: - - name: artifacts - -run: - path: pxf_src/singlecluster/singlecluster.bash - args: - - "2.5.3.0" - - "HDP" diff --git a/singlecluster/package_singlecluster_hdp3.yml b/singlecluster/package_singlecluster_hdp3.yml deleted file mode 100755 index 7e8c077f..00000000 --- a/singlecluster/package_singlecluster_hdp3.yml +++ /dev/null @@ -1,16 +0,0 @@ ---- -platform: linux - -inputs: - - name: pxf_src - - name: hdp_tars_tarball - - name: jdbc - -outputs: - - name: artifacts - -run: - path: pxf_src/singlecluster/singlecluster.bash - args: - - "3.1.4.0" - - "HDP" diff --git a/singlecluster/product.version b/singlecluster/product.version deleted file mode 100755 index 7ec1d6db..00000000 --- a/singlecluster/product.version +++ /dev/null @@ -1 +0,0 @@ -2.1.0 diff --git a/singlecluster/singlecluster.bash b/singlecluster/singlecluster.bash deleted file mode 100755 index 41b35c05..00000000 --- a/singlecluster/singlecluster.bash +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash - -set -exo pipefail - -_main() { - singlecluster=$(pwd)/pxf_src/singlecluster - HADOOP_DISTRO_LOWER=$(echo ${2} | tr A-Z a-z) - mkdir -p ${singlecluster}/tars - mv ${HADOOP_DISTRO_LOWER}_tars_tarball/*.tar.gz ${singlecluster}/tars - mv jdbc/*.jar ${singlecluster} - pushd ${singlecluster} - make HADOOP_VERSION="${1}" HADOOP_DISTRO="${2}" - mv singlecluster-${2}.tar.gz ../../artifacts/singlecluster-${2}.tar.gz - popd -} - -_main "$@" diff --git a/singlecluster/templates/hadoop/etc/hadoop/core-site.xml b/singlecluster/templates/hadoop/etc/hadoop/core-site.xml deleted file mode 100755 index dd41fedc..00000000 --- a/singlecluster/templates/hadoop/etc/hadoop/core-site.xml +++ /dev/null @@ -1,32 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> -<!-- - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. See accompanying LICENSE file. ---> - -<!-- Put site-specific property overrides in this file. --> - -<configuration> - <property> - <name>fs.defaultFS</name> - <value>hdfs://0.0.0.0:8020</value> - </property> - <property> - <name>hadoop.security.key.provider.path</name> - <value>kms://[email protected]:16000/kms</value> - </property> - <property> - <name>ipc.ping.interval</name> - <value>900000</value> - </property> -</configuration> diff --git a/singlecluster/templates/hbase/conf/hbase-site.xml b/singlecluster/templates/hbase/conf/hbase-site.xml deleted file mode 100755 index 2aed86fe..00000000 --- a/singlecluster/templates/hbase/conf/hbase-site.xml +++ /dev/null @@ -1,49 +0,0 @@ -<?xml version="1.0"?> -<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> -<!-- -/** - * Copyright 2010 The Apache Software Foundation - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ ---> -<configuration> - <property> - <name>hbase.rootdir</name> - <value>hdfs://0.0.0.0:8020/hbase</value> - </property> - <property> - <name>dfs.replication</name> - <value>3</value> - </property> - <property> - <name>dfs.support.append</name> - <value>true</value> - </property> - <property> - <name>hbase.cluster.distributed</name> - <value>true</value> - </property> - <property> - <name>hbase.zookeeper.quorum</name> - <value>127.0.0.1</value> - </property> - <property> - <name>hbase.zookeeper.property.clientPort</name> - <value>2181</value> - </property> -</configuration> diff --git a/singlecluster/templates/hive/conf/hive-env.sh b/singlecluster/templates/hive/conf/hive-env.sh deleted file mode 100755 index 5467f51c..00000000 --- a/singlecluster/templates/hive/conf/hive-env.sh +++ /dev/null @@ -1,11 +0,0 @@ -# load singlecluster environment -if [ -f $bin/../bin/gphd-env.sh ]; then - . $bin/../bin/gphd-env.sh -elif [ -f $bin/../../bin/gphd-env.sh ]; then - . $bin/../../bin/gphd-env.sh -fi - -export HIVE_OPTS="-hiveconf hive.log.dir=$LOGS_ROOT -hiveconf derby.stream.error.file=$LOGS_ROOT/derby.log -hiveconf javax.jdo.option.ConnectionURL=jdbc:derby:;databaseName=$HIVE_STORAGE_ROOT/metastore_db;create=true" -export HIVE_SERVER_OPTS="-hiveconf hive.log.dir=$LOGS_ROOT -hiveconf derby.stream.error.file=$LOGS_ROOT/derby.log -hiveconf ;databaseName=$HIVE_STORAGE_ROOT/metastore_db;create=true" -export HADOOP_HOME=$HADOOP_ROOT -export HADOOP_CLASSPATH="$TEZ_CONF:$TEZ_JARS:$HADOOP_CLASSPATH" \ No newline at end of file diff --git a/singlecluster/tools/README.md b/singlecluster/tools/README.md deleted file mode 100755 index 9fef7da1..00000000 --- a/singlecluster/tools/README.md +++ /dev/null @@ -1,40 +0,0 @@ -# Tools - -## Compress HDP - -The HDP tarball we get from Hortonworks is around 5GB of Hadoop components. We only use a small subset of these for singlecluster so the `compressHDP.sh` script downloads the Hortonworks tarball, strips out the unnecessary components and creates a much smaller tarball - -To invoke the command: - -``` -#HDP 2.4 -./compressHDP.sh http://public-repo-1.hortonworks.com HDP-2.4.2.0-centos6-tars-tarball.tar.gz 2.4.2.0 centos6 HDP - -#HDP 2.5 -./compressHDP.sh http://public-repo-1.hortonworks.com HDP-2.5.0.0-centos6-tars-tarball.tar.gz 2.5.0.0 centos6 HDP -``` - -Once the artifact has been created locally scp it to our dist server - -## Download CDH - -Cloudera has different Hadoop components packaged separately. The "downloadCDH.sh" script downloads tarballs of required components of specific versions respectively, and archives them together into one single tarball. - -To invoke the command: - -``` -#CDH 5.12.2 -./downloadCDH.sh -``` - -For other CDH versions, update required component tarballs as needed: -``` -tarballs=( - 'hadoop-<hadoop_version>-cdh<cdh_version>.tar.gz' - 'hbase-<hbase_version>-cdh<cdh_version>.tar.gz' - 'hive-<hive_version>-cdh<cdh_version>.tar.gz' - 'zookeeper-<zookeeper_version>-cdh<cdh_version>.tar.gz' - '<some_component>-<some_component_version>-cdh<cdh_version>.tar.gz' -) -``` -Find CDH tarballs information [here](https://www.cloudera.com/documentation/enterprise/release-notes/topics/cdh_vd_cdh_package_tarball.html). Going forward, please keep this script updated for the preferred CDH version. diff --git a/singlecluster/tools/compressHDP.sh b/singlecluster/tools/compressHDP.sh deleted file mode 100755 index 94a30000..00000000 --- a/singlecluster/tools/compressHDP.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env bash - -set -x -#fetches official HDP tarball - -server=$1 -tarball=$2 -final_archive=$2 -version=$3 -platform=$4 -distro=$5 -major_version=$(echo ${version}| cut -c1) -url=${server}/${distro}/${platform}/${major_version}.x/updates/${version}/${tarball} -destination_dir=${tarball}-data - -echo Latest artifact: ${tarball} | tee -a ${log_file} -echo Downloading: ${url} | tee -a ${log_file} - -wget ${url} - -if [[ $? -ne 0 ]]; then - echo download failed - exit 1 -fi - -echo Untarring artifact -tar xvzf ${tarball} --strip-components 2 -rm -rf tars/source -mv ${tarball} "${tarball}.bak" -# rm $tarball -touch ${final_archive} -mkdir -p ${destination_dir} -mv tars/* ${destination_dir}/ -rm -rf tars -pushd ${destination_dir} -find . -iwholename "*source.tar.gz" | xargs rm - -# Remove tars that are in the root directory -rm *tar.gz - -for file in `find . -iwholename "*${version}*tar.gz" | grep "\(tez\|hadoop\|hbase\|zookeeper\|hive\)" | grep -v -E "phoenix|accumulo|storm|calcite_hive3|tez_hive2|sqoop|plugin|lzo" | grep -v -E "tez-[0-9.-]*-minimal"`; do - mv ${file} . -done; -rm -r -- */ -tar czf ${final_archive} * -mv ${final_archive} ../ -popd -rm -rf ${destination_dir} diff --git a/singlecluster/tools/downloadCDH.sh b/singlecluster/tools/downloadCDH.sh deleted file mode 100755 index aa6ec31d..00000000 --- a/singlecluster/tools/downloadCDH.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env bash - -set -x -#fetches official CDH tarball - -server='http://archive.cloudera.com' -tarballs=( - 'hadoop-2.6.0-cdh5.12.2.tar.gz' - 'hbase-1.2.0-cdh5.12.2.tar.gz' - 'hive-1.1.0-cdh5.12.2.tar.gz' - 'parquet-1.5.0-cdh5.12.2.tar.gz' - 'parquet-format-2.1.0-cdh5.12.2.tar.gz' - 'pig-0.12.0-cdh5.12.2.tar.gz' - 'zookeeper-3.4.5-cdh5.12.2.tar.gz' -) -distro='cdh' -version='5.12.2' -major_version=$(echo ${version}| cut -c1) -destination_dir=CDH-${version} - -rm -r ${destination_dir} -rm ${destination_dir}.tar.gz -mkdir -p ${destination_dir} - -for tarball in ${tarballs[@]} -do - url=${server}/${distro}${major_version}/${distro}/${major_version}/${tarball} - echo Latest artifact: ${tarball} | tee -a ${log_file} - echo Downloading: ${url} | tee -a ${log_file} - wget ${url} - if [[ $? -ne 0 ]]; then - echo download failed - exit 1 - fi - mv ${tarball} ${destination_dir} -done - -tar -czf ${destination_dir}.tar.gz ${destination_dir} -rm -rf ${destination_dir} diff --git a/singlecluster/tools/fetchBuild.sh b/singlecluster/tools/fetchBuild.sh deleted file mode 100755 index 1e79bc15..00000000 --- a/singlecluster/tools/fetchBuild.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env bash - -# fetches latest file matching <pattern> from the <url> - -if [[ "x$1" == "x" || "x$2" == "x" ]]; then - echo "usage: `basename $0` <url> <pattern>" - echo " will fetch latest artifact matching <pattern> from <url>" - exit 1 -fi - -server=$1 -pattern=$2 - -log_file=fetch.log -tmpfile=/tmp/curldata.${RANDOM}.tmp -sort_order=?O=A -page_url=${server}/${sort_order} - -echo Access page: ${page_url} | tee -a ${log_file} -curl -s ${page_url} > ${tmpfile} -if [[ $? -ne 0 ]]; then - echo page download failed | tee -a ${log_file} - exit 1 -fi - -echo ----- page start ----- >> ${log_file} -cat ${tmpfile} >> ${log_file} -echo ----- page end ----- >> ${log_file} - -last_build_file=`cat ${tmpfile} | grep -o "href=\"${pattern}\.tar\.gz\"" | grep -o "${pattern}.tar.gz" | tail -n1` - -if [[ "x$last_build_file" == "x" ]]; then - echo could not find a download link | tee -a ${log_file} - exit 1 -fi - -find . -regex "\.\/${pattern}.*.tar.gz" -delete - -echo Latest artifact: ${last_build_file} | tee -a ${log_file} -echo Downloading: ${server}/${last_build_file} | tee -a ${log_file} -echo use tail -f `pwd`/${log_file} to track download - -wget -a ${log_file} ${server}/${last_build_file} -if [[ $? -ne 0 ]]; then - echo download failed - exit 1 -fi - -touch ${last_build_file} - -rm ${tmpfile} --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
