date:20210416

[accumulo] branch 1451-external-compactions-feature updated: fixed #2019 repopulate selected set for ext compactions w/ other improvements

2021-04-16 Thread kturner

This is an automated email from the ASF dual-hosted git repository.

kturner pushed a commit to branch 1451-external-compactions-feature
in repository https://gitbox.apache.org/repos/asf/accumulo.git


The following commit(s) were added to 
refs/heads/1451-external-compactions-feature by this push:
 new 955c44a  fixed #2019 repopulate selected set for ext compactions w/ 
other improvements
955c44a is described below

commit 955c44a7e409b34f25d9fb5c9786e907083502fa
Author: Keith Turner 
AuthorDate: Thu Apr 15 20:58:37 2021 -0400

fixed #2019 repopulate selected set for ext compactions w/ other 
improvements

  * repopulated set of selected files for external compaction
  * added lots of sanity checks to existing external compactions on tablet 
load
  * added IT to compact 200 tablets externally, test is timing out
---
 .../schema/ExternalCompactionMetadata.java |  48 -
 .../coordinator/DeadCompactionDetector.java|   1 +
 .../accumulo/tserver/tablet/CompactableImpl.java   | 231 ++---
 .../accumulo/tserver/tablet/CompactableUtils.java  |  10 +-
 .../accumulo/tserver/tablet/DatafileManager.java   |  16 +-
 .../apache/accumulo/test/ExternalCompactionIT.java |  64 +-
 6 files changed, 324 insertions(+), 46 deletions(-)

diff --git 
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/ExternalCompactionMetadata.java
 
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/ExternalCompactionMetadata.java
index d1d127c..d64a9a1 100644
--- 
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/ExternalCompactionMetadata.java
+++ 
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/ExternalCompactionMetadata.java
@@ -39,29 +39,42 @@ public class ExternalCompactionMetadata {
   private static final Gson GSON = new GsonBuilder().create();
 
   private final Set jobFiles;
+  private final Set nextFiles;
   private final TabletFile compactTmpName;
   private final TabletFile newFile;
   private final String compactorId;
   private final CompactionKind kind;
   private final long priority;
   private final CompactionExecutorId ceid;
-
-  public ExternalCompactionMetadata(Set jobFiles, TabletFile 
compactTmpName,
-  TabletFile newFile, String compactorId, CompactionKind kind, long 
priority,
-  CompactionExecutorId ceid) {
+  private final boolean propogateDeletes;
+  private final boolean selectedAll;
+  private final Long compactionId;
+
+  public ExternalCompactionMetadata(Set jobFiles, 
Set nextFiles,
+  TabletFile compactTmpName, TabletFile newFile, String compactorId, 
CompactionKind kind,
+  long priority, CompactionExecutorId ceid, boolean propogateDeletes, 
boolean selectedAll,
+  Long compactionId) {
 this.jobFiles = Objects.requireNonNull(jobFiles);
+this.nextFiles = Objects.requireNonNull(nextFiles);
 this.compactTmpName = Objects.requireNonNull(compactTmpName);
 this.newFile = Objects.requireNonNull(newFile);
 this.compactorId = Objects.requireNonNull(compactorId);
 this.kind = Objects.requireNonNull(kind);
 this.priority = priority;
 this.ceid = Objects.requireNonNull(ceid);
+this.propogateDeletes = propogateDeletes;
+this.selectedAll = selectedAll;
+this.compactionId = compactionId;
   }
 
   public Set getJobFiles() {
 return jobFiles;
   }
 
+  public Set getNextFiles() {
+return nextFiles;
+  }
+
   public TabletFile getCompactTmpName() {
 return compactTmpName;
   }
@@ -86,37 +99,62 @@ public class ExternalCompactionMetadata {
 return ceid;
   }
 
+  public boolean isPropogateDeletes() {
+return propogateDeletes;
+  }
+
+  public boolean isSelectedAll() {
+return selectedAll;
+  }
+
+  public Long getCompactionId() {
+return compactionId;
+  }
+
   // This class is used to serialize and deserialize this class using GSon. 
Any changes to this
   // class must consider persisted data.
   private static class GSonData {
 List inputs;
+List nextFiles;
 String tmp;
 String dest;
 String compactor;
 String kind;
 String executorId;
 long priority;
+boolean propDels;
+boolean selectedAll;
+Long compactionId;
   }
 
   public String toJson() {
 GSonData jData = new GSonData();
+
 jData.inputs = 
jobFiles.stream().map(StoredTabletFile::getMetaUpdateDelete).collect(toList());
+jData.nextFiles =
+
nextFiles.stream().map(StoredTabletFile::getMetaUpdateDelete).collect(toList());
 jData.tmp = compactTmpName.getMetaInsert();
 jData.dest = newFile.getMetaInsert();
 jData.compactor = compactorId;
 jData.kind = kind.name();
 jData.executorId = ceid.getExernalName();
 jData.priority = priority;
+jData.propDels = propogateDeletes;
+jData.selectedAll = selectedAll;
+jData.compactionId = compactionId;
 return GSON.toJson(jData);
   }
 
   public static ExternalCompactionMetadata fromJson(String json) {
 GSonData jData =

[accumulo] 01/01: Merge branch '1.10'

2021-04-16 Thread ctubbsii

This is an automated email from the ASF dual-hosted git repository.

ctubbsii pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git

commit dd0d88eb664fc60714a7283b357a51afaad3b899
Merge: af955c8 1dc2101
Author: Christopher Tubbs 
AuthorDate: Fri Apr 16 04:11:49 2021 -0400

Merge branch '1.10'

 .github/workflows/maven-full-its.yaml  | 15 +++
 .github/workflows/maven-on-demand.yaml | 15 +++
 .github/workflows/maven.yaml   | 15 +++
 3 files changed, 45 insertions(+)

diff --cc .github/workflows/maven.yaml
index 3da4bb7,b6dcf91..5a9666d
--- a/.github/workflows/maven.yaml
+++ b/.github/workflows/maven.yaml
@@@ -79,9 -78,24 +79,24 @@@ jobs
!~/.m2/repository/org/apache/accumulo
  key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
  restore-keys: ${{ runner.os }}-m2
+ - name: Override DNS to fix IP address for hostname
+   run: |
+ ip -br addr
+ echo "'hostname -i' shows '$(hostname -i)'"
+ echo "'hostname -I' shows '$(hostname -I)'"
+ hostname_short=$(hostname -s)
+ hostname_long=$(hostname -f)
+ if ! grep -q $hostname_short /etc/hosts; then
+   actual_ip=$(ip -4 addr show dev eth0 | grep -o 'inet [0-9.]*' | cut 
-f2 -d ' ')
+   echo "Setting $hostname_long / $hostname_short to $actual_ip in 
/etc/hosts"
+   echo "$actual_ip $hostname_long $hostname_short" | sudo tee -a 
/etc/hosts
+   ip -br addr
+   echo "'hostname -i' shows '$(hostname -i)'"
+   echo "'hostname -I' shows '$(hostname -I)'"
+ fi
  - name: Build with Maven (${{ matrix.profile.name }})
timeout-minutes: 60
 -  run: mvn -B -V -e -ntp "-Dstyle.color=always" ${{ matrix.profile.args }}
 +  run: mvn -B -V -e -ntp "-Dstyle.color=always" -DskipFormat ${{ 
matrix.profile.args }}
env:
  MAVEN_OPTS: -Djansi.force=true
  - name: Upload unit test results

[accumulo] branch main updated (af955c8 -> dd0d88e)

2021-04-16 Thread ctubbsii

This is an automated email from the ASF dual-hosted git repository.

ctubbsii pushed a change to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git.


from af955c8  Remove ZK stats from Monitor overview page (#2020)
 add 1dc2101  Fix IP address in GitHub Actions container (#2024)
 new dd0d88e  Merge branch '1.10'

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 .github/workflows/maven-full-its.yaml  | 15 +++
 .github/workflows/maven-on-demand.yaml | 15 +++
 .github/workflows/maven.yaml   | 15 +++
 3 files changed, 45 insertions(+)

[accumulo] branch 1.10 updated: Fix IP address in GitHub Actions container (#2024)

2021-04-16 Thread ctubbsii

This is an automated email from the ASF dual-hosted git repository.

ctubbsii pushed a commit to branch 1.10
in repository https://gitbox.apache.org/repos/asf/accumulo.git


The following commit(s) were added to refs/heads/1.10 by this push:
 new 1dc2101  Fix IP address in GitHub Actions container (#2024)
1dc2101 is described below

commit 1dc21016654dc4d8a66178540fe2ce8012deb5be
Author: Christopher Tubbs 
AuthorDate: Fri Apr 16 04:10:48 2021 -0400

Fix IP address in GitHub Actions container (#2024)

Fix #2016 by adding an entry to /etc/hosts to fix incorrect DNS entries,
which return an IP for the current machine's hostname that does not
match any IP address in the machine. Adding an entry to /etc/hosts
to force the hostname to match on eth0's IP address.
---
 .github/workflows/maven-full-its.yaml  | 15 +++
 .github/workflows/maven-on-demand.yaml | 15 +++
 .github/workflows/maven.yaml   | 15 +++
 3 files changed, 45 insertions(+)

diff --git a/.github/workflows/maven-full-its.yaml 
b/.github/workflows/maven-full-its.yaml
index 0036a76..f6f5630 100644
--- a/.github/workflows/maven-full-its.yaml
+++ b/.github/workflows/maven-full-its.yaml
@@ -100,6 +100,21 @@ jobs:
   !~/.m2/repository/org/apache/accumulo
 key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
 restore-keys: ${{ runner.os }}-m2
+- name: Override DNS to fix IP address for hostname
+  run: |
+ip -br addr
+echo "'hostname -i' shows '$(hostname -i)'"
+echo "'hostname -I' shows '$(hostname -I)'"
+hostname_short=$(hostname -s)
+hostname_long=$(hostname -f)
+if ! grep -q $hostname_short /etc/hosts; then
+  actual_ip=$(ip -4 addr show dev eth0 | grep -o 'inet [0-9.]*' | cut 
-f2 -d ' ')
+  echo "Setting $hostname_long / $hostname_short to $actual_ip in 
/etc/hosts"
+  echo "$actual_ip $hostname_long $hostname_short" | sudo tee -a 
/etc/hosts
+  ip -br addr
+  echo "'hostname -i' shows '$(hostname -i)'"
+  echo "'hostname -I' shows '$(hostname -I)'"
+fi
 - name: Build with Maven (${{ matrix.profile.its }})
   timeout-minutes: 60
   run: mvn -B -V -e -ntp "-Dstyle.color=always" verify -PskipQA 
-DskipTests=false -DskipITs=false -Dtest=nomatchingtest -Dit.test="${{ 
matrix.profile.its }}"
diff --git a/.github/workflows/maven-on-demand.yaml 
b/.github/workflows/maven-on-demand.yaml
index 3cf9b7a..c8c719c 100644
--- a/.github/workflows/maven-on-demand.yaml
+++ b/.github/workflows/maven-on-demand.yaml
@@ -76,6 +76,21 @@ jobs:
 restore-keys: ${{ runner.os }}-m2
 - name: Show the first log message
   run: git log -n1
+- name: Override DNS to fix IP address for hostname
+  run: |
+ip -br addr
+echo "'hostname -i' shows '$(hostname -i)'"
+echo "'hostname -I' shows '$(hostname -I)'"
+hostname_short=$(hostname -s)
+hostname_long=$(hostname -f)
+if ! grep -q $hostname_short /etc/hosts; then
+  actual_ip=$(ip -4 addr show dev eth0 | grep -o 'inet [0-9.]*' | cut 
-f2 -d ' ')
+  echo "Setting $hostname_long / $hostname_short to $actual_ip in 
/etc/hosts"
+  echo "$actual_ip $hostname_long $hostname_short" | sudo tee -a 
/etc/hosts
+  ip -br addr
+  echo "'hostname -i' shows '$(hostname -i)'"
+  echo "'hostname -I' shows '$(hostname -I)'"
+fi
 - name: Build with Maven
   timeout-minutes: 345
   run: mvn -B -V -e -ntp "-Dstyle.color=always" ${{ 
github.event.inputs.mvnOpts }} ${{ github.event.inputs.goals }} ${{ 
github.event.inputs.utOpts }} ${{ github.event.inputs.itOpts }} ${{ 
github.event.inputs.addOpts }}
diff --git a/.github/workflows/maven.yaml b/.github/workflows/maven.yaml
index d2d49e4..b6dcf91 100644
--- a/.github/workflows/maven.yaml
+++ b/.github/workflows/maven.yaml
@@ -78,6 +78,21 @@ jobs:
   !~/.m2/repository/org/apache/accumulo
 key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
 restore-keys: ${{ runner.os }}-m2
+- name: Override DNS to fix IP address for hostname
+  run: |
+ip -br addr
+echo "'hostname -i' shows '$(hostname -i)'"
+echo "'hostname -I' shows '$(hostname -I)'"
+hostname_short=$(hostname -s)
+hostname_long=$(hostname -f)
+if ! grep -q $hostname_short /etc/hosts; then
+  actual_ip=$(ip -4 addr show dev eth0 | grep -o 'inet [0-9.]*' | cut 
-f2 -d ' ')
+  echo "Setting $hostname_long / $hostname_short to $actual_ip in 
/etc/hosts"
+  echo "$actual_ip $hostname_long $hostname_short" | sudo tee -a 
/etc/hosts
+  ip -br addr
+  echo "'hostname -i' shows '$(hostname -i)'"
+  echo "'hostname -I' shows '$(hostname -I)'"
+fi
 - name: Build with Maven (${{ matrix.profile.name }})
   timeout-minutes: 60
   run: mvn -B -V -e -ntp

[accumulo] branch 1451-external-compactions-feature updated: fixed #2019 repopulate selected set for ext compactions w/ other improvements

[accumulo] 01/01: Merge branch '1.10'

[accumulo] branch main updated (af955c8 -> dd0d88e)

[accumulo] branch 1.10 updated: Fix IP address in GitHub Actions container (#2024)

4 matches

Site Navigation

Mail list logo

Footer information