Repository: tajo Updated Branches: refs/heads/master a0d67bb60 -> adb34bdf8
TAJO-1244: tajo.worker.tmpdir.locations should use a validator for a list of Path. Closes #298 Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/adb34bdf Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/adb34bdf Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/adb34bdf Branch: refs/heads/master Commit: adb34bdf84299228037d7706bf0c0209c3577846 Parents: a0d67bb Author: Hyunsik Choi <[email protected]> Authored: Thu Dec 11 23:27:03 2014 +0900 Committer: Hyunsik Choi <[email protected]> Committed: Thu Dec 11 23:27:03 2014 +0900 ---------------------------------------------------------------------- CHANGES | 3 + .../java/org/apache/tajo/conf/TajoConf.java | 3 +- .../tajo/rule/SelfDiagnosisRuleDefinition.java | 3 +- .../tajo/rule/SelfDiagnosisRuleEngine.java | 22 ++++--- .../tajo/validation/PathListValidator.java | 63 ++++++++++++++++++++ .../apache/tajo/validation/PathValidator.java | 4 +- .../org/apache/tajo/validation/Validators.java | 4 ++ .../apache/tajo/validation/TestValidators.java | 30 ++++++++++ .../ConnectivityCheckerRuleForTajoWorker.java | 3 +- 9 files changed, 123 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/adb34bdf/CHANGES ---------------------------------------------------------------------- diff --git a/CHANGES b/CHANGES index 6438503..910c175 100644 --- a/CHANGES +++ b/CHANGES @@ -95,6 +95,9 @@ Release 0.9.1 - unreleased BUG FIXES + TAJO-1244: tajo.worker.tmpdir.locations should use a validator for a list + of paths. (hyunsik) + TAJO-1235: ByteBufLineReader can not read text line with CRLF. (jinho) http://git-wip-us.apache.org/repos/asf/tajo/blob/adb34bdf/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java ---------------------------------------------------------------------- diff --git a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java index a38bd6c..252b8f0 100644 --- a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java +++ b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java @@ -181,8 +181,7 @@ public class TajoConf extends Configuration { WORKER_QM_RPC_ADDRESS("tajo.worker.qm-rpc.address", "0.0.0.0:28093", Validators.networkAddr()), // Tajo Worker Temporal Directories - WORKER_TEMPORAL_DIR("tajo.worker.tmpdir.locations", "/tmp/tajo-${user.name}/tmpdir", - Validators.pathUrl()), + WORKER_TEMPORAL_DIR("tajo.worker.tmpdir.locations", "/tmp/tajo-${user.name}/tmpdir", Validators.pathUrlList()), WORKER_TEMPORAL_DIR_CLEANUP("tajo.worker.tmpdir.cleanup-at-startup", false, Validators.bool()), // Tajo Worker Resources http://git-wip-us.apache.org/repos/asf/tajo/blob/adb34bdf/tajo-common/src/main/java/org/apache/tajo/rule/SelfDiagnosisRuleDefinition.java ---------------------------------------------------------------------- diff --git a/tajo-common/src/main/java/org/apache/tajo/rule/SelfDiagnosisRuleDefinition.java b/tajo-common/src/main/java/org/apache/tajo/rule/SelfDiagnosisRuleDefinition.java index ae1d8ac..210756c 100644 --- a/tajo-common/src/main/java/org/apache/tajo/rule/SelfDiagnosisRuleDefinition.java +++ b/tajo-common/src/main/java/org/apache/tajo/rule/SelfDiagnosisRuleDefinition.java @@ -32,5 +32,6 @@ public @interface SelfDiagnosisRuleDefinition { public String name(); public int priority() default -1; - + + public boolean enabled() default true; } http://git-wip-us.apache.org/repos/asf/tajo/blob/adb34bdf/tajo-common/src/main/java/org/apache/tajo/rule/SelfDiagnosisRuleEngine.java ---------------------------------------------------------------------- diff --git a/tajo-common/src/main/java/org/apache/tajo/rule/SelfDiagnosisRuleEngine.java b/tajo-common/src/main/java/org/apache/tajo/rule/SelfDiagnosisRuleEngine.java index 12bd4f6..df0225d 100644 --- a/tajo-common/src/main/java/org/apache/tajo/rule/SelfDiagnosisRuleEngine.java +++ b/tajo-common/src/main/java/org/apache/tajo/rule/SelfDiagnosisRuleEngine.java @@ -65,14 +65,16 @@ public class SelfDiagnosisRuleEngine { private void loadRuleData(List<SelfDiagnosisRule> ruleList) { for (SelfDiagnosisRule rule: ruleList) { RuleWrapper wrapper = new RuleWrapper(rule); - Map<String, RuleWrapper> categoryMap = wrapperMap.get(wrapper.getCategoryName()); - - if (categoryMap == null) { - categoryMap = TUtil.newHashMap(); - wrapperMap.put(wrapper.getCategoryName(), categoryMap); + if (wrapper.isEnabled()) { + Map<String, RuleWrapper> categoryMap = wrapperMap.get(wrapper.getCategoryName()); + + if (categoryMap == null) { + categoryMap = TUtil.newHashMap(); + wrapperMap.put(wrapper.getCategoryName(), categoryMap); + } + + categoryMap.put(wrapper.getRuleName(), wrapper); } - - categoryMap.put(wrapper.getRuleName(), wrapper); } } @@ -92,6 +94,7 @@ public class SelfDiagnosisRuleEngine { private final String categoryName; private final String ruleName; private final int priority; + private final boolean enabled; private final Class<?>[] acceptedCallers; private final SelfDiagnosisRule rule; @@ -105,6 +108,7 @@ public class SelfDiagnosisRuleEngine { categoryName = ruleDefinition.category(); ruleName = ruleDefinition.name(); priority = ruleDefinition.priority(); + enabled = ruleDefinition.enabled(); SelfDiagnosisRuleVisibility.LimitedPrivate limitedPrivateScope = rule.getClass().getAnnotation(SelfDiagnosisRuleVisibility.LimitedPrivate.class); @@ -137,6 +141,10 @@ public class SelfDiagnosisRuleEngine { return priority; } + public boolean isEnabled() { + return enabled; + } + @Override public int compareTo(RuleWrapper o) { if (getPriority() == -1 && o.getPriority() == -1) { http://git-wip-us.apache.org/repos/asf/tajo/blob/adb34bdf/tajo-common/src/main/java/org/apache/tajo/validation/PathListValidator.java ---------------------------------------------------------------------- diff --git a/tajo-common/src/main/java/org/apache/tajo/validation/PathListValidator.java b/tajo-common/src/main/java/org/apache/tajo/validation/PathListValidator.java new file mode 100644 index 0000000..a580cb5 --- /dev/null +++ b/tajo-common/src/main/java/org/apache/tajo/validation/PathListValidator.java @@ -0,0 +1,63 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.validation; + +import org.apache.tajo.util.TUtil; + +import java.util.Collection; + +public class PathListValidator extends AbstractValidator { + private static final String LIST_SEPARATOR = ","; + + @Override + protected <T> String getErrorMessage(T object) { + return object + " is not valid path list."; + } + + @Override + protected <T> boolean validateInternal(T object) { + PathValidator validator = (PathValidator) Validators.pathUrl(); + + boolean result = true; + + if (object != null) { + if (object instanceof CharSequence) { + String valueString = object.toString(); + if (valueString.isEmpty()) { + result = true; + } else { + String [] splits = object.toString().split(LIST_SEPARATOR); + for (String path : splits) { + result &= validator.validateInternal(path.trim()); + } + } + } + } else { + result = true; + } + + return result; + } + + @Override + protected Collection<Validator> getDependantValidators() { + return TUtil.newList(); + } + +} http://git-wip-us.apache.org/repos/asf/tajo/blob/adb34bdf/tajo-common/src/main/java/org/apache/tajo/validation/PathValidator.java ---------------------------------------------------------------------- diff --git a/tajo-common/src/main/java/org/apache/tajo/validation/PathValidator.java b/tajo-common/src/main/java/org/apache/tajo/validation/PathValidator.java index a76587c..9548e80 100644 --- a/tajo-common/src/main/java/org/apache/tajo/validation/PathValidator.java +++ b/tajo-common/src/main/java/org/apache/tajo/validation/PathValidator.java @@ -19,9 +19,11 @@ package org.apache.tajo.validation; public class PathValidator extends PatternValidator { + static final String PATH_REGEXP_PATTERN = + "^(?:[a-zA-Z][a-zA-Z0-9+-.]+:[/]{1,2}[a-zA-Z-.]*[:0-9]*)?(?:/?[a-zA-Z]:)?[/a-zA-Z0-9-_\\\\.\\\\$\\\\{\\\\}]*$"; public PathValidator() { - super("^(?:[a-zA-Z][a-zA-Z0-9+-.]+:[/]{1,2}[a-zA-Z-.]*[:0-9]*)?(?:/?[a-zA-Z]:)?[/a-zA-Z0-9-_\\\\.\\\\$\\\\{\\\\}]*$"); + super(PATH_REGEXP_PATTERN); } @Override http://git-wip-us.apache.org/repos/asf/tajo/blob/adb34bdf/tajo-common/src/main/java/org/apache/tajo/validation/Validators.java ---------------------------------------------------------------------- diff --git a/tajo-common/src/main/java/org/apache/tajo/validation/Validators.java b/tajo-common/src/main/java/org/apache/tajo/validation/Validators.java index acd4876..d4a3fa4 100644 --- a/tajo-common/src/main/java/org/apache/tajo/validation/Validators.java +++ b/tajo-common/src/main/java/org/apache/tajo/validation/Validators.java @@ -53,6 +53,10 @@ public class Validators { public static Validator pathUrl() { return new PathValidator(); } + + public static Validator pathUrlList() { + return new PathListValidator(); + } public static Validator shellVar() { return new ShellVariableValidator(); http://git-wip-us.apache.org/repos/asf/tajo/blob/adb34bdf/tajo-common/src/test/java/org/apache/tajo/validation/TestValidators.java ---------------------------------------------------------------------- diff --git a/tajo-common/src/test/java/org/apache/tajo/validation/TestValidators.java b/tajo-common/src/test/java/org/apache/tajo/validation/TestValidators.java index 16ce49f..c539a51 100644 --- a/tajo-common/src/test/java/org/apache/tajo/validation/TestValidators.java +++ b/tajo-common/src/test/java/org/apache/tajo/validation/TestValidators.java @@ -397,6 +397,36 @@ public class TestValidators { assertThat(new PathValidator().validate(invalidUrl), hasItem(hasAClass(equalTo(PathValidator.class)))); } + + @Test + public void testPathListValidator() { + String validUrls = + "file:///tmp/tajo-$root/a,file:///tmp/tajo-${user.name}/,file:/home/tajo/test-data/TestExternalSortExec"; + assertThat(new PathListValidator().validateInternal(validUrls), is(true)); + assertThat(new PathListValidator().validate(validUrls).size(), is(0)); + + validUrls = + "file:///tmp/tajo-$root/a, file:///tmp/tajo-${user.name}/, file:/home/tajo/test-data/TestExternalSortExec"; + assertThat(new PathListValidator().validateInternal(validUrls), is(true)); + assertThat(new PathListValidator().validate(validUrls).size(), is(0)); + + validUrls = "/tmp/tajo-hyunsik/tmpdir1,/tmp/tajo-hyunsik/tmpdir2"; + assertThat(new PathListValidator().validateInternal(validUrls), is(true)); + assertThat(new PathListValidator().validate(validUrls).size(), is(0)); + + String invalidUrls = + "file:///tmp/tajo-$root/a, t!ef:///tmp/tajo-root, file:/home/tajo/test-data/TestExternalSortExec"; + assertThat(new PathListValidator().validateInternal(invalidUrls), is(false)); + assertThat(new PathListValidator().validate(invalidUrls).size(), is(1)); + assertThat(new PathListValidator().validate(invalidUrls), + hasItem(hasAClass(equalTo(PathListValidator.class)))); + + invalidUrls = "This is not a valid url,This is not a valid url"; + assertThat(new PathListValidator().validateInternal(invalidUrls), is(false)); + assertThat(new PathListValidator().validate(invalidUrls).size(), is(1)); + assertThat(new PathListValidator().validate(invalidUrls), + hasItem(hasAClass(equalTo(PathListValidator.class)))); + } @Test public void testShellVariableValidator() { http://git-wip-us.apache.org/repos/asf/tajo/blob/adb34bdf/tajo-core/src/main/java/org/apache/tajo/worker/rule/ConnectivityCheckerRuleForTajoWorker.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/rule/ConnectivityCheckerRuleForTajoWorker.java b/tajo-core/src/main/java/org/apache/tajo/worker/rule/ConnectivityCheckerRuleForTajoWorker.java index dcd40bf..328a31b 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/rule/ConnectivityCheckerRuleForTajoWorker.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/rule/ConnectivityCheckerRuleForTajoWorker.java @@ -38,7 +38,8 @@ import org.apache.tajo.worker.TajoWorker; /** * With this rule, Tajo worker will check the connectivity to tajo master server. */ -@SelfDiagnosisRuleDefinition(category="worker", name="ConnectivityCheckerRuleForTajoWorker", priority=0) +@SelfDiagnosisRuleDefinition( + category="worker", name="ConnectivityCheckerRuleForTajoWorker", priority=0, enabled = false) @SelfDiagnosisRuleVisibility.LimitedPrivate(acceptedCallers = { TajoWorker.class }) public class ConnectivityCheckerRuleForTajoWorker implements SelfDiagnosisRule {
