Repository: tajo
Updated Branches:
refs/heads/branch-0.11.0 646cec155 -> 855127f2c
TAJO-1867: TajoMaster should handle the change of ${tajo.root}.
Closes #767
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/855127f2
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/855127f2
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/855127f2
Branch: refs/heads/branch-0.11.0
Commit: 855127f2c65fdfeb6efddc82d6565f15dc78ff46
Parents: 646cec1
Author: Hyunsik Choi <[email protected]>
Authored: Thu Sep 17 18:23:39 2015 -0700
Committer: Hyunsik Choi <[email protected]>
Committed: Thu Sep 17 18:33:51 2015 -0700
----------------------------------------------------------------------
CHANGES | 5 +++
.../src/main/proto/CatalogProtos.proto | 6 +--
.../org/apache/tajo/catalog/CatalogServer.java | 4 +-
.../tajo/catalog/store/AbstractDBStore.java | 4 +-
.../org/apache/tajo/catalog/TestCatalog.java | 5 +--
.../TestCatalogAgainstCaseSensitivity.java | 7 +---
.../tajo/catalog/TestCatalogExceptions.java | 8 +---
.../java/org/apache/tajo/master/TajoMaster.java | 41 ++++++++++++++++++--
.../apache/tajo/master/exec/DDLExecutor.java | 2 +-
9 files changed, 55 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/855127f2/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 79537a8..d0111ba 100644
--- a/CHANGES
+++ b/CHANGES
@@ -38,6 +38,8 @@ Release 0.11.0 - unreleased
IMPROVEMENT
+ TAJO-1867: TajoMaster should handle the change of ${tajo.root}. (hyunsik)
+
TAJO-1134: start-tajo.sh should display WEB UI URL and TajoMaster RPC
address. (Contributed YeonSu Han, committed by hyunsik)
@@ -274,6 +276,9 @@ Release 0.11.0 - unreleased
TAJO-1846: Python temp directory path should be selected differently based
on user platform. (Contributed by Dongkyu Hwangbo, Committed by jihoon)
+ TAJO-1848: ShutdownHook in TajoMaster can throw NPE if serviceInit().
+ (hyunsik)
+
TAJO-1851: Can not release a different rack task. (jinho)
TAJO-1830: Fix race condition in HdfsServiceTracker. (jinho)
http://git-wip-us.apache.org/repos/asf/tajo/blob/855127f2/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto
----------------------------------------------------------------------
diff --git
a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto
b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto
index cfac82f..cec528a 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto
+++ b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto
@@ -292,13 +292,9 @@ message AlterTablespaceProto {
LOCATION = 0;
}
- message SetLocation {
- required string uri = 1;
- }
-
message AlterTablespaceCommand {
required AlterTablespaceType type = 1;
- optional SetLocation location = 2;
+ optional string location = 2;
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/855127f2/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java
----------------------------------------------------------------------
diff --git
a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java
b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java
index 8f4e0e8..a60ddda 100644
---
a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java
+++
b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java
@@ -374,11 +374,11 @@ public class CatalogServer extends AbstractService {
for (AlterTablespaceCommand command : request.getCommandList()) {
if (command.getType() ==
AlterTablespaceProto.AlterTablespaceType.LOCATION) {
try {
- URI uri = URI.create(command.getLocation().getUri());
+ URI uri = URI.create(command.getLocation());
Preconditions.checkArgument(uri.getScheme() != null);
} catch (Exception e) {
throw new ServiceException("ALTER TABLESPACE's LOCATION must
be a URI form (scheme:///.../), but "
- + command.getLocation().getUri());
+ + command.getLocation());
}
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/855127f2/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java
----------------------------------------------------------------------
diff --git
a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java
b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java
index 0f07149..c4d1828 100644
---
a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java
+++
b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java
@@ -544,12 +544,12 @@ public abstract class AbstractDBStore extends
CatalogConstants implements Catalo
if (alterProto.getCommandList().size() == 1) {
AlterTablespaceCommand command = alterProto.getCommand(0);
if (command.getType() ==
AlterTablespaceProto.AlterTablespaceType.LOCATION) {
- AlterTablespaceProto.SetLocation setLocation = command.getLocation();
+ final String uri = command.getLocation();
try {
String sql = "UPDATE " + TB_SPACES + " SET SPACE_URI=? WHERE
SPACE_NAME=?";
pstmt = conn.prepareStatement(sql);
- pstmt.setString(1, setLocation.getUri());
+ pstmt.setString(1, uri);
pstmt.setString(2, alterProto.getSpaceName());
pstmt.executeUpdate();
} catch (SQLException se) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/855127f2/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java
----------------------------------------------------------------------
diff --git
a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java
b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java
index 8720105..a5e4861 100644
---
a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java
+++
b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java
@@ -47,7 +47,6 @@ import java.util.*;
import static org.apache.tajo.TajoConstants.DEFAULT_DATABASE_NAME;
import static org.apache.tajo.catalog.proto.CatalogProtos.AlterTablespaceProto;
import static
org.apache.tajo.catalog.proto.CatalogProtos.AlterTablespaceProto.AlterTablespaceType;
-import static
org.apache.tajo.catalog.proto.CatalogProtos.AlterTablespaceProto.SetLocation;
import static org.junit.Assert.*;
public class TestCatalog {
@@ -99,7 +98,7 @@ public class TestCatalog {
AlterTablespaceProto.AlterTablespaceCommand.Builder commandBuilder =
AlterTablespaceProto.AlterTablespaceCommand.newBuilder();
commandBuilder.setType(AlterTablespaceType.LOCATION);
-
commandBuilder.setLocation(SetLocation.newBuilder().setUri("hdfs://zzz.com/warehouse"));
+ commandBuilder.setLocation("hdfs://zzz.com/warehouse");
AlterTablespaceProto.Builder alter = AlterTablespaceProto.newBuilder();
alter.setSpaceName("space1");
alter.addCommand(commandBuilder.build());
@@ -122,7 +121,7 @@ public class TestCatalog {
// ALTER TABLESPACE space1 LOCATION 'hdfs://zzz.com/warehouse';
commandBuilder = AlterTablespaceProto.AlterTablespaceCommand.newBuilder();
commandBuilder.setType(AlterTablespaceType.LOCATION);
-
commandBuilder.setLocation(SetLocation.newBuilder().setUri("hdfs://www.com/warehouse"));
+ commandBuilder.setLocation("hdfs://www.com/warehouse");
alter = AlterTablespaceProto.newBuilder();
alter.setSpaceName("space2");
alter.addCommand(commandBuilder.build());
http://git-wip-us.apache.org/repos/asf/tajo/blob/855127f2/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogAgainstCaseSensitivity.java
----------------------------------------------------------------------
diff --git
a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogAgainstCaseSensitivity.java
b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogAgainstCaseSensitivity.java
index bfff6b4..594f5ce 100644
---
a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogAgainstCaseSensitivity.java
+++
b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogAgainstCaseSensitivity.java
@@ -22,7 +22,6 @@ import org.apache.tajo.catalog.partition.PartitionDesc;
import org.apache.tajo.catalog.proto.CatalogProtos.*;
import
org.apache.tajo.catalog.proto.CatalogProtos.AlterTablespaceProto.AlterTablespaceCommand;
import
org.apache.tajo.catalog.proto.CatalogProtos.AlterTablespaceProto.AlterTablespaceType;
-import
org.apache.tajo.catalog.proto.CatalogProtos.AlterTablespaceProto.SetLocation;
import org.apache.tajo.catalog.statistics.TableStats;
import org.apache.tajo.common.TajoDataTypes.Type;
import org.apache.tajo.exception.UndefinedPartitionException;
@@ -68,16 +67,14 @@ public class TestCatalogAgainstCaseSensitivity {
addCommand(
AlterTablespaceCommand.newBuilder().
setType(AlterTablespaceType.LOCATION).
- setLocation(SetLocation.newBuilder()
- .setUri("hdfs://zzz.com/warehouse"))).build());
+ setLocation("hdfs://zzz.com/warehouse")).build());
catalog.alterTablespace(AlterTablespaceProto.newBuilder().
setSpaceName("SpAcE1").
addCommand(
AlterTablespaceCommand.newBuilder().
setType(AlterTablespaceType.LOCATION).
- setLocation(SetLocation.newBuilder()
- .setUri("hdfs://zzz.com/warehouse"))).build());
+ setLocation("hdfs://zzz.com/warehouse")).build());
Set<TablespaceProto> tablespaceProtos = new HashSet<>();
for (String tablespaceName : catalog.getAllTablespaceNames()) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/855127f2/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogExceptions.java
----------------------------------------------------------------------
diff --git
a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogExceptions.java
b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogExceptions.java
index 7be34d8..6586cec 100644
---
a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogExceptions.java
+++
b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogExceptions.java
@@ -24,7 +24,6 @@ import org.apache.tajo.catalog.partition.PartitionDesc;
import org.apache.tajo.catalog.proto.CatalogProtos.AlterTablespaceProto;
import
org.apache.tajo.catalog.proto.CatalogProtos.AlterTablespaceProto.AlterTablespaceCommand;
import
org.apache.tajo.catalog.proto.CatalogProtos.AlterTablespaceProto.AlterTablespaceType;
-import
org.apache.tajo.catalog.proto.CatalogProtos.AlterTablespaceProto.SetLocation;
import org.apache.tajo.catalog.proto.CatalogProtos.IndexMethod;
import org.apache.tajo.catalog.proto.CatalogProtos.TableStatsProto;
import org.apache.tajo.catalog.proto.CatalogProtos.UpdateTableStatsProto;
@@ -37,7 +36,6 @@ import org.junit.BeforeClass;
import org.junit.Test;
import java.net.URI;
-import java.util.UUID;
public class TestCatalogExceptions {
@@ -85,8 +83,7 @@ public class TestCatalogExceptions {
addCommand(
AlterTablespaceCommand.newBuilder().
setType(AlterTablespaceType.LOCATION).
- setLocation(SetLocation.newBuilder()
- .setUri("hdfs:"))).build());
+ setLocation("hdfs:")).build());
}
@Test(expected = UndefinedTablespaceException.class)
@@ -96,8 +93,7 @@ public class TestCatalogExceptions {
addCommand(
AlterTablespaceCommand.newBuilder().
setType(AlterTablespaceType.LOCATION).
- setLocation(SetLocation.newBuilder()
- .setUri("hdfs://zzz.com/warehouse"))).build());
+ setLocation("hdfs://zzz.com/warehouse")).build());
}
@Test(expected = DuplicateDatabaseException.class)
http://git-wip-us.apache.org/repos/asf/tajo/blob/855127f2/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java
b/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java
index 326bc86..2f37cee 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java
@@ -34,17 +34,20 @@ import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.util.Clock;
import org.apache.hadoop.yarn.util.RackResolver;
import org.apache.hadoop.yarn.util.SystemClock;
+import org.apache.tajo.algebra.AlterTablespace;
import org.apache.tajo.catalog.CatalogServer;
import org.apache.tajo.catalog.CatalogService;
import org.apache.tajo.catalog.FunctionDesc;
import org.apache.tajo.catalog.LocalCatalogWrapper;
+import org.apache.tajo.catalog.proto.CatalogProtos;
+import org.apache.tajo.catalog.proto.CatalogProtos.AlterTablespaceProto;
+import
org.apache.tajo.catalog.proto.CatalogProtos.AlterTablespaceProto.AlterTablespaceCommand;
import org.apache.tajo.catalog.store.AbstractDBStore;
import org.apache.tajo.catalog.store.DerbyStore;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.conf.TajoConf.ConfVars;
import org.apache.tajo.engine.function.FunctionLoader;
-import org.apache.tajo.exception.DuplicateDatabaseException;
-import org.apache.tajo.exception.DuplicateTablespaceException;
+import org.apache.tajo.exception.*;
import org.apache.tajo.function.FunctionSignature;
import org.apache.tajo.master.rm.TajoResourceManager;
import org.apache.tajo.metrics.ClusterResourceMetricSet;
@@ -382,7 +385,39 @@ public class TajoMaster extends CompositeService {
private void checkBaseTBSpaceAndDatabase()
throws IOException, DuplicateDatabaseException,
DuplicateTablespaceException {
- if (!catalog.existTablespace(DEFAULT_TABLESPACE_NAME)) {
+ if (catalog.existTablespace(DEFAULT_TABLESPACE_NAME)) { // if default
tablespace already exists
+
+ CatalogProtos.TablespaceProto tablespace = null;
+ try {
+ tablespace = catalog.getTablespace(DEFAULT_TABLESPACE_NAME);
+ } catch (UndefinedTablespaceException e) {
+ throw new TajoInternalError(e);
+ }
+
+ // if warehouse directory and the location of default tablespace are
different from each other
+ if
(!tablespace.getUri().equals(context.getConf().getVar(ConfVars.WAREHOUSE_DIR)))
{
+ AlterTablespaceCommand.Builder alterCommand =
+ AlterTablespaceCommand.newBuilder()
+ .setType(AlterTablespaceProto.AlterTablespaceType.LOCATION)
+ .setLocation(context.getConf().getVar(ConfVars.WAREHOUSE_DIR));
+
+ AlterTablespaceProto alterTablespace =
AlterTablespaceProto.newBuilder()
+ .setSpaceName(DEFAULT_TABLESPACE_NAME)
+ .addCommand(alterCommand).build();
+
+ // update the location of default tablespace
+ try {
+ catalog.alterTablespace(alterTablespace);
+ } catch (TajoException e) {
+ throw new TajoInternalError(e);
+ }
+
+ LOG.warn(
+ "The location of default tablespace has been changed. " +
+ "You may not accept existing managed tables stored in the previous
default tablespace");
+ }
+
+ } else if (!catalog.existTablespace(DEFAULT_TABLESPACE_NAME)) { // if the
default tablespace does not exists
catalog.createTablespace(DEFAULT_TABLESPACE_NAME,
context.getConf().getVar(ConfVars.WAREHOUSE_DIR));
} else {
LOG.info(String.format("Default tablespace (%s) is already prepared.",
DEFAULT_TABLESPACE_NAME));
http://git-wip-us.apache.org/repos/asf/tajo/blob/855127f2/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java
----------------------------------------------------------------------
diff --git
a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java
b/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java
index 9f80802..c5d5838 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java
@@ -206,7 +206,7 @@ public class DDLExecutor {
AlterTablespaceProto.AlterTablespaceCommand.Builder commandBuilder =
AlterTablespaceProto.AlterTablespaceCommand.newBuilder();
commandBuilder.setType(AlterTablespaceProto.AlterTablespaceType.LOCATION);
-
commandBuilder.setLocation(AlterTablespaceProto.SetLocation.newBuilder().setUri(alterTablespace.getLocation()));
+ commandBuilder.setLocation(alterTablespace.getLocation());
commandBuilder.build();
builder.addCommand(commandBuilder);
} else {
