Repository: tajo Updated Branches: refs/heads/index_support 16970e5b4 -> 5a8fed408
TAJO-1302: Support index metadata backup and restore. (jihoon) Closes #617 Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/5a8fed40 Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/5a8fed40 Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/5a8fed40 Branch: refs/heads/index_support Commit: 5a8fed408f57d1484c947420e4610c49fe0467ba Parents: 16970e5 Author: Jihoon Son <[email protected]> Authored: Fri Jul 24 12:35:42 2015 +0900 Committer: Jihoon Son <[email protected]> Committed: Fri Jul 24 12:36:39 2015 +0900 ---------------------------------------------------------------------- CHANGES | 2 + .../org/apache/tajo/catalog/DDLBuilder.java | 22 +++++ .../org/apache/tajo/catalog/store/MemStore.java | 25 ------ .../org/apache/tajo/cli/tools/TajoDump.java | 11 ++- .../apache/tajo/master/exec/DDLExecutor.java | 2 +- .../java/org/apache/tajo/querymaster/Query.java | 2 +- .../org/apache/tajo/cli/tools/TestTajoDump.java | 95 ++++++++++++++++---- .../results/TestTajoDump/testDump1.result | 3 +- .../results/TestTajoDump/testDump2.result | 3 +- .../results/TestTajoDump/testDump3.result | 18 ++++ 10 files changed, 129 insertions(+), 54 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/5a8fed40/CHANGES ---------------------------------------------------------------------- diff --git a/CHANGES b/CHANGES index 7af611d..9fb7eb3 100644 --- a/CHANGES +++ b/CHANGES @@ -406,6 +406,8 @@ Release 0.11.0 - unreleased SUB TASKS + TAJO-1302: Support index metadata backup and restore. (jihoon) + TAJO-1464: Add ORCFileScanner to read ORCFile table. (Contributed by Jongyoung Park, Committed by jihoon) http://git-wip-us.apache.org/repos/asf/tajo/blob/5a8fed40/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java index f313aa5..2923654 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java @@ -68,6 +68,28 @@ public class DDLBuilder { return sb.toString(); } + public static String buildDDLForIndex(IndexDesc desc) { + StringBuilder sb = new StringBuilder(); + + sb.append("--\n") + .append("-- Name: ").append(CatalogUtil.denormalizeIdentifier(desc.getName())).append("; Type: INDEX;") + .append(" Index Method: ").append(desc.getIndexMethod()); + sb.append("\n--\n"); + sb.append("CREATE INDEX ").append(CatalogUtil.denormalizeIdentifier(desc.getName())); + sb.append(" on ").append(CatalogUtil.denormalizeIdentifier(desc.getTableName())).append(" ( "); + + for (SortSpec sortSpec : desc.getKeySortSpecs()) { + sb.append(sortSpec.getSortKey().getQualifiedName()).append(" "); + sb.append(sortSpec.isAscending() ? "asc" : "desc").append(" "); + sb.append(sortSpec.isNullFirst() ? "null first" : "null last").append(", "); + } + sb.replace(sb.length()-2, sb.length()-1, " )"); + + sb.append(" location '").append(desc.getIndexPath()).append("';"); + + return sb.toString(); + } + public static void buildSchema(StringBuilder sb, Schema schema) { boolean first = true; http://git-wip-us.apache.org/repos/asf/tajo/blob/5a8fed40/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java ---------------------------------------------------------------------- diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java index 2ad11a7..74b6023 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/MemStore.java @@ -722,31 +722,6 @@ public class MemStore implements CatalogStore { return false; } -// public List<IndexProto> getAllIndexes() throws CatalogException { -// List<IndexProto> indexList = new ArrayList<CatalogProtos.IndexProto>(); -// Set<String> databases = indexes.keySet(); -// -// for (String databaseName: databases) { -// Map<String, IndexDescProto> indexMap = indexes.get(databaseName); -// -// for (Map.Entry<String, IndexDescProto> entry: indexMap.entrySet()) { -// IndexDescProto indexDesc = entry.getValue(); -// IndexProto.Builder builder = IndexProto.newBuilder(); -// -// builder.setColumnName(indexDesc.getColumn().getName()); -// builder.setDataType(indexDesc.getColumn().getDataType().getType().toString()); -// builder.setIndexName(entry.getKey()); -// builder.setIndexType(indexDesc.getIndexMethod().toString()); -// builder.setIsAscending(indexDesc.hasIsAscending() && indexDesc.getIsAscending()); -// builder.setIsClustered(indexDesc.hasIsClustered() && indexDesc.getIsClustered()); -// builder.setIsUnique(indexDesc.hasIsUnique() && indexDesc.getIsUnique()); -// -// indexList.add(builder.build()); -// } -// } -// return false; -// } - @Override public List<IndexDescProto> getAllIndexes() throws CatalogException { List<IndexDescProto> indexDescProtos = TUtil.newList(); http://git-wip-us.apache.org/repos/asf/tajo/blob/5a8fed40/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoDump.java ---------------------------------------------------------------------- diff --git a/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoDump.java b/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoDump.java index 7fefc5a..20665e8 100644 --- a/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoDump.java +++ b/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoDump.java @@ -22,10 +22,8 @@ import com.google.protobuf.ServiceException; import org.apache.commons.cli.*; import org.apache.tajo.auth.UserRoleInfo; -import org.apache.tajo.catalog.CatalogConstants; -import org.apache.tajo.catalog.CatalogUtil; -import org.apache.tajo.catalog.DDLBuilder; -import org.apache.tajo.catalog.TableDesc; +import org.apache.tajo.catalog.*; +import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.client.TajoClient; import org.apache.tajo.client.TajoClientImpl; @@ -186,6 +184,11 @@ public class TajoDump { } else { writer.write(DDLBuilder.buildDDLForBaseTable(table)); } + + List<CatalogProtos.IndexDescProto> indexeProtos = client.getIndexes(tableName); + for (CatalogProtos.IndexDescProto eachIndexProto : indexeProtos) { + writer.write(DDLBuilder.buildDDLForIndex(new IndexDesc(eachIndexProto))); + } writer.write("\n\n"); } catch (Exception e) { // dump for each table can throw any exception. We need to skip the exception case. http://git-wip-us.apache.org/repos/asf/tajo/blob/5a8fed40/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java index 96c0812..a535f94 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java @@ -143,7 +143,7 @@ public class DDLExecutor { throw new InternalError("Cannot find the table of the relation"); } - IndexDesc indexDesc = new IndexDesc(databaseName, scanNode.getTableName(), + IndexDesc indexDesc = new IndexDesc(databaseName, CatalogUtil.extractSimpleName(scanNode.getTableName()), simpleIndexName, createIndexNode.getIndexPath(), createIndexNode.getKeySortSpecs(), createIndexNode.getIndexMethod(), createIndexNode.isUnique(), false, scanNode.getLogicalSchema()); http://git-wip-us.apache.org/repos/asf/tajo/blob/5a8fed40/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java index 1f86917..e3629c7 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java @@ -559,7 +559,7 @@ public class Query implements EventHandler<QueryEvent> { if (scanNode == null) { throw new IOException("Cannot find the table of the relation"); } - IndexDesc indexDesc = new IndexDesc(databaseName, scanNode.getTableName(), + IndexDesc indexDesc = new IndexDesc(databaseName, CatalogUtil.extractSimpleName(scanNode.getTableName()), simpleIndexName, createIndexNode.getIndexPath(), createIndexNode.getKeySortSpecs(), createIndexNode.getIndexMethod(), createIndexNode.isUnique(), false, scanNode.getLogicalSchema()); http://git-wip-us.apache.org/repos/asf/tajo/blob/5a8fed40/tajo-core/src/test/java/org/apache/tajo/cli/tools/TestTajoDump.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/cli/tools/TestTajoDump.java b/tajo-core/src/test/java/org/apache/tajo/cli/tools/TestTajoDump.java index b371be2..aa8070e 100644 --- a/tajo-core/src/test/java/org/apache/tajo/cli/tools/TestTajoDump.java +++ b/tajo-core/src/test/java/org/apache/tajo/cli/tools/TestTajoDump.java @@ -18,13 +18,22 @@ package org.apache.tajo.cli.tools; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.apache.tajo.QueryTestCaseBase; import org.apache.tajo.auth.UserRoleInfo; +import org.apache.tajo.storage.StorageUtil; +import org.apache.tajo.storage.TablespaceManager; +import org.apache.tajo.util.FileUtil; import org.junit.Test; import java.io.ByteArrayOutputStream; +import java.io.File; import java.io.PrintWriter; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + public class TestTajoDump extends QueryTestCaseBase { @Test @@ -33,16 +42,18 @@ public class TestTajoDump extends QueryTestCaseBase { executeString("CREATE TABLE \"" + getCurrentDatabase() + "\".\"TableName1\" (\"Age\" int, \"FirstName\" TEXT, lastname TEXT)"); - UserRoleInfo userInfo = UserRoleInfo.getCurrentUser(); - ByteArrayOutputStream bos = new ByteArrayOutputStream(); - PrintWriter printWriter = new PrintWriter(bos); - TajoDump.dump(client, userInfo, getCurrentDatabase(), false, false, false, printWriter); - printWriter.flush(); - printWriter.close(); - assertStrings(new String(bos.toByteArray())); - bos.close(); - - executeString("DROP TABLE \"" + getCurrentDatabase() + "\".\"TableName1\""); + try { + UserRoleInfo userInfo = UserRoleInfo.getCurrentUser(); + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + PrintWriter printWriter = new PrintWriter(bos); + TajoDump.dump(client, userInfo, getCurrentDatabase(), false, false, false, printWriter); + printWriter.flush(); + printWriter.close(); + assertStrings(new String(bos.toByteArray())); + bos.close(); + } finally { + executeString("DROP TABLE \"" + getCurrentDatabase() + "\".\"TableName1\""); + } } } @@ -52,16 +63,62 @@ public class TestTajoDump extends QueryTestCaseBase { executeString("CREATE TABLE \"" + getCurrentDatabase() + "\".\"TableName2\" (\"Age\" int, \"Name\" Record (\"FirstName\" TEXT, lastname TEXT))"); - UserRoleInfo userInfo = UserRoleInfo.getCurrentUser(); - ByteArrayOutputStream bos = new ByteArrayOutputStream(); - PrintWriter printWriter = new PrintWriter(bos); - TajoDump.dump(client, userInfo, getCurrentDatabase(), false, false, false, printWriter); - printWriter.flush(); - printWriter.close(); - assertStrings(new String(bos.toByteArray())); - bos.close(); + try { + UserRoleInfo userInfo = UserRoleInfo.getCurrentUser(); + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + PrintWriter printWriter = new PrintWriter(bos); + TajoDump.dump(client, userInfo, getCurrentDatabase(), false, false, false, printWriter); + printWriter.flush(); + printWriter.close(); + assertStrings(new String(bos.toByteArray())); + bos.close(); + } finally { + executeString("DROP TABLE \"" + getCurrentDatabase() + "\".\"TableName2\""); + } + } + } + + @Test + public void testDump3() throws Exception { + if (!testingCluster.isHiveCatalogStoreRunning()) { + executeString("CREATE TABLE \"" + getCurrentDatabase() + + "\".\"TableName1\" (\"Age\" int, \"FirstName\" TEXT, lastname TEXT)"); + + executeString("CREATE INDEX test_idx on \"" + getCurrentDatabase() + + "\".\"TableName1\" ( \"Age\" asc null first, \"FirstName\" desc null last )"); + + try { + UserRoleInfo userInfo = UserRoleInfo.getCurrentUser(); + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + PrintWriter printWriter = new PrintWriter(bos); + TajoDump.dump(client, userInfo, getCurrentDatabase(), false, false, false, printWriter); + printWriter.flush(); + printWriter.close(); + assertOutputResult("testDump3.result", new String(bos.toByteArray()), new String[]{"${index.path}"}, + new String[]{TablespaceManager.getDefault().getTableUri(getCurrentDatabase(), "test_idx").toString()}); + bos.close(); + } finally { + executeString("DROP INDEX test_idx"); + executeString("DROP TABLE \"" + getCurrentDatabase() + "\".\"TableName1\""); + } + } + } + + private void assertOutputResult(String expectedResultFile, String actual, String[] paramKeys, String[] paramValues) + throws Exception { + FileSystem fs = currentResultPath.getFileSystem(testBase.getTestingCluster().getConfiguration()); + Path resultFile = StorageUtil.concatPath(currentResultPath, expectedResultFile); + assertTrue(resultFile.toString() + " existence check", fs.exists(resultFile)); + + String expectedResult = FileUtil.readTextFile(new File(resultFile.toUri())); - executeString("DROP TABLE \"" + getCurrentDatabase() + "\".\"TableName2\""); + if (paramKeys != null) { + for (int i = 0; i < paramKeys.length; i++) { + if (i < paramValues.length) { + expectedResult = expectedResult.replace(paramKeys[i], paramValues[i]); + } + } } + assertEquals(expectedResult.trim(), actual.trim()); } } http://git-wip-us.apache.org/repos/asf/tajo/blob/5a8fed40/tajo-core/src/test/resources/results/TestTajoDump/testDump1.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestTajoDump/testDump1.result b/tajo-core/src/test/resources/results/TestTajoDump/testDump1.result index 8accece..613093b 100644 --- a/tajo-core/src/test/resources/results/TestTajoDump/testDump1.result +++ b/tajo-core/src/test/resources/results/TestTajoDump/testDump1.result @@ -12,5 +12,4 @@ CREATE DATABASE IF NOT EXISTS "TestTajoDump"; -- -- Name: "TestTajoDump"."TableName1"; Type: TABLE; Storage: TEXT -- -CREATE TABLE "TestTajoDump"."TableName1" ("Age" INT4, "FirstName" TEXT, lastname TEXT) USING TEXT WITH ('text.delimiter'='|'); - +CREATE TABLE "TestTajoDump"."TableName1" ("Age" INT4, "FirstName" TEXT, lastname TEXT) USING TEXT WITH ('text.delimiter'='|'); \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/5a8fed40/tajo-core/src/test/resources/results/TestTajoDump/testDump2.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestTajoDump/testDump2.result b/tajo-core/src/test/resources/results/TestTajoDump/testDump2.result index 787562e..4c96233 100644 --- a/tajo-core/src/test/resources/results/TestTajoDump/testDump2.result +++ b/tajo-core/src/test/resources/results/TestTajoDump/testDump2.result @@ -12,5 +12,4 @@ CREATE DATABASE IF NOT EXISTS "TestTajoDump"; -- -- Name: "TestTajoDump"."TableName2"; Type: TABLE; Storage: TEXT -- -CREATE TABLE "TestTajoDump"."TableName2" ("Age" INT4, "Name" RECORD ("FirstName" TEXT, lastname TEXT)) USING TEXT WITH ('text.delimiter'='|'); - +CREATE TABLE "TestTajoDump"."TableName2" ("Age" INT4, "Name" RECORD ("FirstName" TEXT, lastname TEXT)) USING TEXT WITH ('text.delimiter'='|'); \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/5a8fed40/tajo-core/src/test/resources/results/TestTajoDump/testDump3.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestTajoDump/testDump3.result b/tajo-core/src/test/resources/results/TestTajoDump/testDump3.result new file mode 100644 index 0000000..72a42c7 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestTajoDump/testDump3.result @@ -0,0 +1,18 @@ +-- +-- Tajo database dump +-- + + +-- +-- Database name: "TestTajoDump" +-- + +CREATE DATABASE IF NOT EXISTS "TestTajoDump"; + +-- +-- Name: "TestTajoDump"."TableName1"; Type: TABLE; Storage: TEXT +-- +CREATE TABLE "TestTajoDump"."TableName1" ("Age" INT4, "FirstName" TEXT, lastname TEXT) USING TEXT WITH ('text.delimiter'='|');-- +-- Name: test_idx; Type: INDEX; Index Method: TWO_LEVEL_BIN_TREE +-- +CREATE INDEX test_idx on "TableName1" ( TestTajoDump.TableName1.Age asc null first, TestTajoDump.TableName1.FirstName desc null last ) location '${index.path}'; \ No newline at end of file
