This is an automated email from the ASF dual-hosted git repository. anishek pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push: new 48c0110 HIVE-23309: Lazy Initialization of Hadoop Shims (Aasha Medhi, reviewed by Anishek Agarwal) 48c0110 is described below commit 48c01107cd18d80867369e5addfa1fc5b4e7f698 Author: Aasha Medhi <aasha.medhi2...@gmail.com> AuthorDate: Wed May 6 13:58:29 2020 +0530 HIVE-23309: Lazy Initialization of Hadoop Shims (Aasha Medhi, reviewed by Anishek Agarwal) --- .../hcatalog/listener/DbNotificationListener.java | 9 +++--- .../TestMetaStoreMultipleEncryptionZones.java | 32 +++++++++++----------- .../ql/parse/repl/dump/io/FunctionSerializer.java | 3 +- .../hadoop/hive/metastore/ReplChangeManager.java | 7 +++-- 4 files changed, 27 insertions(+), 24 deletions(-) diff --git a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java index 3a8cb39..ac8922b 100644 --- a/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java +++ b/hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/DbNotificationListener.java @@ -302,9 +302,9 @@ public class DbNotificationListener extends TransactionalMetaStoreEventListener try { FileStatus file = files[i]; i++; - return ReplChangeManager.encodeFileUri(file.getPath().toString(), + return ReplChangeManager.getInstance(conf).encodeFileUri(file.getPath().toString(), ReplChangeManager.checksumFor(file.getPath(), fs), null); - } catch (IOException e) { + } catch (IOException | MetaException e) { throw new RuntimeException(e); } } @@ -521,9 +521,10 @@ public class DbNotificationListener extends TransactionalMetaStoreEventListener public String next() { String result; try { - result = ReplChangeManager.encodeFileUri(files.get(i), (chksums != null && !chksums.isEmpty()) ? chksums.get(i) : null, + result = ReplChangeManager.getInstance(conf). + encodeFileUri(files.get(i), (chksums != null && !chksums.isEmpty()) ? chksums.get(i) : null, subDirs != null ? subDirs.get(i) : null); - } catch (IOException e) { + } catch (IOException | MetaException e) { // File operations failed LOG.error("Encoding file URI failed with error " + e.getMessage()); throw new RuntimeException(e.getMessage()); diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMultipleEncryptionZones.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMultipleEncryptionZones.java index 41a1ce9..8f1bc40 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMultipleEncryptionZones.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMultipleEncryptionZones.java @@ -1302,29 +1302,29 @@ public class TestMetaStoreMultipleEncryptionZones { ReplChangeManager.getInstance(hiveConfCmClearer).recycle(dirTbl3, RecycleType.MOVE, true); assertTrue(fsWarehouse.exists(ReplChangeManager.getCMPath(hiveConfCmClearer, part11.getName(), fileChksum11, - ReplChangeManager.getCmRoot(part11).toString()))); + ReplChangeManager.getInstance(conf).getCmRoot(part11).toString()))); assertTrue(fsWarehouse.exists(ReplChangeManager.getCMPath(hiveConfCmClearer, part12.getName(), fileChksum12, - ReplChangeManager.getCmRoot(part12).toString()))); + ReplChangeManager.getInstance(conf).getCmRoot(part12).toString()))); assertTrue(fsWarehouse.exists(ReplChangeManager.getCMPath(hiveConfCmClearer, part21.getName(), fileChksum21, - ReplChangeManager.getCmRoot(part21).toString()))); + ReplChangeManager.getInstance(conf).getCmRoot(part21).toString()))); assertTrue(fsWarehouse.exists(ReplChangeManager.getCMPath(hiveConfCmClearer, part22.getName(), fileChksum22, - ReplChangeManager.getCmRoot(part22).toString()))); + ReplChangeManager.getInstance(conf).getCmRoot(part22).toString()))); assertTrue(fsWarehouse.exists(ReplChangeManager.getCMPath(hiveConfCmClearer, part31.getName(), fileChksum31, - ReplChangeManager.getCmRoot(part31).toString()))); + ReplChangeManager.getInstance(conf).getCmRoot(part31).toString()))); assertTrue(fsWarehouse.exists(ReplChangeManager.getCMPath(hiveConfCmClearer, part32.getName(), fileChksum32, - ReplChangeManager.getCmRoot(part32).toString()))); + ReplChangeManager.getInstance(conf).getCmRoot(part32).toString()))); fsWarehouse.setTimes(ReplChangeManager.getCMPath(hiveConfCmClearer, part11.getName(), fileChksum11, - ReplChangeManager.getCmRoot(part11).toString()), + ReplChangeManager.getInstance(conf).getCmRoot(part11).toString()), now - 86400*1000*2, now - 86400*1000*2); fsWarehouse.setTimes(ReplChangeManager.getCMPath(hiveConfCmClearer, part21.getName(), fileChksum21, - ReplChangeManager.getCmRoot(part21).toString()), + ReplChangeManager.getInstance(conf).getCmRoot(part21).toString()), now - 86400*1000*2, now - 86400*1000*2); fsWarehouse.setTimes(ReplChangeManager.getCMPath(hiveConfCmClearer, part31.getName(), fileChksum31, - ReplChangeManager.getCmRoot(part31).toString()), + ReplChangeManager.getInstance(conf).getCmRoot(part31).toString()), now - 86400*1000*2, now - 86400*1000*2); fsWarehouse.setTimes(ReplChangeManager.getCMPath(hiveConfCmClearer, part32.getName(), fileChksum32, - ReplChangeManager.getCmRoot(part32).toString()), + ReplChangeManager.getInstance(conf).getCmRoot(part32).toString()), now - 86400*1000*2, now - 86400*1000*2); ReplChangeManager.scheduleCMClearer(hiveConfCmClearer); @@ -1339,17 +1339,17 @@ public class TestMetaStoreMultipleEncryptionZones { Assert.fail("timeout, cmroot has not been cleared"); } if (!fsWarehouse.exists(ReplChangeManager.getCMPath(hiveConfCmClearer, part11.getName(), fileChksum11, - ReplChangeManager.getCmRoot(part11).toString())) && + ReplChangeManager.getInstance(conf).getCmRoot(part11).toString())) && fsWarehouse.exists(ReplChangeManager.getCMPath(hiveConfCmClearer, part12.getName(), fileChksum12, - ReplChangeManager.getCmRoot(part12).toString())) && + ReplChangeManager.getInstance(conf).getCmRoot(part12).toString())) && !fsWarehouse.exists(ReplChangeManager.getCMPath(hiveConfCmClearer, part21.getName(), fileChksum21, - ReplChangeManager.getCmRoot(part21).toString())) && + ReplChangeManager.getInstance(conf).getCmRoot(part21).toString())) && fsWarehouse.exists(ReplChangeManager.getCMPath(hiveConfCmClearer, part22.getName(), fileChksum22, - ReplChangeManager.getCmRoot(part22).toString())) && + ReplChangeManager.getInstance(conf).getCmRoot(part22).toString())) && !fsWarehouse.exists(ReplChangeManager.getCMPath(hiveConfCmClearer, part31.getName(), fileChksum31, - ReplChangeManager.getCmRoot(part31).toString())) && + ReplChangeManager.getInstance(conf).getCmRoot(part31).toString())) && !fsWarehouse.exists(ReplChangeManager.getCMPath(hiveConfCmClearer, part32.getName(), fileChksum32, - ReplChangeManager.getCmRoot(part32).toString()))) { + ReplChangeManager.getInstance(conf).getCmRoot(part32).toString()))) { cleared = true; } } while (!cleared); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/FunctionSerializer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/FunctionSerializer.java index 420c9bf..733bab5 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/FunctionSerializer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/FunctionSerializer.java @@ -58,7 +58,8 @@ public class FunctionSerializer implements JsonWriter.Serializer { FileSystem fileSystem = inputPath.getFileSystem(hiveConf); Path qualifiedUri = PathBuilder.fullyQualifiedHDFSUri(inputPath, fileSystem); String checkSum = ReplChangeManager.checksumFor(qualifiedUri, fileSystem); - String newFileUri = ReplChangeManager.encodeFileUri(qualifiedUri.toString(), checkSum, null); + String newFileUri = ReplChangeManager.getInstance(hiveConf) + .encodeFileUri(qualifiedUri.toString(), checkSum, null); resourceUris.add(new ResourceUri(uri.getResourceType(), newFileUri)); } else { resourceUris.add(uri); diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/ReplChangeManager.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/ReplChangeManager.java index 8e1bb4e..690b1f3 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/ReplChangeManager.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/ReplChangeManager.java @@ -60,7 +60,7 @@ public class ReplChangeManager { private static boolean inited = false; private static boolean enabled = false; private static Map<String, String> encryptionZoneToCmrootMapping = new HashMap<>(); - private static HadoopShims hadoopShims = ShimLoader.getHadoopShims(); + private HadoopShims hadoopShims; private static Configuration conf; private String msUser; private String msGroup; @@ -153,6 +153,7 @@ public class ReplChangeManager { if (MetastoreConf.getBoolVar(conf, ConfVars.REPLCMENABLED)) { ReplChangeManager.enabled = true; ReplChangeManager.conf = conf; + hadoopShims = ShimLoader.getHadoopShims(); cmRootDir = MetastoreConf.getVar(conf, ConfVars.REPLCMDIR); encryptedCmRootDir = MetastoreConf.getVar(conf, ConfVars.REPLCMENCRYPTEDDIR); fallbackNonEncryptedCmRootDir = MetastoreConf.getVar(conf, ConfVars.REPLCMFALLBACKNONENCRYPTEDDIR); @@ -403,7 +404,7 @@ public class ReplChangeManager { */ // TODO: this needs to be enhanced once change management based filesystem is implemented // Currently using fileuri#checksum#cmrooturi#subdirs as the format - public static String encodeFileUri(String fileUriStr, String fileChecksum, String encodedSubDir) + public String encodeFileUri(String fileUriStr, String fileChecksum, String encodedSubDir) throws IOException { if (instance == null) { throw new IllegalStateException("Uninitialized ReplChangeManager instance."); @@ -551,7 +552,7 @@ public class ReplChangeManager { } @VisibleForTesting - static Path getCmRoot(Path path) throws IOException { + Path getCmRoot(Path path) throws IOException { Path cmroot = null; //Default path if hive.repl.cm dir is encrypted String cmrootDir = fallbackNonEncryptedCmRootDir;