http://git-wip-us.apache.org/repos/asf/hive/blob/47cac2d0/standalone-metastore/pom.xml ---------------------------------------------------------------------- diff --git a/standalone-metastore/pom.xml b/standalone-metastore/pom.xml index 58ed741..df769f5 100644 --- a/standalone-metastore/pom.xml +++ b/standalone-metastore/pom.xml @@ -38,6 +38,8 @@ <maven.compiler.target>1.8</maven.compiler.target> <maven.compiler.useIncrementalCompilation>false</maven.compiler.useIncrementalCompilation> <maven.repo.local>${settings.localRepository}</maven.repo.local> + <maven.assembly.plugin.version>2.3</maven.assembly.plugin.version> + <maven.exec.plugin.version>1.6.0</maven.exec.plugin.version> <!-- Test Properties --> <log4j.conf.dir>${project.basedir}/src/test/resources</log4j.conf.dir> @@ -45,6 +47,8 @@ <test.warehouse.dir>${project.build.directory}/warehouse</test.warehouse.dir> <test.warehouse.scheme>file://</test.warehouse.scheme> <test.forkcount>1</test.forkcount> + <skipITests>true</skipITests> + <itest.jdbc.jars>set-this-to-colon-separated-full-path-list-of-jars-to-run-integration-tests</itest.jdbc.jars> <!-- Plugin versions --> <ant.contrib.version>1.0b3</ant.contrib.version> @@ -75,6 +79,7 @@ <log4j2.version>2.8.2</log4j2.version> <mockito-all.version>1.10.19</mockito-all.version> <protobuf.version>2.5.0</protobuf.version> + <sqlline.version>1.3.0</sqlline.version> <storage-api.version>3.0.0-SNAPSHOT</storage-api.version> <!-- Thrift properties --> @@ -277,10 +282,21 @@ <version>1.4.0</version> <scope>test</scope> </dependency> + <dependency> + <groupId>sqlline</groupId> + <artifactId>sqlline</artifactId> + <version>${sqlline.version}</version> + </dependency> <!-- test scope dependencies --> <dependency> + <groupId>com.microsoft.sqlserver</groupId> + <artifactId>mssql-jdbc</artifactId> + <version>6.2.1.jre8</version> + <scope>test</scope> + </dependency> + <dependency> <groupId>junit</groupId> <artifactId>junit</artifactId> <version>${junit.version}</version> @@ -292,6 +308,20 @@ <version>${mockito-all.version}</version> <scope>test</scope> </dependency> + <dependency> + <!-- Note, this is LGPL. But we're only using it in a test and not changing it, so I + believe we are fine. --> + <groupId>org.mariadb.jdbc</groupId> + <artifactId>mariadb-java-client</artifactId> + <version>2.2.0</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.postgresql</groupId> + <artifactId>postgresql</artifactId> + <version>9.3-1102-jdbc41</version> + <scope>test</scope> + </dependency> </dependencies> <profiles> @@ -427,6 +457,11 @@ <artifactId>maven-checkstyle-plugin</artifactId> <version>${maven.checkstyle.plugin.version}</version> </plugin> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>exec-maven-plugin</artifactId> + <version>${maven.exec.plugin.version}</version> + </plugin> </plugins> </pluginManagement> <plugins> @@ -467,6 +502,21 @@ <goal>run</goal> </goals> </execution> + <execution> + <id>setup-metastore-scripts</id> + <phase>process-test-resources</phase> + <goals> + <goal>run</goal> + </goals> + <configuration> + <target> + <mkdir dir="${test.tmp.dir}/scripts/metastore/upgrade" /> + <copy todir="${test.tmp.dir}/scripts/metastore/upgrade"> + <fileset dir="${basedir}/src/main/sql/"/> + </copy> + </target> + </configuration> + </execution> </executions> </plugin> <plugin> @@ -498,11 +548,62 @@ </execution> </executions> </plugin> - <!-- TODO MS-SPLIT assembly plugin --> <!-- TODO MS-SPLIT findbugs plugin --> <!-- TODO MS-SPLIT javadoc plugin --> <plugin> <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-assembly-plugin</artifactId> + <version>${maven.assembly.plugin.version}</version> + <executions> + <execution> + <id>assemble</id> + <phase>package</phase> + <goals> + <goal>single</goal> + </goals> + <configuration> + <finalName>apache-hive-metastore-${project.version}</finalName> + <descriptors> + <descriptor>src/assembly/bin.xml</descriptor> + <descriptor>src/assembly/src.xml</descriptor> + </descriptors> + <tarLongFileMode>gnu</tarLongFileMode> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-failsafe-plugin</artifactId> + <version>2.20.1</version> + <executions> + <execution> + <goals> + <goal>integration-test</goal> + <goal>verify</goal> + </goals> + </execution> + </executions> + <configuration> + <redirectTestOutputToFile>true</redirectTestOutputToFile> + <reuseForks>false</reuseForks> + <argLine>-Xmx2048m</argLine> + <failIfNoTests>false</failIfNoTests> + <systemPropertyVariables> + <log4j.debug>true</log4j.debug> + <java.io.tmpdir>${test.tmp.dir}</java.io.tmpdir> + <test.tmp.dir>${test.tmp.dir}</test.tmp.dir> + <hive.in.test>true</hive.in.test> + </systemPropertyVariables> + <additionalClasspathElements> + <additionalClasspathElement>${log4j.conf.dir}</additionalClasspathElement> + <additionalClasspathElement>${itest.jdbc.jars}</additionalClasspathElement> + </additionalClasspathElements> + <skipITs>${skipITests}</skipITs> <!-- set this to false to run these tests --> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-surefire-plugin</artifactId> <version>2.16</version> <configuration> @@ -583,6 +684,27 @@ </executions> </plugin> <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>exec-maven-plugin</artifactId> + <executions> + <execution> + <phase>prepare-package</phase> + <goals> + <goal>exec</goal> + </goals> + </execution> + </executions> + <configuration> + <executable>java</executable> + <arguments> + <argument>-classpath</argument> + <classpath/> + <argument>org.apache.hadoop.hive.metastore.conf.ConfTemplatePrinter</argument> + <argument>${project.build.directory}/generated-sources/conf/metastore-site.xml.template</argument> + </arguments> + </configuration> + </plugin> + <plugin> <groupId>org.datanucleus</groupId> <artifactId>datanucleus-maven-plugin</artifactId> <version>4.0.5</version>
http://git-wip-us.apache.org/repos/asf/hive/blob/47cac2d0/standalone-metastore/src/assembly/bin.xml ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/assembly/bin.xml b/standalone-metastore/src/assembly/bin.xml new file mode 100644 index 0000000..81912d7 --- /dev/null +++ b/standalone-metastore/src/assembly/bin.xml @@ -0,0 +1,136 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +--> + +<assembly + xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd"> + + <id>bin</id> + + <formats> + <format>dir</format> + <format>tar.gz</format> + </formats> + + <baseDirectory>apache-hive-metastore-${project.version}-bin</baseDirectory> + + <dependencySets> + <dependencySet> + <outputDirectory>lib</outputDirectory> + <unpack>false</unpack> + <useProjectArtifact>true</useProjectArtifact> + <useStrictFiltering>true</useStrictFiltering> + <useTransitiveFiltering>true</useTransitiveFiltering> + <excludes> + <exclude>org.apache.hadoop:*</exclude> + <exclude>org.slf4j:*</exclude> + <exclude>log4j:*</exclude> + </excludes> + </dependencySet> + </dependencySets> + + <fileSets> + <fileSet> + <directory>${project.basedir}</directory> + <excludes> + <exclude>target/**</exclude> + <exclude>.classpath</exclude> + <exclude>.project</exclude> + <exclude>.settings/**</exclude> + <exclude>lib/**</exclude> + </excludes> + + <includes> + <include>README.txt</include> + <include>LICENSE</include> + <include>NOTICE</include> + </includes> + <outputDirectory>/</outputDirectory> + </fileSet> + + <fileSet> + <directory>${project.basedir}/binary-package-licenses</directory> + <includes> + <include>/*</include> + </includes> + <excludes> + <exclude>/README</exclude> + </excludes> + <outputDirectory>binary-package-licenses</outputDirectory> + </fileSet> + + <fileSet> + <fileMode>755</fileMode> + <directory>${project.basedir}/src/main/scripts</directory> + <includes> + <include>base</include> + <include>schematool</include> + <include>start-metastore</include> + <include>metastore-config.sh</include> + <include>ext/**/*</include> + </includes> + <outputDirectory>bin</outputDirectory> + </fileSet> + + <fileSet> + <directory>${project.basedir}/src/main/sql</directory> + <includes> + <include>**/*</include> + </includes> + <outputDirectory>scripts/metastore/upgrade</outputDirectory> + </fileSet> + + <fileSet> + <directory>${project.basedir}/src/gen/thrift/gen-php</directory> + <includes> + <include>**/*</include> + </includes> + <outputDirectory>lib/php/packages/hive_metastore</outputDirectory> + </fileSet> + + <fileSet> + <directory>${project.basedir}/src/gen/thrift/gen-py/hive_metastore</directory> + <fileMode>755</fileMode> + <includes> + <include>**/*</include> + </includes> + <outputDirectory>lib/py/hive_metastore</outputDirectory> + </fileSet> + + <fileSet> + <directory>${project.basedir}/src/main/resources/</directory> + <fileMode>644</fileMode> + <includes> + <include>metastore-site.xml</include> + <include>metastore-log4j2.properties</include> + </includes> + <outputDirectory>conf</outputDirectory> + </fileSet> + </fileSets> + + <files> + <file> + <source>${project.build.directory}/generated-sources/conf/metastore-site.xml.template</source> + <outputDirectory>conf</outputDirectory> + </file> + </files> + +</assembly> + http://git-wip-us.apache.org/repos/asf/hive/blob/47cac2d0/standalone-metastore/src/assembly/src.xml ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/assembly/src.xml b/standalone-metastore/src/assembly/src.xml new file mode 100644 index 0000000..a240544 --- /dev/null +++ b/standalone-metastore/src/assembly/src.xml @@ -0,0 +1,53 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +--> + +<assembly + xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd"> + + <id>src</id> + + <formats> + <format>tar.gz</format> + </formats> + + <baseDirectory>apache-hive-metastore-${project.version}-src</baseDirectory> + + <fileSets> + <fileSet> + <directory>${project.basedir}</directory> + + <excludes> + <exclude>target/**</exclude> + </excludes> + + <includes> + <include>.checkstyle</include> + <include>.gitattributes</include> + <include>.gitignore</include> + <include>LICENSE</include> + <include>NOTICE</include> + <include>pom.xml</include> + <include>src/**/*</include> + </includes> + <outputDirectory>/</outputDirectory> + </fileSet> + </fileSets> +</assembly> http://git-wip-us.apache.org/repos/asf/hive/blob/47cac2d0/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreSchemaInfo.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreSchemaInfo.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreSchemaInfo.java index be89f9b..ed4a2ef 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreSchemaInfo.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreSchemaInfo.java @@ -55,6 +55,13 @@ public interface IMetaStoreSchemaInfo { String generateInitFileName(String toVersion) throws HiveMetaException; /** + * Get SQL script that will create the user and database for Metastore to use. + * @return filename + * @throws HiveMetaException if something goes wrong. + */ + String getCreateUserScript() throws HiveMetaException; + + /** * Find the directory of metastore scripts * * @return the path of directory where the sql scripts are http://git-wip-us.apache.org/repos/asf/hive/blob/47cac2d0/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfo.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfo.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfo.java index 9b014da..88bd42d 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfo.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreSchemaInfo.java @@ -39,12 +39,14 @@ import org.apache.hadoop.hive.metastore.utils.MetastoreVersionInfo; public class MetaStoreSchemaInfo implements IMetaStoreSchemaInfo { protected static final String UPGRADE_FILE_PREFIX = "upgrade-"; - private static final String INIT_FILE_PREFIX = "hive-schema-"; - private static final String VERSION_UPGRADE_LIST = "upgrade.order"; - private static final String PRE_UPGRADE_PREFIX = "pre-"; - protected final String dbType; + protected static final String INIT_FILE_PREFIX = "hive-schema-"; + protected static final String VERSION_UPGRADE_LIST = "upgrade.order"; + protected static final String PRE_UPGRADE_PREFIX = "pre-"; + protected static final String CREATE_USER_PREFIX = "create-user"; + private String[] hiveSchemaVersions; - private final String hiveHome; + private final String metastoreHome; + protected final String dbType; // Some version upgrades often don't change schema. So they are equivalent to // a version @@ -57,8 +59,8 @@ public class MetaStoreSchemaInfo implements IMetaStoreSchemaInfo { "1.2.1", "1.2.0" ); - public MetaStoreSchemaInfo(String hiveHome, String dbType) throws HiveMetaException { - this.hiveHome = hiveHome; + public MetaStoreSchemaInfo(String metastoreHome, String dbType) throws HiveMetaException { + this.metastoreHome = metastoreHome; this.dbType = dbType; } @@ -137,13 +139,24 @@ public class MetaStoreSchemaInfo implements IMetaStoreSchemaInfo { return initScriptName; } + @Override + public String getCreateUserScript() throws HiveMetaException { + String createScript = CREATE_USER_PREFIX + "." + dbType + SQL_FILE_EXTENSION; + // check if the file exists + if (!(new File(getMetaStoreScriptDir() + File.separatorChar + + createScript).exists())) { + throw new HiveMetaException("Unable to find create user file, expected: " + createScript); + } + return createScript; + } + /** * Find the directory of metastore scripts * @return */ @Override public String getMetaStoreScriptDir() { - return hiveHome + File.separatorChar + + return metastoreHome + File.separatorChar + "scripts" + File.separatorChar + "metastore" + File.separatorChar + "upgrade" + File.separatorChar + dbType; } @@ -207,8 +220,8 @@ public class MetaStoreSchemaInfo implements IMetaStoreSchemaInfo { @Override public String getMetaStoreSchemaVersion(MetaStoreConnectionInfo connectionInfo) throws HiveMetaException { String versionQuery; - boolean needsQuotedIdentifier = - HiveSchemaHelper.getDbCommandParser(connectionInfo.getDbType(), connectionInfo.getMetaDbType()).needsQuotedIdentifier(); + boolean needsQuotedIdentifier = HiveSchemaHelper.getDbCommandParser(connectionInfo.getDbType(), + connectionInfo.getMetaDbType(), false).needsQuotedIdentifier(); if (needsQuotedIdentifier) { versionQuery = "select t.\"SCHEMA_VERSION\" from \"VERSION\" t"; } else { http://git-wip-us.apache.org/repos/asf/hive/blob/47cac2d0/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/ConfTemplatePrinter.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/ConfTemplatePrinter.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/ConfTemplatePrinter.java new file mode 100644 index 0000000..f57e2ce --- /dev/null +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/ConfTemplatePrinter.java @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.metastore.conf; + +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.Text; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.transform.OutputKeys; +import javax.xml.transform.Transformer; +import javax.xml.transform.TransformerException; +import javax.xml.transform.TransformerFactory; +import javax.xml.transform.dom.DOMSource; +import javax.xml.transform.stream.StreamResult; +import java.io.File; + +public class ConfTemplatePrinter { + + private Document generateTemplate() throws ParserConfigurationException { + DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); + DocumentBuilder docBuilder = dbf.newDocumentBuilder(); + Document doc = docBuilder.newDocument(); + doc.appendChild(doc.createProcessingInstruction( + "xml-stylesheet", "type=\"text/xsl\" href=\"configuration.xsl\"")); + + doc.appendChild(doc.createComment("\n" + + " Licensed to the Apache Software Foundation (ASF) under one or more\n" + + " contributor license agreements. See the NOTICE file distributed with\n" + + " this work for additional information regarding copyright ownership.\n" + + " The ASF licenses this file to You under the Apache License, Version 2.0\n" + + " (the \"License\"); you may not use this file except in compliance with\n" + + " the License. You may obtain a copy of the License at\n" + + "\n" + + " http://www.apache.org/licenses/LICENSE-2.0\n" + + "\n" + + " Unless required by applicable law or agreed to in writing, software\n" + + " distributed under the License is distributed on an \"AS IS\" BASIS,\n" + + " WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n" + + " See the License for the specific language governing permissions and\n" + + " limitations under the License.\n")); + + Element root = doc.createElement("configuration"); + doc.appendChild(root); + + root.appendChild(doc.createComment( + " WARNING!!! This file is auto generated for documentation purposes ONLY! ")); + root.appendChild(doc.createComment( + " WARNING!!! Any changes you make to this file will be ignored by the metastore. ")); + root.appendChild(doc.createComment( + " WARNING!!! You must make your changes in metastore-site.xml instead. ")); + + root.appendChild(doc.createComment(" Metastore Execution Parameters ")); + + root.appendChild(doc.createComment("================================")); + root.appendChild(doc.createComment("All time unit values have a time unit abbreviation suffix")); + root.appendChild(doc.createComment("Any time value can take any of the units")); + root.appendChild(doc.createComment("d = day")); + root.appendChild(doc.createComment("h = hour")); + root.appendChild(doc.createComment("m = minute")); + root.appendChild(doc.createComment("s = second")); + root.appendChild(doc.createComment("ms = millisecond")); + root.appendChild(doc.createComment("us = microsecond")); + root.appendChild(doc.createComment("ns = nanosecond")); + root.appendChild(doc.createComment("================================")); + + for (MetastoreConf.ConfVars confVars : MetastoreConf.ConfVars.values()) { + Element property = appendElement(root, "property", null); + appendElement(property, "name", confVars.getVarname()); + appendElement(property, "value", confVars.getDefaultVal().toString()); + appendElement(property, "description", normalize(confVars.getDescription())); + // wish to add new line here. + } + return doc; + + } + + private String normalize(String description) { + int index = description.indexOf('\n'); + if (index < 0) { + return description; + } + int prev = 0; + StringBuilder builder = new StringBuilder(description.length() << 1); + for (;index > 0; index = description.indexOf('\n', prev = index + 1)) { + builder.append("\n ").append(description.substring(prev, index)); + } + if (prev < description.length()) { + builder.append("\n ").append(description.substring(prev)); + } + builder.append("\n "); + return builder.toString(); + } + + private void writeToFile(File template, Document document) throws TransformerException { + Transformer transformer = TransformerFactory.newInstance().newTransformer(); + transformer.setOutputProperty(OutputKeys.INDENT, "yes"); + transformer.setOutputProperty("{http://xml.apache.org/xslt}indent-amount", "2"); + DOMSource source = new DOMSource(document); + StreamResult result = new StreamResult(template); + transformer.transform(source, result); + } + + private Element appendElement(Element parent, String name, String text) { + Document document = parent.getOwnerDocument(); + Element child = document.createElement(name); + parent.appendChild(child); + if (text != null) { + Text textNode = document.createTextNode(text); + child.appendChild(textNode); + } + return child; + } + + private void print(String fileName) throws ParserConfigurationException, TransformerException { + Document doc = generateTemplate(); + File file = new File(fileName); + File dir = file.getParentFile(); + // Make certain the target directory exists. + dir.mkdirs(); + writeToFile(file, doc); + } + + public static void main(String[] args) throws Exception { + if (args.length != 1) { + String msg = "Usage: ConfTemplatePrinter filename"; + System.err.println(msg); + throw new RuntimeException(msg); + } + ConfTemplatePrinter printer = new ConfTemplatePrinter(); + printer.print(args[0]); + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/47cac2d0/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java index 5b8e5ca..699a649 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java @@ -96,7 +96,16 @@ public class MetastoreConf { @Override public String toString() { - return Long.toString(unit.toNanos(val)) + "ns"; + switch (unit) { + case NANOSECONDS: return Long.toString(val) + "ns"; + case MICROSECONDS: return Long.toString(val) + "us"; + case MILLISECONDS: return Long.toString(val) + "ms"; + case SECONDS: return Long.toString(val) + "s"; + case MINUTES: return Long.toString(val) + "m"; + case HOURS: return Long.toString(val) + "h"; + case DAYS: return Long.toString(val) + "d"; + } + throw new RuntimeException("Unknown time unit " + unit); } } @@ -537,7 +546,7 @@ public class MetastoreConf { "The default value \"-1\" means no limit."), LOG4J_FILE("metastore.log4j.file", "hive.log4j.file", "", "Hive log4j configuration file.\n" + - "If the property is not set, then logging will be initialized using hive-log4j2.properties found on the classpath.\n" + + "If the property is not set, then logging will be initialized using metastore-log4j2.properties found on the classpath.\n" + "If the property is set, the value must be a valid URI (java.net.URI, e.g. \"file:///tmp/my-logging.xml\"), \n" + "which you can then extract a URL from and pass to PropertyConfigurator.configure(URL)."), MANAGER_FACTORY_CLASS("javax.jdo.PersistenceManagerFactoryClass", @@ -893,81 +902,91 @@ public class MetastoreConf { private final Object defaultVal; private final Validator validator; private final boolean caseSensitive; + private final String description; - ConfVars(String varname, String hiveName, String defaultVal, String comment) { + ConfVars(String varname, String hiveName, String defaultVal, String description) { this.varname = varname; this.hiveName = hiveName; this.defaultVal = defaultVal; validator = null; caseSensitive = false; + this.description = description; } ConfVars(String varname, String hiveName, String defaultVal, Validator validator, - String comment) { + String description) { this.varname = varname; this.hiveName = hiveName; this.defaultVal = defaultVal; this.validator = validator; caseSensitive = false; + this.description = description; } ConfVars(String varname, String hiveName, String defaultVal, boolean caseSensitive, - String comment) { + String description) { this.varname = varname; this.hiveName = hiveName; this.defaultVal = defaultVal; validator = null; this.caseSensitive = caseSensitive; + this.description = description; } - ConfVars(String varname, String hiveName, long defaultVal, String comment) { + ConfVars(String varname, String hiveName, long defaultVal, String description) { this.varname = varname; this.hiveName = hiveName; this.defaultVal = defaultVal; validator = null; caseSensitive = false; + this.description = description; } ConfVars(String varname, String hiveName, long defaultVal, Validator validator, - String comment) { + String description) { this.varname = varname; this.hiveName = hiveName; this.defaultVal = defaultVal; this.validator = validator; caseSensitive = false; + this.description = description; } - ConfVars(String varname, String hiveName, boolean defaultVal, String comment) { + ConfVars(String varname, String hiveName, boolean defaultVal, String description) { this.varname = varname; this.hiveName = hiveName; this.defaultVal = defaultVal; validator = null; caseSensitive = false; + this.description = description; } - ConfVars(String varname, String hiveName, double defaultVal, String comment) { + ConfVars(String varname, String hiveName, double defaultVal, String description) { this.varname = varname; this.hiveName = hiveName; this.defaultVal = defaultVal; validator = null; caseSensitive = false; + this.description = description; } - ConfVars(String varname, String hiveName, long defaultVal, TimeUnit unit, String comment) { + ConfVars(String varname, String hiveName, long defaultVal, TimeUnit unit, String description) { this.varname = varname; this.hiveName = hiveName; this.defaultVal = new TimeValue(defaultVal, unit); validator = new Validator.TimeValidator(unit); caseSensitive = false; + this.description = description; } ConfVars(String varname, String hiveName, long defaultVal, TimeUnit unit, - Validator validator, String comment) { + Validator validator, String description) { this.varname = varname; this.hiveName = hiveName; this.defaultVal = new TimeValue(defaultVal, unit); this.validator = validator; caseSensitive = false; + this.description = description; } public void validate(String value) throws IllegalArgumentException { @@ -1008,6 +1027,10 @@ public class MetastoreConf { return defaultVal; } + public String getDescription() { + return description; + } + @Override public String toString() { return varname; http://git-wip-us.apache.org/repos/asf/hive/blob/47cac2d0/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/HiveSchemaHelper.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/HiveSchemaHelper.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/HiveSchemaHelper.java index 70007cd..37891b6 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/HiveSchemaHelper.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/HiveSchemaHelper.java @@ -22,6 +22,8 @@ import com.google.common.collect.Lists; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.metastore.HiveMetaException; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.BufferedReader; import java.io.File; @@ -34,6 +36,8 @@ import java.util.IllegalFormatException; import java.util.List; public class HiveSchemaHelper { + private static final Logger LOG = LoggerFactory.getLogger(HiveSchemaHelper.class); + public static final String DB_DERBY = "derby"; public static final String DB_HIVE = "hive"; public static final String DB_MSSQL = "mssql"; @@ -43,10 +47,10 @@ public class HiveSchemaHelper { /*** * Get JDBC connection to metastore db - * - * @param dbType the type of the meteastore database * @param userName metastore connection username * @param password metastore connection password + * @param url Metastore URL. If null will be read from config file. + * @param driver Driver class. If null will be read from config file. * @param printInfo print connection parameters * @param conf hive config object * @param schema the schema to create the connection for @@ -59,9 +63,12 @@ public class HiveSchemaHelper { url = url == null ? getValidConfVar(MetastoreConf.ConfVars.CONNECTURLKEY, conf) : url; driver = driver == null ? getValidConfVar(MetastoreConf.ConfVars.CONNECTION_DRIVER, conf) : driver; if (printInfo) { - System.out.println("Metastore connection URL:\t " + url); - System.out.println("Metastore Connection Driver :\t " + driver); - System.out.println("Metastore connection User:\t " + userName); + logAndPrintToStdout("Metastore connection URL:\t " + url); + logAndPrintToStdout("Metastore Connection Driver :\t " + driver); + logAndPrintToStdout("Metastore connection User:\t " + userName); + if (MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.HIVE_IN_TEST)) { + logAndPrintToStdout("Metastore connection Password:\t " + password); + } } if ((userName == null) || userName.isEmpty()) { throw new HiveMetaException("UserName empty "); @@ -79,6 +86,7 @@ public class HiveSchemaHelper { } catch (IOException | SQLException e) { throw new HiveMetaException("Failed to get schema version.", e); } catch (ClassNotFoundException e) { + LOG.error("Unable to find driver class", e); throw new HiveMetaException("Failed to load driver", e); } } @@ -98,6 +106,11 @@ public class HiveSchemaHelper { return confVarStr.trim(); } + private static void logAndPrintToStdout(String msg) { + LOG.info(msg); + System.out.println(msg); + } + public interface NestedScriptParser { enum CommandType { @@ -202,13 +215,17 @@ public class HiveSchemaHelper { private String msUsername; private String msPassword; private Configuration conf; + // Depending on whether we are using beeline or sqlline the line endings have to be handled + // differently. + private final boolean usingSqlLine; public AbstractCommandParser(String dbOpts, String msUsername, String msPassword, - Configuration conf) { + Configuration conf, boolean usingSqlLine) { setDbOpts(dbOpts); this.msUsername = msUsername; this.msPassword = msPassword; this.conf = conf; + this.usingSqlLine = usingSqlLine; } @Override @@ -301,6 +318,7 @@ public class HiveSchemaHelper { // Now we have a complete statement, process it // write the line to buffer sb.append(currentCommand); + if (usingSqlLine) sb.append(";"); sb.append(System.getProperty("line.separator")); } } @@ -340,8 +358,8 @@ public class HiveSchemaHelper { private static final String DERBY_NESTING_TOKEN = "RUN"; public DerbyCommandParser(String dbOpts, String msUsername, String msPassword, - Configuration conf) { - super(dbOpts, msUsername, msPassword, conf); + Configuration conf, boolean usingSqlLine) { + super(dbOpts, msUsername, msPassword, conf, usingSqlLine); } @Override @@ -370,9 +388,9 @@ public class HiveSchemaHelper { private final NestedScriptParser nestedDbCommandParser; public HiveCommandParser(String dbOpts, String msUsername, String msPassword, - Configuration conf, String metaDbType) { - super(dbOpts, msUsername, msPassword, conf); - nestedDbCommandParser = getDbCommandParser(metaDbType); + Configuration conf, String metaDbType, boolean usingSqlLine) { + super(dbOpts, msUsername, msPassword, conf, usingSqlLine); + nestedDbCommandParser = getDbCommandParser(metaDbType, usingSqlLine); } @Override @@ -406,8 +424,8 @@ public class HiveSchemaHelper { private String delimiter = DEFAULT_DELIMITER; public MySqlCommandParser(String dbOpts, String msUsername, String msPassword, - Configuration conf) { - super(dbOpts, msUsername, msPassword, conf); + Configuration conf, boolean usingSqlLine) { + super(dbOpts, msUsername, msPassword, conf, usingSqlLine); } @Override @@ -472,8 +490,8 @@ public class HiveSchemaHelper { public static final String POSTGRES_SKIP_STANDARD_STRINGS_DBOPT = "postgres.filter.81"; public PostgresCommandParser(String dbOpts, String msUsername, String msPassword, - Configuration conf) { - super(dbOpts, msUsername, msPassword, conf); + Configuration conf, boolean usingSqlLine) { + super(dbOpts, msUsername, msPassword, conf, usingSqlLine); } @Override @@ -515,8 +533,8 @@ public class HiveSchemaHelper { private static final String ORACLE_NESTING_TOKEN = "@"; public OracleCommandParser(String dbOpts, String msUsername, String msPassword, - Configuration conf) { - super(dbOpts, msUsername, msPassword, conf); + Configuration conf, boolean usingSqlLine) { + super(dbOpts, msUsername, msPassword, conf, usingSqlLine); } @Override @@ -539,8 +557,8 @@ public class HiveSchemaHelper { private static final String MSSQL_NESTING_TOKEN = ":r"; public MSSQLCommandParser(String dbOpts, String msUsername, String msPassword, - Configuration conf) { - super(dbOpts, msUsername, msPassword, conf); + Configuration conf, boolean usingSqlLine) { + super(dbOpts, msUsername, msPassword, conf, usingSqlLine); } @Override @@ -558,29 +576,29 @@ public class HiveSchemaHelper { } } - public static NestedScriptParser getDbCommandParser(String dbName) { - return getDbCommandParser(dbName, null); + public static NestedScriptParser getDbCommandParser(String dbName, boolean usingSqlLine) { + return getDbCommandParser(dbName, null, usingSqlLine); } - public static NestedScriptParser getDbCommandParser(String dbName, String metaDbName) { - return getDbCommandParser(dbName, null, null, null, null, metaDbName); + public static NestedScriptParser getDbCommandParser(String dbName, String metaDbName, boolean usingSqlLine) { + return getDbCommandParser(dbName, null, null, null, null, metaDbName, usingSqlLine); } public static NestedScriptParser getDbCommandParser(String dbName, String dbOpts, String msUsername, String msPassword, - Configuration conf, String metaDbType) { + Configuration conf, String metaDbType, boolean usingSqlLine) { if (dbName.equalsIgnoreCase(DB_DERBY)) { - return new DerbyCommandParser(dbOpts, msUsername, msPassword, conf); + return new DerbyCommandParser(dbOpts, msUsername, msPassword, conf, usingSqlLine); } else if (dbName.equalsIgnoreCase(DB_HIVE)) { - return new HiveCommandParser(dbOpts, msUsername, msPassword, conf, metaDbType); + return new HiveCommandParser(dbOpts, msUsername, msPassword, conf, metaDbType, usingSqlLine); } else if (dbName.equalsIgnoreCase(DB_MSSQL)) { - return new MSSQLCommandParser(dbOpts, msUsername, msPassword, conf); + return new MSSQLCommandParser(dbOpts, msUsername, msPassword, conf, usingSqlLine); } else if (dbName.equalsIgnoreCase(DB_MYSQL)) { - return new MySqlCommandParser(dbOpts, msUsername, msPassword, conf); + return new MySqlCommandParser(dbOpts, msUsername, msPassword, conf, usingSqlLine); } else if (dbName.equalsIgnoreCase(DB_POSTGRACE)) { - return new PostgresCommandParser(dbOpts, msUsername, msPassword, conf); + return new PostgresCommandParser(dbOpts, msUsername, msPassword, conf, usingSqlLine); } else if (dbName.equalsIgnoreCase(DB_ORACLE)) { - return new OracleCommandParser(dbOpts, msUsername, msPassword, conf); + return new OracleCommandParser(dbOpts, msUsername, msPassword, conf, usingSqlLine); } else { throw new IllegalArgumentException("Unknown dbType " + dbName); }