http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/30aecce5/pxf/pxf-service/tmp/generatedSources/org/apache/hawq/pxf/service/rest/WritableResource.java ---------------------------------------------------------------------- diff --git a/pxf/pxf-service/tmp/generatedSources/org/apache/hawq/pxf/service/rest/WritableResource.java b/pxf/pxf-service/tmp/generatedSources/org/apache/hawq/pxf/service/rest/WritableResource.java deleted file mode 100644 index a6c8d6b..0000000 --- a/pxf/pxf-service/tmp/generatedSources/org/apache/hawq/pxf/service/rest/WritableResource.java +++ /dev/null @@ -1,174 +0,0 @@ -package org.apache.hawq.pxf.service.rest; - -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - -import java.io.DataInputStream; -import java.io.InputStream; -import java.util.Map; - -import javax.servlet.ServletContext; -import javax.ws.rs.Consumes; -import javax.ws.rs.POST; -import javax.ws.rs.Path; -import javax.ws.rs.QueryParam; -import javax.ws.rs.core.Context; -import javax.ws.rs.core.HttpHeaders; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; - -import org.apache.catalina.connector.ClientAbortException; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hawq.pxf.api.utilities.Utilities; -import org.apache.hawq.pxf.service.Bridge; -import org.apache.hawq.pxf.service.WriteBridge; -import org.apache.hawq.pxf.service.utilities.ProtocolData; -import org.apache.hawq.pxf.service.utilities.SecuredHDFS; - -/* - * Running this resource manually: - * - * run: - curl -i -X post "http://localhost:51200/pxf/{version}/Writable/stream?path=/data/curl/curl`date \"+%h%d_%H%M%s\"`" \ - --header "X-GP-Accessor: TextFileWAccessor" \ - --header "X-GP-Resolver: TextWResolver" \ - --header "Content-Type:application/octet-stream" \ - --header "Expect: 100-continue" \ - --header "X-GP-ALIGNMENT: 4" \ - --header "X-GP-SEGMENT-ID: 0" \ - --header "X-GP-SEGMENT-COUNT: 3" \ - --header "X-GP-HAS-FILTER: 0" \ - --header "X-GP-FORMAT: TEXT" \ - --header "X-GP-URI: pxf://localhost:51200/data/curl/?Accessor=TextFileWAccessor&Resolver=TextWResolver" \ - --header "X-GP-URL-HOST: localhost" \ - --header "X-GP-URL-PORT: 51200" \ - --header "X-GP-ATTRS: 0" \ - --header "X-GP-DATA-DIR: data/curl/" \ - -d "data111" -d "data222" - - * result: - - HTTP/1.1 200 OK - Content-Type: text/plain;charset=UTF-8 - Content-Type: text/plain - Transfer-Encoding: chunked - Server: Jetty(7.6.10.v20130312) - - wrote 15 bytes to curlAug11_17271376231245 - - file content: - bin/hdfs dfs -cat /data/curl/*45 - data111&data222 - - */ - - -/** - * This class handles the subpath /<version>/Writable/ of this - * REST component - */ -@Path("/" + Version.PXF_PROTOCOL_VERSION + "/Writable/") -public class WritableResource extends RestResource{ - private static final Log LOG = LogFactory.getLog(WritableResource.class); - - public WritableResource() { - } - - /** - * This function is called when http://nn:port/pxf/{version}/Writable/stream?path=... - * is used. - * - * @param servletContext Servlet context contains attributes required by SecuredHDFS - * @param headers Holds HTTP headers from request - * @param path Holds URI path option used in this request - * @param inputStream stream of bytes to write from Hawq - * @return ok response if the operation finished successfully - * @throws Exception in case of wrong request parameters, failure to - * initialize bridge or to write data - */ - @POST - @Path("stream") - @Consumes(MediaType.APPLICATION_OCTET_STREAM) - public Response stream(@Context final ServletContext servletContext, - @Context HttpHeaders headers, - @QueryParam("path") String path, - InputStream inputStream) throws Exception { - - /* Convert headers into a case-insensitive regular map */ - Map<String, String> params = convertToCaseInsensitiveMap(headers.getRequestHeaders()); - if (LOG.isDebugEnabled()) { - LOG.debug("WritableResource started with parameters: " + params + " and write path: " + path); - } - - ProtocolData protData = new ProtocolData(params); - protData.setDataSource(path); - - SecuredHDFS.verifyToken(protData, servletContext); - Bridge bridge = new WriteBridge(protData); - - // THREAD-SAFE parameter has precedence - boolean isThreadSafe = protData.isThreadSafe() && bridge.isThreadSafe(); - LOG.debug("Request for " + path + " handled " + - (isThreadSafe ? "without" : "with") + " synchronization"); - - return isThreadSafe ? - writeResponse(bridge, path, inputStream) : - synchronizedWriteResponse(bridge, path, inputStream); - } - - private static synchronized Response synchronizedWriteResponse(Bridge bridge, - String path, - InputStream inputStream) - throws Exception { - return writeResponse(bridge, path, inputStream); - } - - private static Response writeResponse(Bridge bridge, - String path, - InputStream inputStream) throws Exception { - - String returnMsg; - - // Open the output file - bridge.beginIteration(); - - long totalWritten = 0; - - // dataStream will close automatically in the end of the try. - // inputStream is closed by dataStream.close(). - try (DataInputStream dataStream = new DataInputStream(inputStream)) { - while (bridge.setNext(dataStream)) { - ++totalWritten; - } - } catch (ClientAbortException e) { - LOG.debug("Remote connection closed by HAWQ", e); - } catch (Exception ex) { - LOG.debug("totalWritten so far " + totalWritten + " to " + path); - throw ex; - } - - String censuredPath = Utilities.maskNonPrintables(path); - returnMsg = "wrote " + totalWritten + " bulks to " + censuredPath; - LOG.debug(returnMsg); - - return Response.ok(returnMsg).build(); - } -}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/30aecce5/pxf/pxf-service/tmp/generatedSources/org/apache/hawq/pxf/service/utilities/AnalyzeUtils.java ---------------------------------------------------------------------- diff --git a/pxf/pxf-service/tmp/generatedSources/org/apache/hawq/pxf/service/utilities/AnalyzeUtils.java b/pxf/pxf-service/tmp/generatedSources/org/apache/hawq/pxf/service/utilities/AnalyzeUtils.java deleted file mode 100644 index 21172c5..0000000 --- a/pxf/pxf-service/tmp/generatedSources/org/apache/hawq/pxf/service/utilities/AnalyzeUtils.java +++ /dev/null @@ -1,147 +0,0 @@ -package org.apache.hawq.pxf.service.utilities; - -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import java.util.ArrayList; -import java.util.BitSet; -import java.util.List; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -import org.apache.hawq.pxf.api.Fragment; - -/** - * Helper class to get statistics for ANALYZE. - */ -public class AnalyzeUtils { - - private static final Log LOG = LogFactory.getLog(AnalyzeUtils.class); - - /** - * In case pxf_max_fragments parameter is declared, make sure not to get - * over the limit. The returned fragments are evenly distributed, in order - * to achieve good sampling. - * - * @param fragments fragments list - * @param protData container for parameters, including sampling data. - * @return a list of fragments no bigger than pxf_max_fragments parameter. - */ - static public List<Fragment> getSampleFragments(List<Fragment> fragments, - ProtocolData protData) { - - int listSize = fragments.size(); - int maxSize = protData.getStatsMaxFragments(); - List<Fragment> samplingList = new ArrayList<Fragment>(); - BitSet bitSet; - - if (maxSize == 0) { - return fragments; - } - - LOG.debug("fragments list has " + listSize - + " fragments, maxFragments = " + maxSize); - - bitSet = generateSamplingBitSet(listSize, maxSize); - - for (int i = 0; i < listSize; ++i) { - if (bitSet.get(i)) { - samplingList.add(fragments.get(i)); - } - } - - return samplingList; - } - - /** - * Marks sampleSize bits out of the poolSize, in a uniform way. - * - * @param poolSize pool size - * @param sampleSize sample size - * @return bit set with sampleSize bits set out of poolSize. - */ - static public BitSet generateSamplingBitSet(int poolSize, int sampleSize) { - - int skip = 0, chosen = 0, curIndex = 0; - BitSet bitSet = new BitSet(); - - if (poolSize <= 0 || sampleSize <= 0) { - return bitSet; - } - - if (sampleSize >= poolSize) { - LOG.debug("sampling bit map has " + poolSize + " elements (100%)"); - bitSet.set(0, poolSize); - return bitSet; - } - - skip = (poolSize / sampleSize) + 1; - - while (chosen < sampleSize) { - - bitSet.set(curIndex); - chosen++; - if (chosen == sampleSize) { - break; - } - - for (int i = 0; i < skip; ++i) { - curIndex = nextClearBitModulo((++curIndex) % poolSize, - poolSize, bitSet); - if (curIndex == -1) { - // should never happen - throw new IllegalArgumentException( - "Trying to sample more than pool size " - + "(pool size " + poolSize - + ", sampling size " + sampleSize); - } - } - } - - LOG.debug("sampling bit map has " + chosen + " elements:" - + bitSet.toString()); - - return bitSet; - } - - /** - * Returns index of next clear (false) bit, starting from and including - * index. If all bits from index to the end are set (true), search from the - * beginning. Return -1 if all bits are set (true). - * - * @param index starting point - * @param poolSize the bit set size - * @param bitSet bitset to search - * @return index of next clear bit, starting in index - */ - static private int nextClearBitModulo(int index, int poolSize, BitSet bitSet) { - - int indexToSet = bitSet.nextClearBit(index); - if (indexToSet == poolSize && index != 0) { - indexToSet = bitSet.nextClearBit(0); - } - /* means that all bits are already set, so we return -1 */ - if (indexToSet == poolSize) { - return -1; - } - - return indexToSet; - } -} http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/30aecce5/pxf/pxf-service/tmp/generatedSources/org/apache/hawq/pxf/service/utilities/CustomWebappLoader.java ---------------------------------------------------------------------- diff --git a/pxf/pxf-service/tmp/generatedSources/org/apache/hawq/pxf/service/utilities/CustomWebappLoader.java b/pxf/pxf-service/tmp/generatedSources/org/apache/hawq/pxf/service/utilities/CustomWebappLoader.java deleted file mode 100644 index 821f2d5..0000000 --- a/pxf/pxf-service/tmp/generatedSources/org/apache/hawq/pxf/service/utilities/CustomWebappLoader.java +++ /dev/null @@ -1,231 +0,0 @@ -package org.apache.hawq.pxf.service.utilities; - -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - -import java.io.File; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.net.URI; -import java.nio.file.DirectoryStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.ArrayList; - -import org.apache.catalina.LifecycleException; -import org.apache.catalina.loader.WebappLoader; -import org.apache.juli.logging.Log; -import org.apache.juli.logging.LogFactory; - -/** - * A WebappLoader that allows a customized classpath to be added through configuration in context xml. - * Any additional classpath entry will be added to the default webapp classpath. - * - * <pre> - * <Context> - * <Loader className="org.apache.hawq.pxf.service.utilities.CustomWebappLoader" - * classpathFiles="/somedir/classpathFile1;/somedir/classpathFile2"/> - * </Context> - * </pre> - */ -public class CustomWebappLoader extends WebappLoader { - - /** - * Because this class belongs in tcServer itself, logs go into tcServer's log facility that is separate - * from the web app's log facility. - * - * Logs are directed to catalina.log file. By default only INFO or higher messages are logged. - * To change log level, add the following line to {catalina.base}/conf/logging.properties - * <code>org.apache.hawq.pxf.level = FINE/INFO/WARNING</code> (FINE = debug). - */ - private static final Log LOG = LogFactory.getLog(CustomWebappLoader.class); - - /** - * Classpath files containing path entries, separated by new line. - * Globbing is supported for the file name. - * e.g: - * somedir - * anotherdir/somejar.jar - * anotherone/hadoop*.jar - * anotherone/pxf*[0-9].jar - * Unix wildcard convention can be used to match a number of files - * (e.g. <code>*</code>, <code>[0-9]</code>, <code>?</code>), but not a number of directories. - * - * The files specified under classpathFiles must exist - if they can't be read an exception will be thrown. - */ - private String classpathFiles; - /** - * Secondary classpath files - if these files are unavailable only a warning will be logged. - */ - private String secondaryClasspathFiles; - - /** - * Constructs a WebappLoader with no defined parent class loader (actual parent will be the system class loader). - */ - public CustomWebappLoader() { - super(); - } - - /** - * Constructs a WebappLoader with the specified class loader to be defined as the parent for this ClassLoader. - * - * @param parent The parent class loader - */ - public CustomWebappLoader(ClassLoader parent) { - super(parent); - } - - /** - * <code>classpathFiles</code> attribute is automatically set from the context xml file. - * - * @param classpathFiles Files separated by <code>;</code> Which contains <code>;</code> separated list of path entries. - */ - public void setClasspathFiles(String classpathFiles) { - this.classpathFiles = classpathFiles; - } - - /** - * <code>secondaryClasspathFiles</code> attribute is automatically set from the context xml file. - * - * @param secondaryClasspathFiles Files separated by <code>;</code> Which contains <code>;</code> separated list of path entries. - */ - public void setSecondaryClasspathFiles(String secondaryClasspathFiles) { - this.secondaryClasspathFiles = secondaryClasspathFiles; - } - - /** - * Implements {@link org.apache.catalina.util.LifecycleBase#startInternal()}. - * - * @throws LifecycleException if this component detects a fatal error that prevents this component from being used. - */ - @Override - protected void startInternal() throws LifecycleException { - - addRepositories(classpathFiles, true); - addRepositories(secondaryClasspathFiles, false); - - super.startInternal(); - } - - private void addRepositories(String classpathFiles, boolean throwException) throws LifecycleException { - - for (String classpathFile : classpathFiles.split(";")) { - - String classpath = readClasspathFile(classpathFile, throwException); - if (classpath == null) { - continue; - } - - ArrayList<String> classpathEntries = trimEntries(classpath.split("[\\r\\n]+")); - LOG.info("Classpath file " + classpathFile + " has " + classpathEntries.size() + " entries"); - - for (String entry : classpathEntries) { - LOG.debug("Trying to load entry " + entry); - int repositoriesCount = 0; - Path pathEntry = Paths.get(entry); - /* - * For each entry, we look at the parent directory and try to match each of the files - * inside it to the file name or pattern in the file name (the last part of the path). - * E.g., for path '/some/path/with/pattern*', the parent directory will be '/some/path/with/' - * and the file name will be 'pattern*'. Each file under that directory matching - * this pattern will be added to the class loader repository. - */ - try (DirectoryStream<Path> repositories = Files.newDirectoryStream(pathEntry.getParent(), - pathEntry.getFileName().toString())) { - for (Path repository : repositories) { - if (addPathToRepository(repository, entry)) { - repositoriesCount++; - } - } - } catch (IOException e) { - LOG.warn("Failed to load entry " + entry + ": " + e); - } - if (repositoriesCount == 0) { - LOG.warn("Entry " + entry + " doesn't match any files"); - } - LOG.debug("Loaded " + repositoriesCount + " repositories from entry " + entry); - } - } - } - - private String readClasspathFile(String classpathFile, boolean throwException) throws LifecycleException { - String classpath = null; - try { - LOG.info("Trying to read classpath file " + classpathFile); - classpath = new String(Files.readAllBytes(Paths.get(classpathFile))); - } catch (IOException ioe) { - LOG.warn("Failed to read classpath file: " + ioe); - if (throwException) { - throw new LifecycleException("Failed to read classpath file: " + ioe, ioe); - } - } - return classpath; - } - - /** - * Returns a list of valid classpath entries, excluding null, empty and comment lines. - * @param classpathEntries original entries - * @return valid entries - */ - private ArrayList<String> trimEntries(String[] classpathEntries) { - - ArrayList<String> trimmed = new ArrayList<String>(); - int line = 0; - for (String entry : classpathEntries) { - - line++; - if (entry == null) { - LOG.debug("Skipping entry #" + line + " (null)"); - continue; - } - - entry = entry.trim(); - if (entry.isEmpty() || entry.startsWith("#")) { - LOG.debug("Skipping entry #" + line + " (" + entry + ")"); - continue; - } - trimmed.add(entry); - } - return trimmed; - } - - private boolean addPathToRepository(Path path, String entry) { - - try { - URI pathUri = path.toUri(); - String pathUriStr = pathUri.toString(); - File file = new File(pathUri); - if (!file.canRead()) { - throw new FileNotFoundException(pathUriStr + " cannot be read"); - } - addRepository(pathUriStr); - LOG.debug("Repository " + pathUriStr + " added from entry " + entry); - return true; - } catch (Exception e) { - LOG.warn("Failed to load path " + path + " to repository: " + e); - } - - return false; - } - -} - - http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/30aecce5/pxf/pxf-service/tmp/generatedSources/org/apache/hawq/pxf/service/utilities/Log4jConfigure.java ---------------------------------------------------------------------- diff --git a/pxf/pxf-service/tmp/generatedSources/org/apache/hawq/pxf/service/utilities/Log4jConfigure.java b/pxf/pxf-service/tmp/generatedSources/org/apache/hawq/pxf/service/utilities/Log4jConfigure.java deleted file mode 100644 index c2ccd20..0000000 --- a/pxf/pxf-service/tmp/generatedSources/org/apache/hawq/pxf/service/utilities/Log4jConfigure.java +++ /dev/null @@ -1,66 +0,0 @@ -package org.apache.hawq.pxf.service.utilities; - -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import java.io.File; - -import javax.servlet.ServletContext; -import javax.servlet.ServletContextEvent; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.log4j.PropertyConfigurator; - -public class Log4jConfigure { - - private static final Log LOG = LogFactory.getLog(Log4jConfigure.class); - - /** - * Initializes log4j logging for the webapp. - * - * Reads log4j properties file location from log4jConfigLocation parameter - * in web.xml. When not using aboslute path, the path starts from the webapp - * root directory. If the file can't be read, reverts to default - * configuration file under WEB-INF/classes/pxf-log4j.properties. - * - * @param event Servlet context, used to determine webapp root directory. - */ - public static void configure(ServletContextEvent event) { - - final String defaultLog4jLocation = "WEB-INF/classes/pxf-log4j.properties"; - - ServletContext context = event.getServletContext(); - String log4jConfigLocation = context.getInitParameter("log4jConfigLocation"); - - if (!log4jConfigLocation.startsWith(File.separator)) { - log4jConfigLocation = context.getRealPath("") + File.separator - + log4jConfigLocation; - } - - // revert to default properties file if file doesn't exist - File log4jConfigFile = new File(log4jConfigLocation); - if (!log4jConfigFile.canRead()) { - log4jConfigLocation = context.getRealPath("") + File.separator - + defaultLog4jLocation; - } - PropertyConfigurator.configure(log4jConfigLocation); - LOG.info("log4jConfigLocation = " + log4jConfigLocation); - } -} http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/30aecce5/pxf/pxf-service/tmp/generatedSources/org/apache/hawq/pxf/service/utilities/ProtocolData.java ---------------------------------------------------------------------- diff --git a/pxf/pxf-service/tmp/generatedSources/org/apache/hawq/pxf/service/utilities/ProtocolData.java b/pxf/pxf-service/tmp/generatedSources/org/apache/hawq/pxf/service/utilities/ProtocolData.java deleted file mode 100644 index 2838232..0000000 --- a/pxf/pxf-service/tmp/generatedSources/org/apache/hawq/pxf/service/utilities/ProtocolData.java +++ /dev/null @@ -1,491 +0,0 @@ -package org.apache.hawq.pxf.service.utilities; - -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -import org.apache.commons.codec.binary.Base64; -import org.apache.commons.lang.StringUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hawq.pxf.api.OutputFormat; -import org.apache.hawq.pxf.api.utilities.ColumnDescriptor; -import org.apache.hawq.pxf.api.utilities.InputData; -import org.apache.hawq.pxf.api.utilities.ProfilesConf; - -/** - * Common configuration of all MetaData classes. Provides read-only access to - * common parameters supplied using system properties. - */ -public class ProtocolData extends InputData { - - private static final String TRUE_LCASE = "true"; - private static final String FALSE_LCASE = "false"; - private static final String PROP_PREFIX = "X-GP-"; - public static final int INVALID_SPLIT_IDX = -1; - - private static final Log LOG = LogFactory.getLog(ProtocolData.class); - - protected OutputFormat outputFormat; - protected int port; - protected String host; - protected String profile; - protected String token; - // statistics parameters - protected int statsMaxFragments; - protected float statsSampleRatio; - - /** - * Constructs a ProtocolData. Parses X-GP-* configuration variables. - * - * @param paramsMap contains all query-specific parameters from Hawq - */ - public ProtocolData(Map<String, String> paramsMap) { - - requestParametersMap = paramsMap; - segmentId = getIntProperty("SEGMENT-ID"); - totalSegments = getIntProperty("SEGMENT-COUNT"); - filterStringValid = getBoolProperty("HAS-FILTER"); - - if (filterStringValid) { - filterString = getProperty("FILTER"); - } - - parseFormat(getProperty("FORMAT")); - - host = getProperty("URL-HOST"); - port = getIntProperty("URL-PORT"); - - tupleDescription = new ArrayList<ColumnDescriptor>(); - recordkeyColumn = null; - parseTupleDescription(); - - /* - * accessor - will throw exception from getPropery() if outputFormat is - * BINARY and the user did not supply accessor=... or profile=... - * resolver - will throw exception from getPropery() if outputFormat is - * BINARY and the user did not supply resolver=... or profile=... - */ - profile = getOptionalProperty("PROFILE"); - if (profile != null) { - setProfilePlugins(); - } - accessor = getProperty("ACCESSOR"); - resolver = getProperty("RESOLVER"); - fragmenter = getOptionalProperty("FRAGMENTER"); - metadata = getOptionalProperty("METADATA"); - dataSource = getProperty("DATA-DIR"); - - /* Kerberos token information */ - if (UserGroupInformation.isSecurityEnabled()) { - token = getProperty("TOKEN"); - } - - parseFragmentMetadata(); - parseUserData(); - parseThreadSafe(); - parseRemoteCredentials(); - - dataFragment = INVALID_SPLIT_IDX; - parseDataFragment(getOptionalProperty("DATA-FRAGMENT")); - - statsMaxFragments = 0; - statsSampleRatio = 0; - parseStatsParameters(); - - // Store alignment for global use as a system property - System.setProperty("greenplum.alignment", getProperty("ALIGNMENT")); - } - - /** - * Constructs an InputDataBuilder from a copy. Used to create from an - * extending class. - * - * @param copy the input data to copy - */ - public ProtocolData(ProtocolData copy) { - this.requestParametersMap = copy.requestParametersMap; - this.segmentId = copy.segmentId; - this.totalSegments = copy.totalSegments; - this.outputFormat = copy.outputFormat; - this.host = copy.host; - this.port = copy.port; - this.fragmentMetadata = copy.fragmentMetadata; - this.userData = copy.userData; - this.tupleDescription = copy.tupleDescription; - this.recordkeyColumn = copy.recordkeyColumn; - this.filterStringValid = copy.filterStringValid; - this.filterString = copy.filterString; - this.dataSource = copy.dataSource; - this.accessor = copy.accessor; - this.resolver = copy.resolver; - this.fragmenter = copy.fragmenter; - this.threadSafe = copy.threadSafe; - this.remoteLogin = copy.remoteLogin; - this.remoteSecret = copy.remoteSecret; - this.token = copy.token; - this.statsMaxFragments = copy.statsMaxFragments; - this.statsSampleRatio = copy.statsSampleRatio; - } - - /** - * Constructs a ProtocolData. Parses X-GP-* configuration variables. - * - * @param paramsMap contains all query-specific parameters from Hawq - * @param profileString contains the profile name - */ - public ProtocolData(Map<String, String> paramsMap, String profileString) { - requestParametersMap = paramsMap; - profile = profileString; - setProfilePlugins(); - metadata = getProperty("METADATA"); - - /* Kerberos token information */ - if (UserGroupInformation.isSecurityEnabled()) { - token = getProperty("TOKEN"); - } - } - - /** - * Sets the requested profile plugins from profile file into - * {@link #requestParametersMap}. - */ - private void setProfilePlugins() { - Map<String, String> pluginsMap = ProfilesConf.getProfilePluginsMap(profile); - checkForDuplicates(pluginsMap, requestParametersMap); - requestParametersMap.putAll(pluginsMap); - } - - /** - * Verifies there are no duplicates between parameters declared in the table - * definition and parameters defined in a profile. - * - * The parameters' names are case insensitive. - */ - private void checkForDuplicates(Map<String, String> plugins, - Map<String, String> params) { - List<String> duplicates = new ArrayList<>(); - for (String key : plugins.keySet()) { - if (params.containsKey(key)) { - duplicates.add(key); - } - } - - if (!duplicates.isEmpty()) { - throw new IllegalArgumentException("Profile '" + profile - + "' already defines: " - + String.valueOf(duplicates).replace("X-GP-", "")); - } - } - - /** - * Returns the request parameters. - * - * @return map of request parameters - */ - public Map<String, String> getParametersMap() { - return requestParametersMap; - } - - /** - * Throws an exception when the given property value is missing in request. - * - * @param property missing property name - * @throws IllegalArgumentException throws an exception with the property - * name in the error message - */ - public void protocolViolation(String property) { - String error = "Internal server error. Property \"" + property - + "\" has no value in current request"; - - LOG.error(error); - throw new IllegalArgumentException(error); - } - - /** - * Returns the value to which the specified property is mapped in - * {@link #requestParametersMap}. - * - * @param property the lookup property key - * @throws IllegalArgumentException if property key is missing - */ - private String getProperty(String property) { - String result = requestParametersMap.get(PROP_PREFIX + property); - - if (result == null) { - protocolViolation(property); - } - - return result; - } - - /** - * Returns the optional property value. Unlike {@link #getProperty}, it will - * not fail if the property is not found. It will just return null instead. - * - * @param property the lookup optional property - * @return property value as a String - */ - private String getOptionalProperty(String property) { - return requestParametersMap.get(PROP_PREFIX + property); - } - - /** - * Returns a property value as an int type. - * - * @param property the lookup property - * @return property value as an int type - * @throws NumberFormatException if the value is missing or can't be - * represented by an Integer - */ - private int getIntProperty(String property) { - return Integer.parseInt(getProperty(property)); - } - - /** - * Returns a property value as boolean type. A boolean property is defined - * as an int where 0 means false, and anything else true (like C). - * - * @param property the lookup property - * @return property value as boolean - * @throws NumberFormatException if the value is missing or can't be - * represented by an Integer - */ - private boolean getBoolProperty(String property) { - return getIntProperty(property) != 0; - } - - /** - * Returns the current output format, either {@link OutputFormat#TEXT} or - * {@link OutputFormat#BINARY}. - * - * @return output format - */ - public OutputFormat outputFormat() { - return outputFormat; - } - - /** - * Returns the server name providing the service. - * - * @return server name - */ - public String serverName() { - return host; - } - - /** - * Returns the server port providing the service. - * - * @return server port - */ - public int serverPort() { - return port; - } - - /** - * Returns Kerberos token information. - * - * @return token - */ - public String getToken() { - return token; - } - - /** - * Statistics parameter. Returns the max number of fragments to return for - * ANALYZE sampling. The value is set in HAWQ side using the GUC - * pxf_stats_max_fragments. - * - * @return max number of fragments to be processed by analyze - */ - public int getStatsMaxFragments() { - return statsMaxFragments; - } - - /** - * Statistics parameter. Returns a number between 0.0001 and 1.0, - * representing the sampling ratio on each fragment for ANALYZE sampling. - * The value is set in HAWQ side based on ANALYZE computations and the - * number of sampled fragments. - * - * @return sampling ratio - */ - public float getStatsSampleRatio() { - return statsSampleRatio; - } - - /** - * Sets the thread safe parameter. Default value - true. - */ - private void parseThreadSafe() { - - threadSafe = true; - String threadSafeStr = getOptionalProperty("THREAD-SAFE"); - if (threadSafeStr != null) { - threadSafe = parseBooleanValue(threadSafeStr); - } - } - - private boolean parseBooleanValue(String threadSafeStr) { - - if (threadSafeStr.equalsIgnoreCase(TRUE_LCASE)) { - return true; - } - if (threadSafeStr.equalsIgnoreCase(FALSE_LCASE)) { - return false; - } - throw new IllegalArgumentException("Illegal boolean value '" - + threadSafeStr + "'." + " Usage: [TRUE|FALSE]"); - } - - /** - * Sets the format type based on the input string. Allowed values are: - * "TEXT", "GPDBWritable". - * - * @param formatString format string - */ - protected void parseFormat(String formatString) { - switch (formatString) { - case "TEXT": - outputFormat = OutputFormat.TEXT; - break; - case "GPDBWritable": - outputFormat = OutputFormat.BINARY; - break; - default: - throw new IllegalArgumentException( - "Wrong value for greenplum.format " + formatString); - } - } - - /* - * Sets the tuple description for the record - */ - void parseTupleDescription() { - int columns = getIntProperty("ATTRS"); - for (int i = 0; i < columns; ++i) { - String columnName = getProperty("ATTR-NAME" + i); - int columnTypeCode = getIntProperty("ATTR-TYPECODE" + i); - String columnTypeName = getProperty("ATTR-TYPENAME" + i); - String[] columnTypeMods = parseTypeMods(i); - - ColumnDescriptor column = new ColumnDescriptor(columnName, - columnTypeCode, i, columnTypeName, columnTypeMods); - tupleDescription.add(column); - - if (columnName.equalsIgnoreCase(ColumnDescriptor.RECORD_KEY_NAME)) { - recordkeyColumn = column; - } - } - } - - private String[] parseTypeMods(int columnIndex) { - String typeModeCountStr = getOptionalProperty("ATTR-TYPEMOD" + columnIndex + "COUNT"); - String[] result = null; - if (typeModeCountStr != null) { - Integer typeModeCount = Integer.parseInt(typeModeCountStr); - result = new String[typeModeCount]; - for (int i = 0; i < typeModeCount; i++) { - result[i] = getProperty("ATTR-TYPEMOD" + columnIndex + "-" + i); - } - } - return result; - } - - /** - * Sets the index of the allocated data fragment - * - * @param fragment the allocated data fragment - */ - protected void parseDataFragment(String fragment) { - - /* - * Some resources don't require a fragment, hence the list can be empty. - */ - if (StringUtils.isEmpty(fragment)) { - return; - } - dataFragment = Integer.parseInt(fragment); - } - - private void parseFragmentMetadata() { - fragmentMetadata = parseBase64("FRAGMENT-METADATA", - "Fragment metadata information"); - } - - private void parseUserData() { - userData = parseBase64("FRAGMENT-USER-DATA", "Fragment user data"); - } - - private byte[] parseBase64(String key, String errName) { - String encoded = getOptionalProperty(key); - if (encoded == null) { - return null; - } - if (!Base64.isArrayByteBase64(encoded.getBytes())) { - throw new IllegalArgumentException(errName - + " must be Base64 encoded." + "(Bad value: " + encoded - + ")"); - } - byte[] parsed = Base64.decodeBase64(encoded); - LOG.debug("decoded " + key + ": " + new String(parsed)); - return parsed; - } - - private void parseRemoteCredentials() { - remoteLogin = getOptionalProperty("REMOTE-USER"); - remoteSecret = getOptionalProperty("REMOTE-PASS"); - } - - private void parseStatsParameters() { - - String maxFrags = getOptionalProperty("STATS-MAX-FRAGMENTS"); - if (!StringUtils.isEmpty(maxFrags)) { - statsMaxFragments = Integer.parseInt(maxFrags); - if (statsMaxFragments <= 0) { - throw new IllegalArgumentException("Wrong value '" - + statsMaxFragments + "'. " - + "STATS-MAX-FRAGMENTS must be a positive integer"); - } - } - - String sampleRatioStr = getUserProperty("STATS-SAMPLE-RATIO"); - if (!StringUtils.isEmpty(sampleRatioStr)) { - statsSampleRatio = Float.parseFloat(sampleRatioStr); - if (statsSampleRatio < 0.0001 || statsSampleRatio > 1.0) { - throw new IllegalArgumentException( - "Wrong value '" - + statsSampleRatio - + "'. " - + "STATS-SAMPLE-RATIO must be a value between 0.0001 and 1.0"); - } - } - - if ((statsSampleRatio > 0) != (statsMaxFragments > 0)) { - throw new IllegalArgumentException( - "Missing parameter: STATS-SAMPLE-RATIO and STATS-MAX-FRAGMENTS must be set together"); - } - } -} http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/30aecce5/pxf/pxf-service/tmp/generatedSources/org/apache/hawq/pxf/service/utilities/SecureLogin.java ---------------------------------------------------------------------- diff --git a/pxf/pxf-service/tmp/generatedSources/org/apache/hawq/pxf/service/utilities/SecureLogin.java b/pxf/pxf-service/tmp/generatedSources/org/apache/hawq/pxf/service/utilities/SecureLogin.java deleted file mode 100644 index 6ce05ed..0000000 --- a/pxf/pxf-service/tmp/generatedSources/org/apache/hawq/pxf/service/utilities/SecureLogin.java +++ /dev/null @@ -1,61 +0,0 @@ -package org.apache.hawq.pxf.service.utilities; - -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.security.SecurityUtil; - -/** - * This class relies heavily on Hadoop API to - * <ul> - * <li>Check need for secure login in Hadoop</li> - * <li>Parse and load .xml configuration file</li> - * <li>Do a Kerberos login with a kaytab file</li> - * <li>convert _HOST in Kerberos principal to current hostname</li> - * </ul> - * - * It uses Hadoop Configuration to parse XML configuration files.<br> - * It uses Hadoop Security to modify principal and perform the login. - * - * The major limitation in this class is its dependency on Hadoop. If Hadoop - * security is off, no login will be performed regardless of connector being - * used. - */ -public class SecureLogin { - private static final Log LOG = LogFactory.getLog(SecureLogin.class); - private static final String CONFIG_KEY_SERVICE_KEYTAB = "pxf.service.kerberos.keytab"; - private static final String CONFIG_KEY_SERVICE_PRINCIPAL = "pxf.service.kerberos.principal"; - - public static void login() { - try { - Configuration config = new Configuration(); - config.addResource("pxf-site.xml"); - - SecurityUtil.login(config, CONFIG_KEY_SERVICE_KEYTAB, - CONFIG_KEY_SERVICE_PRINCIPAL); - } catch (Exception e) { - LOG.error("PXF service login failed"); - throw new RuntimeException(e); - } - } -} http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/30aecce5/pxf/pxf-service/tmp/generatedSources/org/apache/hawq/pxf/service/utilities/SecuredHDFS.java ---------------------------------------------------------------------- diff --git a/pxf/pxf-service/tmp/generatedSources/org/apache/hawq/pxf/service/utilities/SecuredHDFS.java b/pxf/pxf-service/tmp/generatedSources/org/apache/hawq/pxf/service/utilities/SecuredHDFS.java deleted file mode 100644 index f442a6d..0000000 --- a/pxf/pxf-service/tmp/generatedSources/org/apache/hawq/pxf/service/utilities/SecuredHDFS.java +++ /dev/null @@ -1,114 +0,0 @@ -package org.apache.hawq.pxf.service.utilities; - -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; -import org.apache.hadoop.hdfs.server.namenode.NameNode; -import org.apache.hadoop.hdfs.server.namenode.NameNodeHttpServer; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.security.token.Token; - -import javax.servlet.ServletContext; -import java.io.ByteArrayInputStream; -import java.io.DataInputStream; -import java.io.IOException; - -/** - * The class handles security functions for handling secured HDFS - */ -public class SecuredHDFS { - private static final Log LOG = LogFactory.getLog(SecuredHDFS.class); - - /** - * The function will get the token information from parameters and call - * SecuredHDFS to verify the token. - * - * All token properties will be deserialized from string to a Token object - * - * @param protData input parameters - * @param context servlet context which contains the NN address - * - * @throws SecurityException Thrown when authentication fails - */ - public static void verifyToken(ProtocolData protData, ServletContext context) { - try { - if (UserGroupInformation.isSecurityEnabled()) { - Token<DelegationTokenIdentifier> token = new Token<DelegationTokenIdentifier>(); - String tokenString = protData.getToken(); - token.decodeFromUrlString(tokenString); - - verifyToken(token.getIdentifier(), token.getPassword(), - token.getKind(), token.getService(), context); - } - } catch (IOException e) { - throw new SecurityException("Failed to verify delegation token " - + e, e); - } - } - - /** - * The function will verify the token with NameNode if available and will - * create a UserGroupInformation. - * - * Code in this function is copied from JspHelper.getTokenUGI - * - * @param identifier Delegation token identifier - * @param password Delegation token password - * @param kind the kind of token - * @param service the service for this token - * @param servletContext Jetty servlet context which contains the NN address - * - * @throws SecurityException Thrown when authentication fails - */ - private static void verifyToken(byte[] identifier, byte[] password, - Text kind, Text service, - ServletContext servletContext) { - try { - Token<DelegationTokenIdentifier> token = new Token<DelegationTokenIdentifier>( - identifier, password, kind, service); - - ByteArrayInputStream buf = new ByteArrayInputStream( - token.getIdentifier()); - DataInputStream in = new DataInputStream(buf); - DelegationTokenIdentifier id = new DelegationTokenIdentifier(); - id.readFields(in); - - final NameNode nn = NameNodeHttpServer.getNameNodeFromContext(servletContext); - if (nn != null) { - nn.getNamesystem().verifyToken(id, token.getPassword()); - } - - UserGroupInformation userGroupInformation = id.getUser(); - userGroupInformation.addToken(token); - LOG.debug("user " + userGroupInformation.getUserName() + " (" - + userGroupInformation.getShortUserName() - + ") authenticated"); - - // re-login if necessary - userGroupInformation.checkTGTAndReloginFromKeytab(); - } catch (IOException e) { - throw new SecurityException("Failed to verify delegation token " - + e, e); - } - } -}
