Author: rvs
Date: Thu Nov 3 02:55:36 2011
New Revision: 1196924
URL: http://svn.apache.org/viewvc?rev=1196924&view=rev
Log:
Enabling Hive
Added:
incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hive/patch
Modified:
incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hive/do-component-build
incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hive/rules
incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/rpm/hive/SPECS/hive.spec
incubator/bigtop/branches/hadoop-0.23/bigtop.mk
Modified:
incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hive/do-component-build
URL:
http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hive/do-component-build?rev=1196924&r1=1196923&r2=1196924&view=diff
==============================================================================
---
incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hive/do-component-build
(original)
+++
incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hive/do-component-build
Thu Nov 3 02:55:36 2011
@@ -15,5 +15,5 @@
# limitations under the License.
set -ex
-
-ant -f src/build.xml -Dhadoop.mirror=http://archive.cloudera.com/hive-deps
package "$@"
+# -f src/build.xml
+ant package "$@"
Added:
incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hive/patch
URL:
http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hive/patch?rev=1196924&view=auto
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hive/patch
(added)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/hive/patch
Thu Nov 3 02:55:36 2011
@@ -0,0 +1,1016 @@
+diff --git build.properties build.properties
+index fc80b03..6247e4b 100644
+--- build.properties
++++ build.properties
+@@ -10,10 +10,10 @@ javac.deprecation=off
+ javac.args=
+ javac.args.warnings=
+
+-hadoop.version=0.20.1
+-hadoop.security.version=0.20.3-CDH3-SNAPSHOT
+-hadoop.mirror=http://mirror.facebook.net/facebook/hive-deps
+-hadoop.mirror2=http://archive.cloudera.com/hive-deps
++hadoop.version=0.23.0
++hadoop.security.version=0.23.0
++hadoop.mirror=http://people.apache.org/~rvs/
++hadoop.mirror2=http://people.apache.org/~rvs/
+
+ build.dir.hive=${hive.root}/build
+ build.dir.hadoop=${build.dir.hive}/hadoopcore
+diff --git shims/src/0.23/java/org/apache/hadoop/fs/ProxyFileSystem.java
shims/src/0.23/java/org/apache/hadoop/fs/ProxyFileSystem.java
+new file mode 100644
+index 0000000..218236f
+--- /dev/null
++++ shims/src/0.23/java/org/apache/hadoop/fs/ProxyFileSystem.java
+@@ -0,0 +1,273 @@
++/**
++ * Licensed to the Apache Software Foundation (ASF) under one
++ * or more contributor license agreements. See the NOTICE file
++ * distributed with this work for additional information
++ * regarding copyright ownership. The ASF licenses this file
++ * to you under the Apache License, Version 2.0 (the
++ * "License"); you may not use this file except in compliance
++ * with the License. You may obtain a copy of the License at
++ *
++ * http://www.apache.org/licenses/LICENSE-2.0
++ *
++ * Unless required by applicable law or agreed to in writing, software
++ * distributed under the License is distributed on an "AS IS" BASIS,
++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ * See the License for the specific language governing permissions and
++ * limitations under the License.
++ */
++
++package org.apache.hadoop.fs;
++
++import java.io.IOException;
++import java.net.URI;
++import java.net.URISyntaxException;
++
++import org.apache.hadoop.conf.Configuration;
++import org.apache.hadoop.fs.permission.FsPermission;
++import org.apache.hadoop.util.Progressable;
++
++/****************************************************************
++ * A FileSystem that can serve a given scheme/authority using some
++ * other file system. In that sense, it serves as a proxy for the
++ * real/underlying file system
++ *****************************************************************/
++
++public class ProxyFileSystem extends FilterFileSystem {
++
++ protected String myScheme;
++ protected String myAuthority;
++ protected URI myUri;
++
++ protected String realScheme;
++ protected String realAuthority;
++ protected URI realUri;
++
++
++
++ private Path swizzleParamPath(Path p) {
++ return new Path (realScheme, realAuthority, p.toUri().getPath());
++ }
++
++ private Path swizzleReturnPath(Path p) {
++ return new Path (myScheme, myAuthority, p.toUri().getPath());
++ }
++
++ private FileStatus swizzleFileStatus(FileStatus orig, boolean isParam) {
++ FileStatus ret =
++ new FileStatus(orig.getLen(), orig.isDir(), orig.getReplication(),
++ orig.getBlockSize(), orig.getModificationTime(),
++ orig.getAccessTime(), orig.getPermission(),
++ orig.getOwner(), orig.getGroup(),
++ isParam ? swizzleParamPath(orig.getPath()) :
++ swizzleReturnPath(orig.getPath()));
++ return ret;
++ }
++
++ public ProxyFileSystem() {
++ throw new RuntimeException ("Unsupported constructor");
++ }
++
++ public ProxyFileSystem(FileSystem fs) {
++ throw new RuntimeException ("Unsupported constructor");
++ }
++
++ /**
++ * Create a proxy file system for fs.
++ *
++ * @param fs FileSystem to create proxy for
++ * @param myUri URI to use as proxy. Only the scheme and authority from
++ * this are used right now
++ */
++ public ProxyFileSystem(FileSystem fs, URI myUri) {
++ super(fs);
++
++ URI realUri = fs.getUri();
++ this.realScheme = realUri.getScheme();
++ this.realAuthority=realUri.getAuthority();
++ this.realUri = realUri;
++
++ this.myScheme = myUri.getScheme();
++ this.myAuthority=myUri.getAuthority();
++ this.myUri = myUri;
++ }
++
++ @Override
++ public void initialize(URI name, Configuration conf) throws IOException {
++ try {
++ URI realUri = new URI (realScheme, realAuthority,
++ name.getPath(), name.getQuery(),
name.getFragment());
++ super.initialize(realUri, conf);
++ } catch (URISyntaxException e) {
++ throw new RuntimeException(e);
++ }
++ }
++
++ @Override
++ public URI getUri() {
++ return myUri;
++ }
++
++ @Override
++ public String getName() {
++ return getUri().toString();
++ }
++
++ @Override
++ public Path makeQualified(Path path) {
++ return swizzleReturnPath(super.makeQualified(swizzleParamPath(path)));
++ }
++
++
++ @Override
++ protected void checkPath(Path path) {
++ super.checkPath(swizzleParamPath(path));
++ }
++
++ @Override
++ public BlockLocation[] getFileBlockLocations(FileStatus file, long start,
++ long len) throws IOException {
++ return super.getFileBlockLocations(swizzleFileStatus(file, true),
++ start, len);
++ }
++
++ @Override
++ public FSDataInputStream open(Path f, int bufferSize) throws IOException {
++ return super.open(swizzleParamPath(f), bufferSize);
++ }
++
++ @Override
++ public FSDataOutputStream append(Path f, int bufferSize,
++ Progressable progress) throws IOException {
++ return super.append(swizzleParamPath(f), bufferSize, progress);
++ }
++
++ @Override
++ public FSDataOutputStream create(Path f, FsPermission permission,
++ boolean overwrite, int bufferSize, short replication, long blockSize,
++ Progressable progress) throws IOException {
++ return super.create(swizzleParamPath(f), permission,
++ overwrite, bufferSize, replication, blockSize, progress);
++ }
++
++ @Override
++ public boolean setReplication(Path src, short replication) throws
IOException {
++ return super.setReplication(swizzleParamPath(src), replication);
++ }
++
++ @Override
++ public boolean rename(Path src, Path dst) throws IOException {
++ return super.rename(swizzleParamPath(src), swizzleParamPath(dst));
++ }
++
++ @Override
++ public boolean delete(Path f, boolean recursive) throws IOException {
++ return super.delete(swizzleParamPath(f), recursive);
++ }
++
++ @Override
++ public boolean deleteOnExit(Path f) throws IOException {
++ return super.deleteOnExit(swizzleParamPath(f));
++ }
++
++ @Override
++ public FileStatus[] listStatus(Path f) throws IOException {
++ FileStatus[] orig = super.listStatus(swizzleParamPath(f));
++ FileStatus[] ret = new FileStatus [orig.length];
++ for (int i=0; i<orig.length; i++) {
++ ret[i] = swizzleFileStatus(orig[i], false);
++ }
++ return ret;
++ }
++
++ @Override
++ public Path getHomeDirectory() {
++ return swizzleReturnPath(super.getHomeDirectory());
++ }
++
++ @Override
++ public void setWorkingDirectory(Path newDir) {
++ super.setWorkingDirectory(swizzleParamPath(newDir));
++ }
++
++ @Override
++ public Path getWorkingDirectory() {
++ return swizzleReturnPath(super.getWorkingDirectory());
++ }
++
++ @Override
++ public boolean mkdirs(Path f, FsPermission permission) throws IOException {
++ return super.mkdirs(swizzleParamPath(f), permission);
++ }
++
++ @Override
++ public void copyFromLocalFile(boolean delSrc, Path src, Path dst)
++ throws IOException {
++ super.copyFromLocalFile(delSrc, swizzleParamPath(src),
swizzleParamPath(dst));
++ }
++
++ @Override
++ public void copyFromLocalFile(boolean delSrc, boolean overwrite,
++ Path[] srcs, Path dst)
++ throws IOException {
++ super.copyFromLocalFile(delSrc, overwrite, srcs, swizzleParamPath(dst));
++ }
++
++ @Override
++ public void copyFromLocalFile(boolean delSrc, boolean overwrite,
++ Path src, Path dst)
++ throws IOException {
++ super.copyFromLocalFile(delSrc, overwrite, src, swizzleParamPath(dst));
++ }
++
++ @Override
++ public void copyToLocalFile(boolean delSrc, Path src, Path dst)
++ throws IOException {
++ super.copyToLocalFile(delSrc, swizzleParamPath(src), dst);
++ }
++
++ @Override
++ public Path startLocalOutput(Path fsOutputFile, Path tmpLocalFile)
++ throws IOException {
++ return super.startLocalOutput(swizzleParamPath(fsOutputFile),
tmpLocalFile);
++ }
++
++ @Override
++ public void completeLocalOutput(Path fsOutputFile, Path tmpLocalFile)
++ throws IOException {
++ super.completeLocalOutput(swizzleParamPath(fsOutputFile), tmpLocalFile);
++ }
++
++ @Override
++ public ContentSummary getContentSummary(Path f) throws IOException {
++ return super.getContentSummary(swizzleParamPath(f));
++ }
++
++ @Override
++ public FileStatus getFileStatus(Path f) throws IOException {
++ return swizzleFileStatus(super.getFileStatus(swizzleParamPath(f)), false);
++ }
++
++ @Override
++ public FileChecksum getFileChecksum(Path f) throws IOException {
++ return super.getFileChecksum(swizzleParamPath(f));
++ }
++
++ @Override
++ public void setOwner(Path p, String username, String groupname
++ ) throws IOException {
++ super.setOwner(swizzleParamPath(p), username, groupname);
++ }
++
++ @Override
++ public void setTimes(Path p, long mtime, long atime
++ ) throws IOException {
++ super.setTimes(swizzleParamPath(p), mtime, atime);
++ }
++
++ @Override
++ public void setPermission(Path p, FsPermission permission
++ ) throws IOException {
++ super.setPermission(swizzleParamPath(p), permission);
++ }
++}
++
+diff --git shims/src/0.23/java/org/apache/hadoop/fs/ProxyLocalFileSystem.java
shims/src/0.23/java/org/apache/hadoop/fs/ProxyLocalFileSystem.java
+new file mode 100644
+index 0000000..b28bf4e
+--- /dev/null
++++ shims/src/0.23/java/org/apache/hadoop/fs/ProxyLocalFileSystem.java
+@@ -0,0 +1,61 @@
++/**
++ * Licensed to the Apache Software Foundation (ASF) under one
++ * or more contributor license agreements. See the NOTICE file
++ * distributed with this work for additional information
++ * regarding copyright ownership. The ASF licenses this file
++ * to you under the Apache License, Version 2.0 (the
++ * "License"); you may not use this file except in compliance
++ * with the License. You may obtain a copy of the License at
++ *
++ * http://www.apache.org/licenses/LICENSE-2.0
++ *
++ * Unless required by applicable law or agreed to in writing, software
++ * distributed under the License is distributed on an "AS IS" BASIS,
++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ * See the License for the specific language governing permissions and
++ * limitations under the License.
++ */
++
++package org.apache.hadoop.fs;
++
++import java.io.*;
++import java.net.URI;
++import java.net.URISyntaxException;
++
++import org.apache.hadoop.conf.Configuration;
++import org.apache.hadoop.fs.permission.FsPermission;
++import org.apache.hadoop.util.Progressable;
++
++/****************************************************************
++ * A Proxy for LocalFileSystem
++ *
++ * Serves uri's corresponding to 'pfile:///' namespace with using
++ * a LocalFileSystem
++ *****************************************************************/
++
++public class ProxyLocalFileSystem extends FilterFileSystem {
++
++ protected LocalFileSystem localFs;
++
++ public ProxyLocalFileSystem() {
++ localFs = new LocalFileSystem();
++ }
++
++ public ProxyLocalFileSystem(FileSystem fs) {
++ throw new RuntimeException ("Unsupported Constructor");
++ }
++
++ @Override
++ public void initialize(URI name, Configuration conf) throws IOException {
++ // create a proxy for the local filesystem
++ // the scheme/authority serving as the proxy is derived
++ // from the supplied URI
++
++ String scheme = name.getScheme();
++ String authority = name.getAuthority() != null ? name.getAuthority() : "";
++ String proxyUriString = name + "://" + authority + "/";
++ fs = new ProxyFileSystem(localFs, URI.create(proxyUriString));
++
++ fs.initialize(name, conf);
++ }
++}
+diff --git shims/src/0.23/java/org/apache/hadoop/hive/shims/EmptyShim.java
shims/src/0.23/java/org/apache/hadoop/hive/shims/EmptyShim.java
+new file mode 100644
+index 0000000..0e5715d
+--- /dev/null
++++ shims/src/0.23/java/org/apache/hadoop/hive/shims/EmptyShim.java
+@@ -0,0 +1,4 @@
++package org.apache.hadoop.hive.shims;
++
++class EmptyShim {
++}
+diff --git shims/src/0.23/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java
shims/src/0.23/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java
+new file mode 100644
+index 0000000..3da7e28
+--- /dev/null
++++ shims/src/0.23/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java
+@@ -0,0 +1,501 @@
++/**
++ * Licensed to the Apache Software Foundation (ASF) under one
++ * or more contributor license agreements. See the NOTICE file
++ * distributed with this work for additional information
++ * regarding copyright ownership. The ASF licenses this file
++ * to you under the Apache License, Version 2.0 (the
++ * "License"); you may not use this file except in compliance
++ * with the License. You may obtain a copy of the License at
++ *
++ * http://www.apache.org/licenses/LICENSE-2.0
++ *
++ * Unless required by applicable law or agreed to in writing, software
++ * distributed under the License is distributed on an "AS IS" BASIS,
++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ * See the License for the specific language governing permissions and
++ * limitations under the License.
++ */
++package org.apache.hadoop.hive.shims;
++
++import java.io.DataInput;
++import java.io.DataOutput;
++import java.io.IOException;
++import java.lang.reflect.Constructor;
++import java.util.ArrayList;
++import java.util.List;
++
++import javax.security.auth.login.LoginException;
++
++import org.apache.hadoop.conf.Configuration;
++import org.apache.hadoop.fs.FileStatus;
++import org.apache.hadoop.fs.FileSystem;
++import org.apache.hadoop.fs.Path;
++import org.apache.hadoop.fs.PathFilter;
++import org.apache.hadoop.hdfs.MiniDFSCluster;
++import org.apache.hadoop.hive.io.HiveIOExceptionHandlerChain;
++import org.apache.hadoop.hive.io.HiveIOExceptionHandlerUtil;
++import org.apache.hadoop.io.Text;
++import org.apache.hadoop.mapred.FileInputFormat;
++import org.apache.hadoop.mapred.InputFormat;
++import org.apache.hadoop.mapred.InputSplit;
++import org.apache.hadoop.mapred.JobConf;
++import org.apache.hadoop.mapred.JobContext;
++import org.apache.hadoop.mapred.JobStatus;
++import org.apache.hadoop.mapred.OutputCommitter;
++import org.apache.hadoop.mapred.RecordReader;
++import org.apache.hadoop.mapred.Reporter;
++import org.apache.hadoop.mapred.RunningJob;
++import org.apache.hadoop.mapred.TaskAttemptContext;
++import org.apache.hadoop.mapred.TaskCompletionEvent;
++import org.apache.hadoop.mapred.TaskID;
++import org.apache.hadoop.mapred.lib.CombineFileInputFormat;
++import org.apache.hadoop.mapred.lib.CombineFileSplit;
++import org.apache.hadoop.mapred.lib.NullOutputFormat;
++import org.apache.hadoop.security.UserGroupInformation;
++import org.apache.hadoop.tools.HadoopArchives;
++import org.apache.hadoop.util.ToolRunner;
++
++/**
++ * Implemention of shims against Hadoop 0.20.0.
++ */
++public class Hadoop20Shims implements HadoopShims {
++ public boolean usesJobShell() {
++ return false;
++ }
++
++ public boolean fileSystemDeleteOnExit(FileSystem fs, Path path)
++ throws IOException {
++
++ return fs.deleteOnExit(path);
++ }
++
++ public void inputFormatValidateInput(InputFormat fmt, JobConf conf)
++ throws IOException {
++ // gone in 0.18+
++ }
++
++ public boolean isJobPreparing(RunningJob job) throws IOException {
++ return job.getJobState() == JobStatus.PREP;
++ }
++ /**
++ * Workaround for hadoop-17 - jobclient only looks at commandlineconfig.
++ */
++ public void setTmpFiles(String prop, String files) {
++ // gone in 20+
++ }
++
++ public HadoopShims.MiniDFSShim getMiniDfs(Configuration conf,
++ int numDataNodes,
++ boolean format,
++ String[] racks) throws IOException {
++ return new MiniDFSShim(new MiniDFSCluster(conf, numDataNodes, format,
racks));
++ }
++
++ /**
++ * MiniDFSShim.
++ *
++ */
++ public class MiniDFSShim implements HadoopShims.MiniDFSShim {
++ private final MiniDFSCluster cluster;
++
++ public MiniDFSShim(MiniDFSCluster cluster) {
++ this.cluster = cluster;
++ }
++
++ public FileSystem getFileSystem() throws IOException {
++ return cluster.getFileSystem();
++ }
++
++ public void shutdown() {
++ cluster.shutdown();
++ }
++ }
++
++ /**
++ * We define this function here to make the code compatible between
++ * hadoop 0.17 and hadoop 0.20.
++ *
++ * Hive binary that compiled Text.compareTo(Text) with hadoop 0.20 won't
++ * work with hadoop 0.17 because in hadoop 0.20, Text.compareTo(Text) is
++ * implemented in org.apache.hadoop.io.BinaryComparable, and Java compiler
++ * references that class, which is not available in hadoop 0.17.
++ */
++ public int compareText(Text a, Text b) {
++ return a.compareTo(b);
++ }
++
++ @Override
++ public long getAccessTime(FileStatus file) {
++ return file.getAccessTime();
++ }
++
++ public HadoopShims.CombineFileInputFormatShim getCombineFileInputFormat() {
++ return new CombineFileInputFormatShim() {
++ @Override
++ public RecordReader getRecordReader(InputSplit split,
++ JobConf job, Reporter reporter) throws IOException {
++ throw new IOException("CombineFileInputFormat.getRecordReader not
needed.");
++ }
++ };
++ }
++
++ public static class InputSplitShim extends CombineFileSplit implements
HadoopShims.InputSplitShim {
++ long shrinkedLength;
++ boolean _isShrinked;
++ public InputSplitShim() {
++ super();
++ _isShrinked = false;
++ }
++
++ public InputSplitShim(CombineFileSplit old) throws IOException {
++ super(old);
++ _isShrinked = false;
++ }
++
++ @Override
++ public void shrinkSplit(long length) {
++ _isShrinked = true;
++ shrinkedLength = length;
++ }
++
++ public boolean isShrinked() {
++ return _isShrinked;
++ }
++
++ public long getShrinkedLength() {
++ return shrinkedLength;
++ }
++
++ @Override
++ public void readFields(DataInput in) throws IOException {
++ super.readFields(in);
++ _isShrinked = in.readBoolean();
++ if (_isShrinked) {
++ shrinkedLength = in.readLong();
++ }
++ }
++
++ @Override
++ public void write(DataOutput out) throws IOException {
++ super.write(out);
++ out.writeBoolean(_isShrinked);
++ if (_isShrinked) {
++ out.writeLong(shrinkedLength);
++ }
++ }
++ }
++
++ /* This class should be replaced with
org.apache.hadoop.mapred.lib.CombineFileRecordReader class, once
++ * https://issues.apache.org/jira/browse/MAPREDUCE-955 is fixed. This code
should be removed - it is a copy
++ * of org.apache.hadoop.mapred.lib.CombineFileRecordReader
++ */
++ public static class CombineFileRecordReader<K, V> implements
RecordReader<K, V> {
++
++ static final Class[] constructorSignature = new Class[] {
++ InputSplit.class,
++ Configuration.class,
++ Reporter.class,
++ Integer.class
++ };
++
++ protected CombineFileSplit split;
++ protected JobConf jc;
++ protected Reporter reporter;
++ protected Class<RecordReader<K, V>> rrClass;
++ protected Constructor<RecordReader<K, V>> rrConstructor;
++ protected FileSystem fs;
++
++ protected int idx;
++ protected long progress;
++ protected RecordReader<K, V> curReader;
++ protected boolean isShrinked;
++ protected long shrinkedLength;
++
++ public boolean next(K key, V value) throws IOException {
++
++ while ((curReader == null)
++ || !doNextWithExceptionHandler((K) ((CombineHiveKey) key).getKey(),
++ value)) {
++ if (!initNextRecordReader(key)) {
++ return false;
++ }
++ }
++ return true;
++ }
++
++ public K createKey() {
++ K newKey = curReader.createKey();
++ return (K)(new CombineHiveKey(newKey));
++ }
++
++ public V createValue() {
++ return curReader.createValue();
++ }
++
++ /**
++ * Return the amount of data processed.
++ */
++ public long getPos() throws IOException {
++ return progress;
++ }
++
++ public void close() throws IOException {
++ if (curReader != null) {
++ curReader.close();
++ curReader = null;
++ }
++ }
++
++ /**
++ * Return progress based on the amount of data processed so far.
++ */
++ public float getProgress() throws IOException {
++ return Math.min(1.0f, progress / (float) (split.getLength()));
++ }
++
++ /**
++ * A generic RecordReader that can hand out different recordReaders
++ * for each chunk in the CombineFileSplit.
++ */
++ public CombineFileRecordReader(JobConf job, CombineFileSplit split,
++ Reporter reporter,
++ Class<RecordReader<K, V>> rrClass)
++ throws IOException {
++ this.split = split;
++ this.jc = job;
++ this.rrClass = rrClass;
++ this.reporter = reporter;
++ this.idx = 0;
++ this.curReader = null;
++ this.progress = 0;
++
++ isShrinked = false;
++
++ assert (split instanceof InputSplitShim);
++ if (((InputSplitShim) split).isShrinked()) {
++ isShrinked = true;
++ shrinkedLength = ((InputSplitShim) split).getShrinkedLength();
++ }
++
++ try {
++ rrConstructor = rrClass.getDeclaredConstructor(constructorSignature);
++ rrConstructor.setAccessible(true);
++ } catch (Exception e) {
++ throw new RuntimeException(rrClass.getName() +
++ " does not have valid constructor", e);
++ }
++ initNextRecordReader(null);
++ }
++
++ /**
++ * do next and handle exception inside it.
++ * @param key
++ * @param value
++ * @return
++ * @throws IOException
++ */
++ private boolean doNextWithExceptionHandler(K key, V value) throws
IOException {
++ try {
++ return curReader.next(key, value);
++ } catch (Exception e) {
++ return HiveIOExceptionHandlerUtil.handleRecordReaderNextException(e,
jc);
++ }
++ }
++
++ /**
++ * Get the record reader for the next chunk in this CombineFileSplit.
++ */
++ protected boolean initNextRecordReader(K key) throws IOException {
++
++ if (curReader != null) {
++ curReader.close();
++ curReader = null;
++ if (idx > 0) {
++ progress += split.getLength(idx - 1); // done processing so far
++ }
++ }
++
++ // if all chunks have been processed or reached the length, nothing
more to do.
++ if (idx == split.getNumPaths() || (isShrinked && progress >
shrinkedLength)) {
++ return false;
++ }
++
++ // get a record reader for the idx-th chunk
++ try {
++ curReader = rrConstructor.newInstance(new Object[]
++ {split, jc, reporter, Integer.valueOf(idx)});
++
++ // change the key if need be
++ if (key != null) {
++ K newKey = curReader.createKey();
++ ((CombineHiveKey)key).setKey(newKey);
++ }
++
++ // setup some helper config variables.
++ jc.set("map.input.file", split.getPath(idx).toString());
++ jc.setLong("map.input.start", split.getOffset(idx));
++ jc.setLong("map.input.length", split.getLength(idx));
++ } catch (Exception e) {
++
curReader=HiveIOExceptionHandlerUtil.handleRecordReaderCreationException(e, jc);
++ }
++ idx++;
++ return true;
++ }
++ }
++
++ public abstract static class CombineFileInputFormatShim<K, V> extends
++ CombineFileInputFormat<K, V>
++ implements HadoopShims.CombineFileInputFormatShim<K, V> {
++
++ public Path[] getInputPathsShim(JobConf conf) {
++ try {
++ return FileInputFormat.getInputPaths(conf);
++ } catch (Exception e) {
++ throw new RuntimeException(e);
++ }
++ }
++
++ @Override
++ public void createPool(JobConf conf, PathFilter... filters) {
++ super.createPool(conf, filters);
++ }
++
++ @Override
++ public InputSplitShim[] getSplits(JobConf job, int numSplits) throws
IOException {
++ long minSize = job.getLong("mapred.min.split.size", 0);
++
++ // For backward compatibility, let the above parameter be used
++ if (job.getLong("mapred.min.split.size.per.node", 0) == 0) {
++ super.setMinSplitSizeNode(minSize);
++ }
++
++ if (job.getLong("mapred.min.split.size.per.rack", 0) == 0) {
++ super.setMinSplitSizeRack(minSize);
++ }
++
++ if (job.getLong("mapred.max.split.size", 0) == 0) {
++ super.setMaxSplitSize(minSize);
++ }
++
++ CombineFileSplit[] splits = (CombineFileSplit[]) super.getSplits(job,
numSplits);
++
++ InputSplitShim[] isplits = new InputSplitShim[splits.length];
++ for (int pos = 0; pos < splits.length; pos++) {
++ isplits[pos] = new InputSplitShim(splits[pos]);
++ }
++
++ return isplits;
++ }
++
++ public InputSplitShim getInputSplitShim() throws IOException {
++ return new InputSplitShim();
++ }
++
++ public RecordReader getRecordReader(JobConf job,
HadoopShims.InputSplitShim split,
++ Reporter reporter,
++ Class<RecordReader<K, V>> rrClass)
++ throws IOException {
++ CombineFileSplit cfSplit = (CombineFileSplit) split;
++ return new CombineFileRecordReader(job, cfSplit, reporter, rrClass);
++ }
++
++ }
++
++ public String getInputFormatClassName() {
++ return "org.apache.hadoop.hive.ql.io.CombineHiveInputFormat";
++ }
++
++ String[] ret = new String[2];
++
++ @Override
++ public String[] getTaskJobIDs(TaskCompletionEvent t) {
++ TaskID tid = t.getTaskAttemptId().getTaskID();
++ ret[0] = tid.toString();
++ ret[1] = tid.getJobID().toString();
++ return ret;
++ }
++
++ public void setFloatConf(Configuration conf, String varName, float val) {
++ conf.setFloat(varName, val);
++ }
++
++ @Override
++ public int createHadoopArchive(Configuration conf, Path sourceDir, Path
destDir,
++ String archiveName) throws Exception {
++
++ HadoopArchives har = new HadoopArchives(conf);
++ List<String> args = new ArrayList<String>();
++
++ if (conf.get("hive.archive.har.parentdir.settable") == null) {
++ throw new RuntimeException("hive.archive.har.parentdir.settable is not
set");
++ }
++ boolean parentSettable =
++ conf.getBoolean("hive.archive.har.parentdir.settable", false);
++
++ if (parentSettable) {
++ args.add("-archiveName");
++ args.add(archiveName);
++ args.add("-p");
++ args.add(sourceDir.toString());
++ args.add(destDir.toString());
++ } else {
++ args.add("-archiveName");
++ args.add(archiveName);
++ args.add(sourceDir.toString());
++ args.add(destDir.toString());
++ }
++
++ return ToolRunner.run(har, args.toArray(new String[0]));
++ }
++
++ public static class NullOutputCommitter extends OutputCommitter {
++ @Override
++ public void setupJob(JobContext jobContext) { }
++ @Override
++ public void cleanupJob(JobContext jobContext) { }
++
++ @Override
++ public void setupTask(TaskAttemptContext taskContext) { }
++ @Override
++ public boolean needsTaskCommit(TaskAttemptContext taskContext) {
++ return false;
++ }
++ @Override
++ public void commitTask(TaskAttemptContext taskContext) { }
++ @Override
++ public void abortTask(TaskAttemptContext taskContext) { }
++ }
++
++ public void setNullOutputFormat(JobConf conf) {
++ conf.setOutputFormat(NullOutputFormat.class);
++ conf.setOutputCommitter(Hadoop20Shims.NullOutputCommitter.class);
++
++ // option to bypass job setup and cleanup was introduced in hadoop-21
(MAPREDUCE-463)
++ // but can be backported. So we disable setup/cleanup in all versions >=
0.19
++ conf.setBoolean("mapred.committer.job.setup.cleanup.needed", false);
++
++ // option to bypass task cleanup task was introduced in hadoop-23
(MAPREDUCE-2206)
++ // but can be backported. So we disable setup/cleanup in all versions >=
0.19
++ conf.setBoolean("mapreduce.job.committer.task.cleanup.needed", false);
++ }
++
++ @Override
++ public UserGroupInformation getUGIForConf(Configuration conf) throws
IOException {
++ return UserGroupInformation.getCurrentUser();
++ }
++
++ @Override
++ public boolean isSecureShimImpl() {
++ return false;
++ }
++
++ @Override
++ public String getShortUserName(UserGroupInformation ugi) {
++ return ugi.getUserName();
++ }
++
++ @Override
++ public String getTokenStrForm(String tokenSignature) throws IOException {
++ throw new UnsupportedOperationException("Tokens are not supported in
current hadoop version");
++ }
++}
+diff --git
shims/src/0.23/java/org/apache/hadoop/hive/shims/HiveHarFileSystem.java
shims/src/0.23/java/org/apache/hadoop/hive/shims/HiveHarFileSystem.java
+new file mode 100644
+index 0000000..323ebbb
+--- /dev/null
++++ shims/src/0.23/java/org/apache/hadoop/hive/shims/HiveHarFileSystem.java
+@@ -0,0 +1,66 @@
++/**
++ * Licensed to the Apache Software Foundation (ASF) under one
++ * or more contributor license agreements. See the NOTICE file
++ * distributed with this work for additional information
++ * regarding copyright ownership. The ASF licenses this file
++ * to you under the Apache License, Version 2.0 (the
++ * "License"); you may not use this file except in compliance
++ * with the License. You may obtain a copy of the License at
++ *
++ * http://www.apache.org/licenses/LICENSE-2.0
++ *
++ * Unless required by applicable law or agreed to in writing, software
++ * distributed under the License is distributed on an "AS IS" BASIS,
++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ * See the License for the specific language governing permissions and
++ * limitations under the License.
++ */
++
++package org.apache.hadoop.hive.shims;
++
++import java.io.IOException;
++
++import org.apache.hadoop.fs.BlockLocation;
++import org.apache.hadoop.fs.ContentSummary;
++import org.apache.hadoop.fs.FileStatus;
++import org.apache.hadoop.fs.HarFileSystem;
++import org.apache.hadoop.fs.Path;
++
++/**
++ * HiveHarFileSystem - fixes issues with Hadoop's HarFileSystem
++ *
++ */
++public class HiveHarFileSystem extends HarFileSystem {
++
++ @Override
++ public BlockLocation[] getFileBlockLocations(FileStatus file, long start,
++ long len) throws IOException {
++
++ // In some places (e.g. FileInputFormat) this BlockLocation is used to
++ // figure out sizes/offsets and so a completely blank one will not work.
++ String [] hosts = {"DUMMY_HOST"};
++ return new BlockLocation[]{new BlockLocation(null, hosts, 0,
file.getLen())};
++ }
++
++ @Override
++ public ContentSummary getContentSummary(Path f) throws IOException {
++ // HarFileSystem has a bug where this method does not work properly
++ // if the underlying FS is HDFS. See MAPREDUCE-1877 for more
++ // information. This method is from FileSystem.
++ FileStatus status = getFileStatus(f);
++ if (!status.isDir()) {
++ // f is a file
++ return new ContentSummary(status.getLen(), 1, 0);
++ }
++ // f is a directory
++ long[] summary = {0, 0, 1};
++ for(FileStatus s : listStatus(f)) {
++ ContentSummary c = s.isDir() ? getContentSummary(s.getPath()) :
++ new ContentSummary(s.getLen(), 1, 0);
++ summary[0] += c.getLength();
++ summary[1] += c.getFileCount();
++ summary[2] += c.getDirectoryCount();
++ }
++ return new ContentSummary(summary[0], summary[1], summary[2]);
++ }
++}
+diff --git shims/src/0.23/java/org/apache/hadoop/hive/shims/Jetty20Shims.java
shims/src/0.23/java/org/apache/hadoop/hive/shims/Jetty20Shims.java
+new file mode 100644
+index 0000000..13c6b31
+--- /dev/null
++++ shims/src/0.23/java/org/apache/hadoop/hive/shims/Jetty20Shims.java
+@@ -0,0 +1,56 @@
++/**
++ * Licensed to the Apache Software Foundation (ASF) under one
++ * or more contributor license agreements. See the NOTICE file
++ * distributed with this work for additional information
++ * regarding copyright ownership. The ASF licenses this file
++ * to you under the Apache License, Version 2.0 (the
++ * "License"); you may not use this file except in compliance
++ * with the License. You may obtain a copy of the License at
++ *
++ * http://www.apache.org/licenses/LICENSE-2.0
++ *
++ * Unless required by applicable law or agreed to in writing, software
++ * distributed under the License is distributed on an "AS IS" BASIS,
++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ * See the License for the specific language governing permissions and
++ * limitations under the License.
++ */
++package org.apache.hadoop.hive.shims;
++
++import java.io.IOException;
++
++import org.mortbay.jetty.bio.SocketConnector;
++import org.mortbay.jetty.handler.RequestLogHandler;
++import org.mortbay.jetty.webapp.WebAppContext;
++
++/**
++ * Jetty20Shims.
++ *
++ */
++public class Jetty20Shims implements JettyShims {
++ public Server startServer(String listen, int port) throws IOException {
++ Server s = new Server();
++ s.setupListenerHostPort(listen, port);
++ return s;
++ }
++
++ private static class Server extends org.mortbay.jetty.Server implements
JettyShims.Server {
++ public void addWar(String war, String contextPath) {
++ WebAppContext wac = new WebAppContext();
++ wac.setContextPath(contextPath);
++ wac.setWar(war);
++ RequestLogHandler rlh = new RequestLogHandler();
++ rlh.setHandler(wac);
++ this.addHandler(rlh);
++ }
++
++ public void setupListenerHostPort(String listen, int port)
++ throws IOException {
++
++ SocketConnector connector = new SocketConnector();
++ connector.setPort(port);
++ connector.setHost(listen);
++ this.addConnector(connector);
++ }
++ }
++}
Modified:
incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hive/rules
URL:
http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hive/rules?rev=1196924&r1=1196923&r2=1196924&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hive/rules
(original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/hive/rules
Thu Nov 3 02:55:36 2011
@@ -26,13 +26,14 @@ export DH_OPTIONS
clean:
dh_testdir
dh_testroot
- ant -f src/build.xml clean
+ ant clean
rm -f *-stamp
rm -Rf debian/.ivy
dh_clean
build-indep: build-indep-stamp
build-indep-stamp:
+ patch -p0 < debian/patch
mkdir -p /tmp/debian-hive/.ivy
bash debian/do-component-build -Divy.home=`pwd`/debian/.ivy
touch $@
Modified:
incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/rpm/hive/SPECS/hive.spec
URL:
http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/rpm/hive/SPECS/hive.spec?rev=1196924&r1=1196923&r2=1196924&view=diff
==============================================================================
---
incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/rpm/hive/SPECS/hive.spec
(original)
+++
incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/rpm/hive/SPECS/hive.spec
Thu Nov 3 02:55:36 2011
@@ -70,6 +70,7 @@ Source5: hadoop-hive-server.default
Source6: hadoop-hive-metastore.default
Source7: hive.1
Source8: hive-site.xml
+Patch0: patch
Requires: hadoop >= 0.20.2, bigtop-utils
Obsoletes: %{name}-webinterface
@@ -114,7 +115,8 @@ This optional package hosts a metadata s
%prep
-%setup -n hive-%{hive_base_version}
+%setup -n apache-hive-6e882d8
+%patch0 -p0
%build
bash %{SOURCE1}
Modified: incubator/bigtop/branches/hadoop-0.23/bigtop.mk
URL:
http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop.mk?rev=1196924&r1=1196923&r2=1196924&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop.mk (original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop.mk Thu Nov 3 02:55:36 2011
@@ -72,12 +72,14 @@ $(eval $(call PACKAGE,pig,PIG))
HIVE_NAME=hive
HIVE_RELNOTES_NAME=Apache Hive
HIVE_PKG_NAME=hadoop-hive
-HIVE_BASE_VERSION=0.7.1
+HIVE_BASE_VERSION=0.9.0
HIVE_PKG_VERSION=$(HIVE_BASE_VERSION)
HIVE_RELEASE_VERSION=1
HIVE_TARBALL_DST=hive-$(HIVE_BASE_VERSION).tar.gz
-HIVE_TARBALL_SRC=$(HIVE_TARBALL_DST)
-HIVE_SITE=$(APACHE_MIRROR)/hive/hive-$(HIVE_BASE_VERSION)/
+#HIVE_TARBALL_SRC=$(HIVE_TARBALL_DST)
+#HIVE_SITE=$(APACHE_MIRROR)/hive/hive-$(HIVE_BASE_VERSION)/
+HIVE_SITE=https://github.com/apache/hive/tarball
+HIVE_TARBALL_SRC=6e882d8
$(eval $(call PACKAGE,hive,HIVE))
# Sqoop
