Author: olga Date: Mon Oct 29 14:44:47 2007 New Revision: 589866 URL: http://svn.apache.org/viewvc?rev=589866&view=rev Log: initial code submission
Added: incubator/pig/trunk/build.xml incubator/pig/trunk/lib/ incubator/pig/trunk/lib-src/ incubator/pig/trunk/lib-src/bzip2/ incubator/pig/trunk/lib-src/bzip2/org/ incubator/pig/trunk/lib-src/bzip2/org/apache/ incubator/pig/trunk/lib-src/bzip2/org/apache/tools/ incubator/pig/trunk/lib-src/bzip2/org/apache/tools/bzip2r/ incubator/pig/trunk/lib-src/bzip2/org/apache/tools/bzip2r/BZip2Constants.java incubator/pig/trunk/lib-src/bzip2/org/apache/tools/bzip2r/CBZip2InputStream.java incubator/pig/trunk/lib-src/bzip2/org/apache/tools/bzip2r/CBZip2OutputStream.java incubator/pig/trunk/lib-src/bzip2/org/apache/tools/bzip2r/CRC.java incubator/pig/trunk/lib-src/shock/ incubator/pig/trunk/lib-src/shock/org/ incubator/pig/trunk/lib-src/shock/org/apache/ incubator/pig/trunk/lib-src/shock/org/apache/pig/ incubator/pig/trunk/lib-src/shock/org/apache/pig/shock/ incubator/pig/trunk/lib-src/shock/org/apache/pig/shock/SSHSocketImplFactory.java incubator/pig/trunk/lib/hadoop13.jar (with props) incubator/pig/trunk/lib/hadoop14.jar (with props) incubator/pig/trunk/lib/javacc.jar (with props) incubator/pig/trunk/lib/jsch-0.1.33.jar (with props) incubator/pig/trunk/lib/junit-4.1.jar (with props) incubator/pig/trunk/scripts/ incubator/pig/trunk/scripts/pig.pl incubator/pig/trunk/scripts/startHOD.expect (with props) incubator/pig/trunk/src/ incubator/pig/trunk/src/org/ incubator/pig/trunk/src/org/apache/ incubator/pig/trunk/src/org/apache/pig/ incubator/pig/trunk/src/org/apache/pig/Algebraic.java incubator/pig/trunk/src/org/apache/pig/EvalFunc.java incubator/pig/trunk/src/org/apache/pig/FilterFunc.java incubator/pig/trunk/src/org/apache/pig/LoadFunc.java incubator/pig/trunk/src/org/apache/pig/Main.java incubator/pig/trunk/src/org/apache/pig/PigServer.java incubator/pig/trunk/src/org/apache/pig/StandAloneParser.java incubator/pig/trunk/src/org/apache/pig/StoreFunc.java incubator/pig/trunk/src/org/apache/pig/builtin/ incubator/pig/trunk/src/org/apache/pig/builtin/ARITY.java incubator/pig/trunk/src/org/apache/pig/builtin/AVG.java incubator/pig/trunk/src/org/apache/pig/builtin/BinStorage.java incubator/pig/trunk/src/org/apache/pig/builtin/COUNT.java incubator/pig/trunk/src/org/apache/pig/builtin/DIFF.java incubator/pig/trunk/src/org/apache/pig/builtin/IsEmpty.java incubator/pig/trunk/src/org/apache/pig/builtin/MAX.java incubator/pig/trunk/src/org/apache/pig/builtin/MIN.java incubator/pig/trunk/src/org/apache/pig/builtin/PigDump.java incubator/pig/trunk/src/org/apache/pig/builtin/PigStorage.java incubator/pig/trunk/src/org/apache/pig/builtin/SUM.java incubator/pig/trunk/src/org/apache/pig/builtin/TOKENIZE.java incubator/pig/trunk/src/org/apache/pig/builtin/TextLoader.java incubator/pig/trunk/src/org/apache/pig/data/ incubator/pig/trunk/src/org/apache/pig/data/AmendableTuple.java incubator/pig/trunk/src/org/apache/pig/data/BagFactory.java incubator/pig/trunk/src/org/apache/pig/data/BigDataBag.java incubator/pig/trunk/src/org/apache/pig/data/DataAtom.java incubator/pig/trunk/src/org/apache/pig/data/DataBag.java incubator/pig/trunk/src/org/apache/pig/data/DataMap.java incubator/pig/trunk/src/org/apache/pig/data/Datum.java incubator/pig/trunk/src/org/apache/pig/data/IndexedTuple.java incubator/pig/trunk/src/org/apache/pig/data/TimestampedTuple.java incubator/pig/trunk/src/org/apache/pig/data/Tuple.java incubator/pig/trunk/src/org/apache/pig/impl/ incubator/pig/trunk/src/org/apache/pig/impl/FunctionInstantiator.java incubator/pig/trunk/src/org/apache/pig/impl/PigContext.java incubator/pig/trunk/src/org/apache/pig/impl/builtin/ incubator/pig/trunk/src/org/apache/pig/impl/builtin/ADD.java incubator/pig/trunk/src/org/apache/pig/impl/builtin/DIVIDE.java incubator/pig/trunk/src/org/apache/pig/impl/builtin/FindQuantiles.java incubator/pig/trunk/src/org/apache/pig/impl/builtin/GFAny.java incubator/pig/trunk/src/org/apache/pig/impl/builtin/GFCross.java incubator/pig/trunk/src/org/apache/pig/impl/builtin/GFReplicate.java incubator/pig/trunk/src/org/apache/pig/impl/builtin/MULTIPLY.java incubator/pig/trunk/src/org/apache/pig/impl/builtin/RandomSampleLoader.java incubator/pig/trunk/src/org/apache/pig/impl/builtin/SUBTRACT.java incubator/pig/trunk/src/org/apache/pig/impl/builtin/ShellBagEvalFunc.java incubator/pig/trunk/src/org/apache/pig/impl/eval/ incubator/pig/trunk/src/org/apache/pig/impl/eval/BinCondSpec.java incubator/pig/trunk/src/org/apache/pig/impl/eval/CompositeEvalSpec.java incubator/pig/trunk/src/org/apache/pig/impl/eval/ConstSpec.java incubator/pig/trunk/src/org/apache/pig/impl/eval/EvalSpec.java incubator/pig/trunk/src/org/apache/pig/impl/eval/FilterSpec.java incubator/pig/trunk/src/org/apache/pig/impl/eval/FuncEvalSpec.java incubator/pig/trunk/src/org/apache/pig/impl/eval/GenerateSpec.java incubator/pig/trunk/src/org/apache/pig/impl/eval/MapLookupSpec.java incubator/pig/trunk/src/org/apache/pig/impl/eval/ProjectSpec.java incubator/pig/trunk/src/org/apache/pig/impl/eval/SimpleEvalSpec.java incubator/pig/trunk/src/org/apache/pig/impl/eval/SortDistinctSpec.java incubator/pig/trunk/src/org/apache/pig/impl/eval/StarSpec.java incubator/pig/trunk/src/org/apache/pig/impl/eval/collector/ incubator/pig/trunk/src/org/apache/pig/impl/eval/collector/DataCollector.java incubator/pig/trunk/src/org/apache/pig/impl/eval/collector/FlattenCollector.java incubator/pig/trunk/src/org/apache/pig/impl/eval/collector/UnflattenCollector.java incubator/pig/trunk/src/org/apache/pig/impl/eval/cond/ incubator/pig/trunk/src/org/apache/pig/impl/eval/cond/AndCond.java incubator/pig/trunk/src/org/apache/pig/impl/eval/cond/CompCond.java incubator/pig/trunk/src/org/apache/pig/impl/eval/cond/Cond.java incubator/pig/trunk/src/org/apache/pig/impl/eval/cond/FalseCond.java incubator/pig/trunk/src/org/apache/pig/impl/eval/cond/FuncCond.java incubator/pig/trunk/src/org/apache/pig/impl/eval/cond/NotCond.java incubator/pig/trunk/src/org/apache/pig/impl/eval/cond/OrCond.java incubator/pig/trunk/src/org/apache/pig/impl/eval/cond/RegexpCond.java incubator/pig/trunk/src/org/apache/pig/impl/eval/cond/TrueCond.java incubator/pig/trunk/src/org/apache/pig/impl/eval/window/ incubator/pig/trunk/src/org/apache/pig/impl/eval/window/TimeWindowSpec.java incubator/pig/trunk/src/org/apache/pig/impl/eval/window/TupleWindowSpec.java incubator/pig/trunk/src/org/apache/pig/impl/eval/window/WindowSpec.java incubator/pig/trunk/src/org/apache/pig/impl/io/ incubator/pig/trunk/src/org/apache/pig/impl/io/BufferedPositionedInputStream.java incubator/pig/trunk/src/org/apache/pig/impl/io/DataBagFileReader.java incubator/pig/trunk/src/org/apache/pig/impl/io/DataBagFileWriter.java incubator/pig/trunk/src/org/apache/pig/impl/io/FileLocalizer.java incubator/pig/trunk/src/org/apache/pig/impl/io/FileSpec.java incubator/pig/trunk/src/org/apache/pig/impl/io/PigFile.java incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/ incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/LOCogroup.java incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/LOEval.java incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/LOLoad.java incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/LORead.java incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/LOSort.java incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/LOSplit.java incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/LOStore.java incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/LOUnion.java incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/LogicalOperator.java incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/LogicalPlan.java incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/LogicalPlanBuilder.java incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/parser/ incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/parser/grammar incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/schema/ incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/schema/AtomSchema.java incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/schema/Schema.java incubator/pig/trunk/src/org/apache/pig/impl/logicalLayer/schema/TupleSchema.java incubator/pig/trunk/src/org/apache/pig/impl/mapreduceExec/ incubator/pig/trunk/src/org/apache/pig/impl/mapreduceExec/MapReduceLauncher.java incubator/pig/trunk/src/org/apache/pig/impl/mapreduceExec/PigCombine.java incubator/pig/trunk/src/org/apache/pig/impl/mapreduceExec/PigInputFormat.java incubator/pig/trunk/src/org/apache/pig/impl/mapreduceExec/PigMapReduce.java incubator/pig/trunk/src/org/apache/pig/impl/mapreduceExec/PigOutputFormat.java incubator/pig/trunk/src/org/apache/pig/impl/mapreduceExec/PigSplit.java incubator/pig/trunk/src/org/apache/pig/impl/mapreduceExec/SortPartitioner.java incubator/pig/trunk/src/org/apache/pig/impl/physicalLayer/ incubator/pig/trunk/src/org/apache/pig/impl/physicalLayer/IntermedResult.java incubator/pig/trunk/src/org/apache/pig/impl/physicalLayer/LocalPlanCompiler.java incubator/pig/trunk/src/org/apache/pig/impl/physicalLayer/MapreducePlanCompiler.java incubator/pig/trunk/src/org/apache/pig/impl/physicalLayer/POCogroup.java incubator/pig/trunk/src/org/apache/pig/impl/physicalLayer/POEval.java incubator/pig/trunk/src/org/apache/pig/impl/physicalLayer/POLoad.java incubator/pig/trunk/src/org/apache/pig/impl/physicalLayer/POMapreduce.java incubator/pig/trunk/src/org/apache/pig/impl/physicalLayer/PORead.java incubator/pig/trunk/src/org/apache/pig/impl/physicalLayer/POSort.java incubator/pig/trunk/src/org/apache/pig/impl/physicalLayer/POSplit.java incubator/pig/trunk/src/org/apache/pig/impl/physicalLayer/POSplitMaster.java incubator/pig/trunk/src/org/apache/pig/impl/physicalLayer/POSplitSlave.java incubator/pig/trunk/src/org/apache/pig/impl/physicalLayer/POStore.java incubator/pig/trunk/src/org/apache/pig/impl/physicalLayer/POUnion.java incubator/pig/trunk/src/org/apache/pig/impl/physicalLayer/PhysicalOperator.java incubator/pig/trunk/src/org/apache/pig/impl/physicalLayer/PhysicalPlan.java incubator/pig/trunk/src/org/apache/pig/impl/physicalLayer/PlanCompiler.java incubator/pig/trunk/src/org/apache/pig/impl/physicalLayer/SplitSpec.java incubator/pig/trunk/src/org/apache/pig/impl/util/ incubator/pig/trunk/src/org/apache/pig/impl/util/DataBuffer.java incubator/pig/trunk/src/org/apache/pig/impl/util/IdentityHashSet.java incubator/pig/trunk/src/org/apache/pig/impl/util/JarManager.java incubator/pig/trunk/src/org/apache/pig/impl/util/LineageTracer.java incubator/pig/trunk/src/org/apache/pig/impl/util/ObjectSerializer.java incubator/pig/trunk/src/org/apache/pig/impl/util/RewindableIterator.java incubator/pig/trunk/src/org/apache/pig/tools/ incubator/pig/trunk/src/org/apache/pig/tools/cmdline/ incubator/pig/trunk/src/org/apache/pig/tools/cmdline/CmdLineParser.java incubator/pig/trunk/src/org/apache/pig/tools/grunt/ incubator/pig/trunk/src/org/apache/pig/tools/grunt/Command.java incubator/pig/trunk/src/org/apache/pig/tools/grunt/Grunt.java incubator/pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.jj incubator/pig/trunk/src/org/apache/pig/tools/streams/ incubator/pig/trunk/src/org/apache/pig/tools/streams/StreamGenerator.java incubator/pig/trunk/src/org/apache/pig/tools/timer/ incubator/pig/trunk/src/org/apache/pig/tools/timer/PerformanceTimer.java incubator/pig/trunk/src/org/apache/pig/tools/timer/PerformanceTimerFactory.java incubator/pig/trunk/test/ incubator/pig/trunk/test/com/ incubator/pig/trunk/test/com/yahoo/ incubator/pig/trunk/test/com/yahoo/pig/ incubator/pig/trunk/test/org/ incubator/pig/trunk/test/org/apache/ incubator/pig/trunk/test/org/apache/pig/ incubator/pig/trunk/test/org/apache/pig/test/ incubator/pig/trunk/test/org/apache/pig/test/FakeFSInputStream.java incubator/pig/trunk/test/org/apache/pig/test/FakeFSOutputStream.java incubator/pig/trunk/test/org/apache/pig/test/TestAlgebraicEval.java incubator/pig/trunk/test/org/apache/pig/test/TestBuiltin.java incubator/pig/trunk/test/org/apache/pig/test/TestCmdLineParser.java incubator/pig/trunk/test/org/apache/pig/test/TestCompressedFiles.java incubator/pig/trunk/test/org/apache/pig/test/TestDataModel.java incubator/pig/trunk/test/org/apache/pig/test/TestEvalPipeline.java incubator/pig/trunk/test/org/apache/pig/test/TestFilterOpNumeric.java incubator/pig/trunk/test/org/apache/pig/test/TestFilterOpString.java incubator/pig/trunk/test/org/apache/pig/test/TestInfixArithmetic.java incubator/pig/trunk/test/org/apache/pig/test/TestLargeFile.java incubator/pig/trunk/test/org/apache/pig/test/TestLogicalPlanBuilder.java incubator/pig/trunk/test/org/apache/pig/test/TestMapReduce.java incubator/pig/trunk/test/org/apache/pig/test/TestPi.java incubator/pig/trunk/test/org/apache/pig/test/TestPigFile.java incubator/pig/trunk/test/org/apache/pig/test/TestPigSplit.java incubator/pig/trunk/test/org/apache/pig/test/TestStore.java incubator/pig/trunk/test/org/apache/pig/test/Util.java incubator/pig/trunk/test/reports/ incubator/pig/trunk/test/reports/TEST-org.apache.pig.test.TestAlgebraicEval.txt incubator/pig/trunk/test/reports/TEST-org.apache.pig.test.TestBuiltin.txt incubator/pig/trunk/test/reports/TEST-org.apache.pig.test.TestCmdLineParser.txt incubator/pig/trunk/test/reports/TEST-org.apache.pig.test.TestCompressedFiles.txt incubator/pig/trunk/test/reports/TEST-org.apache.pig.test.TestDataModel.txt incubator/pig/trunk/test/reports/TEST-org.apache.pig.test.TestEvalPipeline.txt Added: incubator/pig/trunk/build.xml URL: http://svn.apache.org/viewvc/incubator/pig/trunk/build.xml?rev=589866&view=auto ============================================================================== --- incubator/pig/trunk/build.xml (added) +++ incubator/pig/trunk/build.xml Mon Oct 29 14:44:47 2007 @@ -0,0 +1,129 @@ +<project name="Pig" default="jar"> + + <!-- properties --> + <property name="dist.dir" value="${basedir}/dist" /> + <property name="lib.dir" value="${basedir}/lib" /> + <property name="src.dir" value="${basedir}/src" /> + <property name="doc.dir" value="${basedir}/doc" /> + <property name="shock.src.dir" value="${basedir}/lib-src/shock" /> + <property name="bzip2.src.dir" value="${basedir}/lib-src/bzip2" /> + <property name="test.src.dir" value="${basedir}/test" /> + <property name="output.jarfile" value="pig.jar" /> + <property name="hadoop.jarfile" value="hadoop14.jar"/> + <property name="ssh.gateway" value=""/> + <property name="hod.server" value=""/> + <property name="hod.command" value=""/> + <property name="hod.param" value=""/> + <property name="test.junit.output.format" value="plain"/> + <property name="junit.hadoop.conf" value=""/> + + <!-- setup the classpath --> + <path id="classpath"> + <fileset file="${lib.dir}/${hadoop.jarfile}" /> + <fileset dir="${lib.dir}" /> + </path> + + <target name="clean"> + <delete dir="${dist.dir}" /> + <delete dir="${doc.dir}" /> + <delete file="${output.jarfile}" /> + <delete file="${src.dir}/org/apache/pig/impl/logicalLayer/parser/QueryParser.java" /> + <delete file="${src.dir}/org/apache/pig/impl/logicalLayer/parser/JJTQueryParserState.java" /> + <delete file="${src.dir}/org/apache/pig/impl/logicalLayer/parser/Node.java" /> + <delete file="${src.dir}/org/apache/pig/impl/logicalLayer/parser/ParseException.java" /> + <delete file="${src.dir}/org/apache/pig/impl/logicalLayer/parser/QueryParser.jj" /> + <delete file="${src.dir}/org/apache/pig/impl/logicalLayer/parser/QueryParserConstants.java" /> + <delete file="${src.dir}/org/apache/pig/impl/logicalLayer/parser/QueryParserTokenManager.java" /> + <delete file="${src.dir}/org/apache/pig/impl/logicalLayer/parser/QueryParserTreeConstants.java" /> + <delete file="${src.dir}/org/apache/pig/impl/logicalLayer/parser/SimpleCharStream.java" /> + <delete file="${src.dir}/org/apache/pig/impl/logicalLayer/parser/SimpleNode.java" /> + <delete file="${src.dir}/org/apache/pig/impl/logicalLayer/parser/Token.java" /> + <delete file="${src.dir}/org/apache/pig/impl/logicalLayer/parser/TokenMgrError.java" /> + <delete file="${src.dir}/org/apache/pig/tools/grunt/GruntParser.java" /> + <delete file="${src.dir}/org/apache/pig/tools/grunt/GruntParserConstants.java" /> + <delete file="${src.dir}/org/apache/pig/tools/grunt/GruntParserTokenManager.java" /> + <delete file="${src.dir}/org/apache/pig/tools/grunt/ParseException.java" /> + <delete file="${src.dir}/org/apache/pig/tools/grunt/SimpleCharStream.java" /> + <delete file="${src.dir}/org/apache/pig/tools/grunt/Token.java" /> + <delete file="${src.dir}/org/apache/pig/tools/grunt/TokenMgrError.java" /> + </target> + + <target name="compile" depends="cc-compile"> + <mkdir dir="${dist.dir}" /> + <echo>*** Building Main Sources ***</echo> + <javac srcdir="${src.dir};${shock.src.dir};${bzip2.src.dir}" destdir="${dist.dir}" target="1.5" debug="on"> + <classpath refid="classpath" /> + </javac> + <echo>*** Building Test Sources ***</echo> + <javac srcdir="test" destdir="${dist.dir}" debug="on"> + <classpath refid="classpath" /> + </javac> + </target> + + <target name="cc-compile"> + <jjtree + target="${src.dir}/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt" + outputdirectory="${src.dir}/org/apache/pig/impl/logicalLayer/parser" + javacchome="${basedir}/lib" /> + <javacc + target="${src.dir}/org/apache/pig/impl/logicalLayer/parser/QueryParser.jj" + outputdirectory="${src.dir}/org/apache/pig/impl/logicalLayer/parser" + javacchome="${basedir}/lib" /> + <javacc + target="${src.dir}/org/apache/pig/tools/grunt/GruntParser.jj" + outputdirectory="${src.dir}/org/apache/pig/tools/grunt" + javacchome="${basedir}/lib" /> + </target> + + <target name="jar" depends="compile"> + <jar duplicate="preserve" jarfile="${output.jarfile}" basedir="${dist.dir}"> + <manifest> + <attribute name="Main-Class" value="org.apache.pig.Main"/> + </manifest> + <zipfileset src="${lib.dir}/junit-4.1.jar"/> + <zipfileset src="${lib.dir}/${hadoop.jarfile}"/> + <zipfileset src="${lib.dir}/jsch-0.1.33.jar"/> + </jar> + </target> + + <target name="test"> + <mkdir dir="test/reports"/> + <junit printsummary="on" haltonfailure="no" + errorProperty="tests.failed" failureProperty="tests.failed"> + <sysproperty key="ssh.gateway" value="${ssh.gateway}"/> + <sysproperty key="hod.server" value="${hod.server}"/> + <!-- <sysproperty key="hod.command" value="${hod.command}"/> + <sysproperty key="hod.param" value="${hod.param}"/> --> + <classpath> + <pathelement location="${output.jarfile}"/> + <pathelement location="${junit.hadoop.conf}"/> + <fileset dir="${lib.dir}"> + <include name="**/*.jar" /> + <exclude name="hadoop*.jar" /> + <include name="${hadoop.jarfile}" /> + </fileset> + </classpath> + <formatter type="${test.junit.output.format}" /> + <batchtest fork="yes" todir="test/reports"> + <fileset dir="test"> + <include name="**/*Test*.java" /> + <exclude name="**/TestLargeFile.java" /> + <exclude name="**/nightly/**" /> + </fileset> + </batchtest> + </junit> + <fail if="tests.failed">Tests failed!</fail> + </target> + + <target name="doc"> + <mkdir dir="doc"/> + <echo>*** Building Java Docs ***</echo> + <javadoc packagenames="org.apache.pig.*" sourcepath="${src.dir}" destdir="${doc.dir}"> + <classpath> + <path refid="classpath"/> + <pathelement path="${java.class.path}"/> + </classpath> + </javadoc> + </target> + +</project> Added: incubator/pig/trunk/lib-src/bzip2/org/apache/tools/bzip2r/BZip2Constants.java URL: http://svn.apache.org/viewvc/incubator/pig/trunk/lib-src/bzip2/org/apache/tools/bzip2r/BZip2Constants.java?rev=589866&view=auto ============================================================================== --- incubator/pig/trunk/lib-src/bzip2/org/apache/tools/bzip2r/BZip2Constants.java (added) +++ incubator/pig/trunk/lib-src/bzip2/org/apache/tools/bzip2r/BZip2Constants.java Mon Oct 29 14:44:47 2007 @@ -0,0 +1,136 @@ +/* + * The Apache Software License, Version 1.1 + * + * Copyright (c) 2001 The Apache Software Foundation. All rights + * reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * 3. The end-user documentation included with the redistribution, if + * any, must include the following acknowlegement: + * "This product includes software developed by the + * Apache Software Foundation (http://www.apache.org/)." + * Alternately, this acknowlegement may appear in the software itself, + * if and wherever such third-party acknowlegements normally appear. + * + * 4. The names "Ant" and "Apache Software + * Foundation" must not be used to endorse or promote products derived + * from this software without prior written permission. For written + * permission, please contact [EMAIL PROTECTED] + * + * 5. Products derived from this software may not be called "Apache" + * nor may "Apache" appear in their names without prior written + * permission of the Apache Group. + * + * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR + * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF + * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * ==================================================================== + * + * This software consists of voluntary contributions made by many + * individuals on behalf of the Apache Software Foundation. For more + * information on the Apache Software Foundation, please see + * <http://www.apache.org/>. + */ + +/* + * This package is based on the work done by Keiron Liddle, Aftex Software + * <[EMAIL PROTECTED]> to whom the Ant project is very grateful for his + * great code. + */ + +package org.apache.tools.bzip2r; + +/** + * Base class for both the compress and decompress classes. + * Holds common arrays, and static data. + * + * @author <a href="mailto:[EMAIL PROTECTED]">Keiron Liddle</a> + */ +public interface BZip2Constants { + + int baseBlockSize = 100000; + int MAX_ALPHA_SIZE = 258; + int MAX_CODE_LEN = 23; + int RUNA = 0; + int RUNB = 1; + int N_GROUPS = 6; + int G_SIZE = 50; + int N_ITERS = 4; + int MAX_SELECTORS = (2 + (900000 / G_SIZE)); + int NUM_OVERSHOOT_BYTES = 20; + + int[] rNums = { + 619, 720, 127, 481, 931, 816, 813, 233, 566, 247, + 985, 724, 205, 454, 863, 491, 741, 242, 949, 214, + 733, 859, 335, 708, 621, 574, 73, 654, 730, 472, + 419, 436, 278, 496, 867, 210, 399, 680, 480, 51, + 878, 465, 811, 169, 869, 675, 611, 697, 867, 561, + 862, 687, 507, 283, 482, 129, 807, 591, 733, 623, + 150, 238, 59, 379, 684, 877, 625, 169, 643, 105, + 170, 607, 520, 932, 727, 476, 693, 425, 174, 647, + 73, 122, 335, 530, 442, 853, 695, 249, 445, 515, + 909, 545, 703, 919, 874, 474, 882, 500, 594, 612, + 641, 801, 220, 162, 819, 984, 589, 513, 495, 799, + 161, 604, 958, 533, 221, 400, 386, 867, 600, 782, + 382, 596, 414, 171, 516, 375, 682, 485, 911, 276, + 98, 553, 163, 354, 666, 933, 424, 341, 533, 870, + 227, 730, 475, 186, 263, 647, 537, 686, 600, 224, + 469, 68, 770, 919, 190, 373, 294, 822, 808, 206, + 184, 943, 795, 384, 383, 461, 404, 758, 839, 887, + 715, 67, 618, 276, 204, 918, 873, 777, 604, 560, + 951, 160, 578, 722, 79, 804, 96, 409, 713, 940, + 652, 934, 970, 447, 318, 353, 859, 672, 112, 785, + 645, 863, 803, 350, 139, 93, 354, 99, 820, 908, + 609, 772, 154, 274, 580, 184, 79, 626, 630, 742, + 653, 282, 762, 623, 680, 81, 927, 626, 789, 125, + 411, 521, 938, 300, 821, 78, 343, 175, 128, 250, + 170, 774, 972, 275, 999, 639, 495, 78, 352, 126, + 857, 956, 358, 619, 580, 124, 737, 594, 701, 612, + 669, 112, 134, 694, 363, 992, 809, 743, 168, 974, + 944, 375, 748, 52, 600, 747, 642, 182, 862, 81, + 344, 805, 988, 739, 511, 655, 814, 334, 249, 515, + 897, 955, 664, 981, 649, 113, 974, 459, 893, 228, + 433, 837, 553, 268, 926, 240, 102, 654, 459, 51, + 686, 754, 806, 760, 493, 403, 415, 394, 687, 700, + 946, 670, 656, 610, 738, 392, 760, 799, 887, 653, + 978, 321, 576, 617, 626, 502, 894, 679, 243, 440, + 680, 879, 194, 572, 640, 724, 926, 56, 204, 700, + 707, 151, 457, 449, 797, 195, 791, 558, 945, 679, + 297, 59, 87, 824, 713, 663, 412, 693, 342, 606, + 134, 108, 571, 364, 631, 212, 174, 643, 304, 329, + 343, 97, 430, 751, 497, 314, 983, 374, 822, 928, + 140, 206, 73, 263, 980, 736, 876, 478, 430, 305, + 170, 514, 364, 692, 829, 82, 855, 953, 676, 246, + 369, 970, 294, 750, 807, 827, 150, 790, 288, 923, + 804, 378, 215, 828, 592, 281, 565, 555, 710, 82, + 896, 831, 547, 261, 524, 462, 293, 465, 502, 56, + 661, 821, 976, 991, 658, 869, 905, 758, 745, 193, + 768, 550, 608, 933, 378, 286, 215, 979, 792, 961, + 61, 688, 793, 644, 986, 403, 106, 366, 905, 644, + 372, 567, 466, 434, 645, 210, 389, 550, 919, 135, + 780, 773, 635, 389, 707, 100, 626, 958, 165, 504, + 920, 176, 193, 713, 857, 265, 203, 50, 668, 108, + 645, 990, 626, 197, 510, 357, 358, 850, 858, 364, + 936, 638 + }; +} Added: incubator/pig/trunk/lib-src/bzip2/org/apache/tools/bzip2r/CBZip2InputStream.java URL: http://svn.apache.org/viewvc/incubator/pig/trunk/lib-src/bzip2/org/apache/tools/bzip2r/CBZip2InputStream.java?rev=589866&view=auto ============================================================================== --- incubator/pig/trunk/lib-src/bzip2/org/apache/tools/bzip2r/CBZip2InputStream.java (added) +++ incubator/pig/trunk/lib-src/bzip2/org/apache/tools/bzip2r/CBZip2InputStream.java Mon Oct 29 14:44:47 2007 @@ -0,0 +1,931 @@ +/* + * The Apache Software License, Version 1.1 + * + * Copyright (c) 2001-2003 The Apache Software Foundation. All rights + * reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * 3. The end-user documentation included with the redistribution, if + * any, must include the following acknowlegement: + * "This product includes software developed by the + * Apache Software Foundation (http://www.apache.org/)." + * Alternately, this acknowlegement may appear in the software itself, + * if and wherever such third-party acknowlegements normally appear. + * + * 4. The names "Ant" and "Apache Software + * Foundation" must not be used to endorse or promote products derived + * from this software without prior written permission. For written + * permission, please contact [EMAIL PROTECTED] + * + * 5. Products derived from this software may not be called "Apache" + * nor may "Apache" appear in their names without prior written + * permission of the Apache Group. + * + * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR + * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF + * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * ==================================================================== + * + * This software consists of voluntary contributions made by many + * individuals on behalf of the Apache Software Foundation. For more + * information on the Apache Software Foundation, please see + * <http://www.apache.org/>. + */ + +/* + * This package is based on the work done by Keiron Liddle, Aftex Software + * <[EMAIL PROTECTED]> to whom the Ant project is very grateful for his + * great code. + */ +package org.apache.tools.bzip2r; + +import java.io.BufferedInputStream; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.InputStream; +import java.io.IOException; +import java.util.Random; + +import org.apache.hadoop.fs.FSDataInputStream; + +/** + * An input stream that decompresses from the BZip2 format (without the file + * header chars) to be read as any other stream. + * + * @author <a href="mailto:[EMAIL PROTECTED]">Keiron Liddle</a> + */ +public class CBZip2InputStream extends InputStream implements BZip2Constants { + private static void cadvise(String reason) throws IOException { + throw new IOException(reason); + } + + private static void compressedStreamEOF() throws IOException { + cadvise("compressedStream EOF"); + } + + private void makeMaps() { + int i; + nInUse = 0; + for (i = 0; i < 256; i++) { + if (inUse[i]) { + seqToUnseq[nInUse] = (char) i; + unseqToSeq[i] = (char) nInUse; + nInUse++; + } + } + } + + /* + index of the last char in the block, so + the block size == last + 1. + */ + private int last; + + /* + index in zptr[] of original string after sorting. + */ + private int origPtr; + + /* + always: in the range 0 .. 9. + The current block size is 100000 * this number. + */ + private int blockSize100k; + + private boolean blockRandomised; + + private int bsBuff; + private int bsLive; + private CRC mCrc = new CRC(); + + private boolean[] inUse = new boolean[256]; + private int nInUse; + + private char[] seqToUnseq = new char[256]; + private char[] unseqToSeq = new char[256]; + + private char[] selector = new char[MAX_SELECTORS]; + private char[] selectorMtf = new char[MAX_SELECTORS]; + + private int[] tt; + private char[] ll8; + + /* + freq table collected to save a pass over the data + during decompression. + */ + private int[] unzftab = new int[256]; + + private int[][] limit = new int[N_GROUPS][MAX_ALPHA_SIZE]; + private int[][] base = new int[N_GROUPS][MAX_ALPHA_SIZE]; + private int[][] perm = new int[N_GROUPS][MAX_ALPHA_SIZE]; + private int[] minLens = new int[N_GROUPS]; + + private FSDataInputStream innerBsStream; + long readLimit = Long.MAX_VALUE; + public long getReadLimit() { + return readLimit; + } + public void setReadLimit(long readLimit) { + this.readLimit = readLimit; + } + long readCount; + public long getReadCount() { + return readCount; + } + + private boolean streamEnd = false; + + private int currentChar = -1; + + private static final int START_BLOCK_STATE = 1; + private static final int RAND_PART_A_STATE = 2; + private static final int RAND_PART_B_STATE = 3; + private static final int RAND_PART_C_STATE = 4; + private static final int NO_RAND_PART_A_STATE = 5; + private static final int NO_RAND_PART_B_STATE = 6; + private static final int NO_RAND_PART_C_STATE = 7; + + private int currentState = START_BLOCK_STATE; + + private int storedBlockCRC, storedCombinedCRC; + private int computedBlockCRC, computedCombinedCRC; + private boolean checkComputedCombinedCRC = true; + + int i2, count, chPrev, ch2; + int i, tPos; + int rNToGo = 0; + int rTPos = 0; + int j2; + char z; + + private long retPos, oldPos; + + public CBZip2InputStream(FSDataInputStream zStream, int blockSize) throws IOException { + retPos = oldPos = zStream.getPos(); + ll8 = null; + tt = null; + checkComputedCombinedCRC = blockSize == -1; + bsSetStream(zStream); + initialize(blockSize); + initBlock(blockSize != -1); + setupBlock(); + } + + public CBZip2InputStream(FSDataInputStream zStream) throws IOException { + this(zStream, -1); + } + + public int read() throws IOException { + if (streamEnd) { + return -1; + } else { + int retChar = currentChar; + switch(currentState) { + case START_BLOCK_STATE: + break; + case RAND_PART_A_STATE: + break; + case RAND_PART_B_STATE: + setupRandPartB(); + break; + case RAND_PART_C_STATE: + setupRandPartC(); + break; + case NO_RAND_PART_A_STATE: + break; + case NO_RAND_PART_B_STATE: + setupNoRandPartB(); + break; + case NO_RAND_PART_C_STATE: + setupNoRandPartC(); + break; + default: + break; + } + return retChar; + } + } + + public long getPos() throws IOException{ + if (innerBsStream == null) + return retPos; + long newPos = innerBsStream.getPos(); + + if (newPos != oldPos){ + retPos = oldPos; + oldPos = newPos; + } + return retPos; + } + + private void initialize(int blockSize) throws IOException { + if (blockSize == -1) { + char magic1, magic2; + char magic3, magic4; + magic1 = bsGetUChar(); + magic2 = bsGetUChar(); + magic3 = bsGetUChar(); + magic4 = bsGetUChar(); + if (magic1 != 'B' || magic2 != 'Z' || + magic3 != 'h' || magic4 < '1' || magic4 > '9') { + bsFinishedWithStream(); + streamEnd = true; + return; + } + blockSize = magic4 - '0'; + } + + setDecompressStructureSizes(blockSize); + computedCombinedCRC = 0; + } + + private final static long mask = 0x1ffffffffffL; + private final static long eob = 0x314159265359L & mask; + private final static long eos = 0x177245385090L & mask; + + private void initBlock(boolean searchForMagic) throws IOException { + if (readCount >= readLimit) { + bsFinishedWithStream(); + streamEnd = true; + return; + } + + if (!searchForMagic) { + char magic1, magic2, magic3, magic4; + char magic5, magic6; + magic1 = bsGetUChar(); + magic2 = bsGetUChar(); + magic3 = bsGetUChar(); + magic4 = bsGetUChar(); + magic5 = bsGetUChar(); + magic6 = bsGetUChar(); + if (magic1 == 0x17 && magic2 == 0x72 && magic3 == 0x45 + && magic4 == 0x38 && magic5 == 0x50 && magic6 == 0x90) { + complete(); + return; + } + + if (magic1 != 0x31 || magic2 != 0x41 || magic3 != 0x59 + || magic4 != 0x26 || magic5 != 0x53 || magic6 != 0x59) { + badBlockHeader(); + streamEnd = true; + return; + } + } else { + long magic = bsR(41); + while(magic != eos && magic != eob) { + magic <<= 1; + magic &= mask; + magic |= bsR(1); + } + if (magic == eos) { + complete(); + return; + } + } + + storedBlockCRC = bsGetInt32(); + + if (bsR(1) == 1) { + blockRandomised = true; + } else { + blockRandomised = false; + } + + // currBlockNo++; + getAndMoveToFrontDecode(); + + mCrc.initialiseCRC(); + currentState = START_BLOCK_STATE; + } + + private void endBlock() throws IOException { + computedBlockCRC = mCrc.getFinalCRC(); + /* A bad CRC is considered a fatal error. */ + if (storedBlockCRC != computedBlockCRC) { + crcError(); + } + + computedCombinedCRC = (computedCombinedCRC << 1) + | (computedCombinedCRC >>> 31); + computedCombinedCRC ^= computedBlockCRC; + } + + private void complete() throws IOException { + storedCombinedCRC = bsGetInt32(); + if (checkComputedCombinedCRC && + storedCombinedCRC != computedCombinedCRC) { + crcError(); + } + + bsFinishedWithStream(); + streamEnd = true; + } + + private static void blockOverrun() throws IOException { + cadvise("block overrun"); + } + + private static void badBlockHeader() throws IOException { + cadvise("bad block header"); + } + + private static void crcError() throws IOException { + cadvise("CRC error"); + } + + private void bsFinishedWithStream() { + if (this.innerBsStream != null) { + if (this.innerBsStream != System.in) { + this.innerBsStream = null; + } + } + } + + private void bsSetStream(FSDataInputStream f) { + innerBsStream = f; + bsLive = 0; + bsBuff = 0; + } + + final private int readBs() throws IOException { + readCount++; + return innerBsStream.read(); + } + private int bsR(int n) throws IOException { + int v; + while (bsLive < n) { + int zzi; + char thech = 0; + thech = (char) readBs(); + if (thech == -1) { + compressedStreamEOF(); + } + zzi = thech; + bsBuff = (bsBuff << 8) | (zzi & 0xff); + bsLive += 8; + } + + v = (bsBuff >> (bsLive - n)) & ((1 << n) - 1); + bsLive -= n; + return v; + } + + + private char bsGetUChar() throws IOException { + return (char) bsR(8); + } + + private int bsGetint() throws IOException { + int u = 0; + u = (u << 8) | bsR(8); + u = (u << 8) | bsR(8); + u = (u << 8) | bsR(8); + u = (u << 8) | bsR(8); + return u; + } + + private int bsGetIntVS(int numBits) throws IOException { + return (int) bsR(numBits); + } + + private int bsGetInt32() throws IOException { + return (int) bsGetint(); + } + + private void hbCreateDecodeTables(int[] limit, int[] base, + int[] perm, char[] length, + int minLen, int maxLen, int alphaSize) { + int pp, i, j, vec; + + pp = 0; + for (i = minLen; i <= maxLen; i++) { + for (j = 0; j < alphaSize; j++) { + if (length[j] == i) { + perm[pp] = j; + pp++; + } + } + } + + for (i = 0; i < MAX_CODE_LEN; i++) { + base[i] = 0; + } + for (i = 0; i < alphaSize; i++) { + base[length[i] + 1]++; + } + + for (i = 1; i < MAX_CODE_LEN; i++) { + base[i] += base[i - 1]; + } + + for (i = 0; i < MAX_CODE_LEN; i++) { + limit[i] = 0; + } + vec = 0; + + for (i = minLen; i <= maxLen; i++) { + vec += (base[i + 1] - base[i]); + limit[i] = vec - 1; + vec <<= 1; + } + for (i = minLen + 1; i <= maxLen; i++) { + base[i] = ((limit[i - 1] + 1) << 1) - base[i]; + } + } + + private void recvDecodingTables() throws IOException { + char len[][] = new char[N_GROUPS][MAX_ALPHA_SIZE]; + int i, j, t, nGroups, nSelectors, alphaSize; + int minLen, maxLen; + boolean[] inUse16 = new boolean[16]; + + /* Receive the mapping table */ + for (i = 0; i < 16; i++) { + if (bsR(1) == 1) { + inUse16[i] = true; + } else { + inUse16[i] = false; + } + } + + for (i = 0; i < 256; i++) { + inUse[i] = false; + } + + for (i = 0; i < 16; i++) { + if (inUse16[i]) { + for (j = 0; j < 16; j++) { + if (bsR(1) == 1) { + inUse[i * 16 + j] = true; + } + } + } + } + + makeMaps(); + alphaSize = nInUse + 2; + + /* Now the selectors */ + nGroups = bsR(3); + nSelectors = bsR(15); + for (i = 0; i < nSelectors; i++) { + j = 0; + while (bsR(1) == 1) { + j++; + } + selectorMtf[i] = (char) j; + } + + /* Undo the MTF values for the selectors. */ + { + char[] pos = new char[N_GROUPS]; + char tmp, v; + for (v = 0; v < nGroups; v++) { + pos[v] = v; + } + + for (i = 0; i < nSelectors; i++) { + v = selectorMtf[i]; + tmp = pos[v]; + while (v > 0) { + pos[v] = pos[v - 1]; + v--; + } + pos[0] = tmp; + selector[i] = tmp; + } + } + + /* Now the coding tables */ + for (t = 0; t < nGroups; t++) { + int curr = bsR(5); + for (i = 0; i < alphaSize; i++) { + while (bsR(1) == 1) { + if (bsR(1) == 0) { + curr++; + } else { + curr--; + } + } + len[t][i] = (char) curr; + } + } + + /* Create the Huffman decoding tables */ + for (t = 0; t < nGroups; t++) { + minLen = 32; + maxLen = 0; + for (i = 0; i < alphaSize; i++) { + if (len[t][i] > maxLen) { + maxLen = len[t][i]; + } + if (len[t][i] < minLen) { + minLen = len[t][i]; + } + } + hbCreateDecodeTables(limit[t], base[t], perm[t], len[t], minLen, + maxLen, alphaSize); + minLens[t] = minLen; + } + } + + private void getAndMoveToFrontDecode() throws IOException { + char[] yy = new char[256]; + int i, j, nextSym, limitLast; + int EOB, groupNo, groupPos; + + limitLast = baseBlockSize * blockSize100k; + origPtr = bsGetIntVS(24); + + recvDecodingTables(); + EOB = nInUse + 1; + groupNo = -1; + groupPos = 0; + + /* + Setting up the unzftab entries here is not strictly + necessary, but it does save having to do it later + in a separate pass, and so saves a block's worth of + cache misses. + */ + for (i = 0; i <= 255; i++) { + unzftab[i] = 0; + } + + for (i = 0; i <= 255; i++) { + yy[i] = (char) i; + } + + last = -1; + + { + int zt, zn, zvec, zj; + if (groupPos == 0) { + groupNo++; + groupPos = G_SIZE; + } + groupPos--; + zt = selector[groupNo]; + zn = minLens[zt]; + zvec = bsR(zn); + while (zvec > limit[zt][zn]) { + zn++; + { + { + while (bsLive < 1) { + int zzi; + char thech = 0; + try { + thech = (char) readBs(); + } catch (IOException e) { + compressedStreamEOF(); + } + if (thech == -1) { + compressedStreamEOF(); + } + zzi = thech; + bsBuff = (bsBuff << 8) | (zzi & 0xff); + bsLive += 8; + } + } + zj = (bsBuff >> (bsLive - 1)) & 1; + bsLive--; + } + zvec = (zvec << 1) | zj; + } + nextSym = perm[zt][zvec - base[zt][zn]]; + } + + while (true) { + + if (nextSym == EOB) { + break; + } + + if (nextSym == RUNA || nextSym == RUNB) { + char ch; + int s = -1; + int N = 1; + do { + if (nextSym == RUNA) { + s = s + (0 + 1) * N; + } else if (nextSym == RUNB) { + s = s + (1 + 1) * N; + } + N = N * 2; + { + int zt, zn, zvec, zj; + if (groupPos == 0) { + groupNo++; + groupPos = G_SIZE; + } + groupPos--; + zt = selector[groupNo]; + zn = minLens[zt]; + zvec = bsR(zn); + while (zvec > limit[zt][zn]) { + zn++; + { + { + while (bsLive < 1) { + int zzi; + char thech = 0; + try { + thech = (char) readBs(); + } catch (IOException e) { + compressedStreamEOF(); + } + if (thech == -1) { + compressedStreamEOF(); + } + zzi = thech; + bsBuff = (bsBuff << 8) | (zzi & 0xff); + bsLive += 8; + } + } + zj = (bsBuff >> (bsLive - 1)) & 1; + bsLive--; + } + zvec = (zvec << 1) | zj; + } + nextSym = perm[zt][zvec - base[zt][zn]]; + } + } while (nextSym == RUNA || nextSym == RUNB); + + s++; + ch = seqToUnseq[yy[0]]; + unzftab[ch] += s; + + while (s > 0) { + last++; + ll8[last] = ch; + s--; + } + + if (last >= limitLast) { + blockOverrun(); + } + continue; + } else { + char tmp; + last++; + if (last >= limitLast) { + blockOverrun(); + } + + tmp = yy[nextSym - 1]; + unzftab[seqToUnseq[tmp]]++; + ll8[last] = seqToUnseq[tmp]; + + /* + This loop is hammered during decompression, + hence the unrolling. + + for (j = nextSym-1; j > 0; j--) yy[j] = yy[j-1]; + */ + + j = nextSym - 1; + for (; j > 3; j -= 4) { + yy[j] = yy[j - 1]; + yy[j - 1] = yy[j - 2]; + yy[j - 2] = yy[j - 3]; + yy[j - 3] = yy[j - 4]; + } + for (; j > 0; j--) { + yy[j] = yy[j - 1]; + } + + yy[0] = tmp; + { + int zt, zn, zvec, zj; + if (groupPos == 0) { + groupNo++; + groupPos = G_SIZE; + } + groupPos--; + zt = selector[groupNo]; + zn = minLens[zt]; + zvec = bsR(zn); + while (zvec > limit[zt][zn]) { + zn++; + { + { + while (bsLive < 1) { + int zzi; + char thech = 0; + try { + thech = (char) readBs(); + } catch (IOException e) { + compressedStreamEOF(); + } + zzi = thech; + bsBuff = (bsBuff << 8) | (zzi & 0xff); + bsLive += 8; + } + } + zj = (bsBuff >> (bsLive - 1)) & 1; + bsLive--; + } + zvec = (zvec << 1) | zj; + } + nextSym = perm[zt][zvec - base[zt][zn]]; + } + continue; + } + } + } + + private void setupBlock() throws IOException { + int[] cftab = new int[257]; + char ch; + + cftab[0] = 0; + for (i = 1; i <= 256; i++) { + cftab[i] = unzftab[i - 1]; + } + for (i = 1; i <= 256; i++) { + cftab[i] += cftab[i - 1]; + } + + for (i = 0; i <= last; i++) { + ch = (char) ll8[i]; + tt[cftab[ch]] = i; + cftab[ch]++; + } + cftab = null; + + tPos = tt[origPtr]; + + count = 0; + i2 = 0; + ch2 = 256; /* not a char and not EOF */ + + if (blockRandomised) { + rNToGo = 0; + rTPos = 0; + setupRandPartA(); + } else { + setupNoRandPartA(); + } + } + + private void setupRandPartA() throws IOException { + if (i2 <= last) { + chPrev = ch2; + ch2 = ll8[tPos]; + tPos = tt[tPos]; + if (rNToGo == 0) { + rNToGo = rNums[rTPos]; + rTPos++; + if (rTPos == 512) { + rTPos = 0; + } + } + rNToGo--; + ch2 ^= (int) ((rNToGo == 1) ? 1 : 0); + i2++; + + currentChar = ch2; + currentState = RAND_PART_B_STATE; + mCrc.updateCRC(ch2); + } else { + endBlock(); + initBlock(false); + setupBlock(); + } + } + + private void setupNoRandPartA() throws IOException { + if (i2 <= last) { + chPrev = ch2; + ch2 = ll8[tPos]; + tPos = tt[tPos]; + i2++; + + currentChar = ch2; + currentState = NO_RAND_PART_B_STATE; + mCrc.updateCRC(ch2); + } else { + endBlock(); + initBlock(false); + setupBlock(); + } + } + + private void setupRandPartB() throws IOException { + if (ch2 != chPrev) { + currentState = RAND_PART_A_STATE; + count = 1; + setupRandPartA(); + } else { + count++; + if (count >= 4) { + z = ll8[tPos]; + tPos = tt[tPos]; + if (rNToGo == 0) { + rNToGo = rNums[rTPos]; + rTPos++; + if (rTPos == 512) { + rTPos = 0; + } + } + rNToGo--; + z ^= ((rNToGo == 1) ? 1 : 0); + j2 = 0; + currentState = RAND_PART_C_STATE; + setupRandPartC(); + } else { + currentState = RAND_PART_A_STATE; + setupRandPartA(); + } + } + } + + private void setupRandPartC() throws IOException { + if (j2 < (int) z) { + currentChar = ch2; + mCrc.updateCRC(ch2); + j2++; + } else { + currentState = RAND_PART_A_STATE; + i2++; + count = 0; + setupRandPartA(); + } + } + + private void setupNoRandPartB() throws IOException { + if (ch2 != chPrev) { + currentState = NO_RAND_PART_A_STATE; + count = 1; + setupNoRandPartA(); + } else { + count++; + if (count >= 4) { + z = ll8[tPos]; + tPos = tt[tPos]; + currentState = NO_RAND_PART_C_STATE; + j2 = 0; + setupNoRandPartC(); + } else { + currentState = NO_RAND_PART_A_STATE; + setupNoRandPartA(); + } + } + } + + private void setupNoRandPartC() throws IOException { + if (j2 < (int) z) { + currentChar = ch2; + mCrc.updateCRC(ch2); + j2++; + } else { + currentState = NO_RAND_PART_A_STATE; + i2++; + count = 0; + setupNoRandPartA(); + } + } + + private void setDecompressStructureSizes(int newSize100k) { + if (!(0 <= newSize100k && newSize100k <= 9 && 0 <= blockSize100k + && blockSize100k <= 9)) { + // throw new IOException("Invalid block size"); + } + + blockSize100k = newSize100k; + + if (newSize100k == 0) { + return; + } + + int n = baseBlockSize * newSize100k; + ll8 = new char[n]; + tt = new int[n]; + } + +} +