svn commit: r959865 - in /hadoop/pig/trunk/contrib: CHANGES.txt piggybank/java/src/main/java/org/apache/pig/piggybank/storage/RegExLoader.java piggybank/java/src/test/java/org/apache/pig/piggybank/tes
Author: hashutosh
Date: Fri Jul 2 06:05:22 2010
New Revision: 959865
URL: http://svn.apache.org/viewvc?rev=959865view=rev
Log:
PIG-1449: RegExLoader hangs on lines that don't match the regular expression
Modified:
hadoop/pig/trunk/contrib/CHANGES.txt
hadoop/pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/RegExLoader.java
hadoop/pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestRegExLoader.java
Modified: hadoop/pig/trunk/contrib/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/CHANGES.txt?rev=959865r1=959864r2=959865view=diff
==
--- hadoop/pig/trunk/contrib/CHANGES.txt (original)
+++ hadoop/pig/trunk/contrib/CHANGES.txt Fri Jul 2 06:05:22 2010
@@ -32,6 +32,8 @@ OPTIMIZATIONS
BUG FIXES
+PIG-1449 RegExLoader hangs on lines that don't match the regular expression
+(Christian Hargraves via hashutosh)
PIG 0.7.0
Modified:
hadoop/pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/RegExLoader.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/RegExLoader.java?rev=959865r1=959864r2=959865view=diff
==
---
hadoop/pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/RegExLoader.java
(original)
+++
hadoop/pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/RegExLoader.java
Fri Jul 2 06:05:22 2010
@@ -48,21 +48,13 @@ public abstract class RegExLoader extend
@Override
public Tuple getNext() throws IOException {
-if (!in.nextKeyValue()) {
- return null;
-}
-
Pattern pattern = getPattern();
Matcher matcher = pattern.matcher();
TupleFactory mTupleFactory = DefaultTupleFactory.getInstance();
String line;
-boolean tryNext = true;
-while (tryNext) {
- Text val = in.getCurrentValue();
- if (val == null) {
-break;
- }
+while (in.nextKeyValue()) {
+ Text val = in.getCurrentValue();
line = val.toString();
if (line.length() 0 line.charAt(line.length() - 1) == '\r') {
line = line.substring(0, line.length() - 1);
@@ -70,14 +62,12 @@ public abstract class RegExLoader extend
matcher = matcher.reset(line);
ArrayListDataByteArray list = new ArrayListDataByteArray();
if (matcher.find()) {
-tryNext=false;
for (int i = 1; i = matcher.groupCount(); i++) {
list.add(new DataByteArray(matcher.group(i)));
}
return mTupleFactory.newTuple(list);
}
}
-
return null;
}
Modified:
hadoop/pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestRegExLoader.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestRegExLoader.java?rev=959865r1=959864r2=959865view=diff
==
---
hadoop/pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestRegExLoader.java
(original)
+++
hadoop/pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestRegExLoader.java
Fri Jul 2 06:05:22 2010
@@ -31,6 +31,8 @@ import org.junit.Test;
public class TestRegExLoader extends TestCase {
private static String patternString = (\\w+),(\\w+);(\\w+);
private final static Pattern pattern = Pattern.compile(patternString);
+private static String patternString2 = (3),(three);(iii);
+private final static Pattern pattern2 = Pattern.compile(patternString);
public static class DummyRegExLoader extends RegExLoader {
public DummyRegExLoader() {}
@@ -41,6 +43,15 @@ public class TestRegExLoader extends Tes
}
}
+public static class DummyRegExLoader2 extends RegExLoader {
+public DummyRegExLoader2() {}
+
+@Override
+public Pattern getPattern() {
+return Pattern.compile(patternString2);
+}
+}
+
public static ArrayListString[] data = new ArrayListString[]();
static {
data.add(new String[] { 1,one;i });
@@ -71,4 +82,30 @@ public class TestRegExLoader extends Tes
assertEquals(data.size(), tupleCount);
}
+@Test
+public void testOnlyLastMatch() throws Exception {
+PigServer pigServer = new PigServer(LOCAL);
+
+String filename = TestHelper.createTempFile(data, );
+
+ ArrayListString[] dataE = new ArrayListString[]();
+dataE.add(new String[] { 3,three;iii });
+ ArrayListDataByteArray[] expected =
TestHelper.getExpected(dataE, pattern2);
+
+pigServer.registerQuery(A = LOAD 'file: +
svn commit: r960062 - in /hadoop/pig/trunk: CHANGES.txt src/org/apache/pig/data/DefaultDataBag.java
Author: dvryaboy
Date: Fri Jul 2 17:16:17 2010
New Revision: 960062
URL: http://svn.apache.org/viewvc?rev=960062view=rev
Log:
PIG-1469: DefaultDataBag assumes ArrayList as default List type (azaroth via
dvryaboy)
Modified:
hadoop/pig/trunk/CHANGES.txt
hadoop/pig/trunk/src/org/apache/pig/data/DefaultDataBag.java
Modified: hadoop/pig/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=960062r1=960061r2=960062view=diff
==
--- hadoop/pig/trunk/CHANGES.txt (original)
+++ hadoop/pig/trunk/CHANGES.txt Fri Jul 2 17:16:17 2010
@@ -95,6 +95,8 @@ PIG-1309: Map-side Cogroup (ashutoshc)
BUG FIXES
+PIG-1469: DefaultDataBag assumes ArrayList as default List type (azaroth via
dvryaboy)
+
PIG-1467: order by fail when set fs.file.impl.disable.cache to true (daijy)
PIG-1463: Replace bz with .bz in setStoreLocation in PigStorage (zjffdu)
Modified: hadoop/pig/trunk/src/org/apache/pig/data/DefaultDataBag.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/data/DefaultDataBag.java?rev=960062r1=960061r2=960062view=diff
==
--- hadoop/pig/trunk/src/org/apache/pig/data/DefaultDataBag.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/data/DefaultDataBag.java Fri Jul 2
17:16:17 2010
@@ -36,7 +36,7 @@ import org.apache.pig.PigWarning;
/**
* An unordered collection of Tuples (possibly) with multiples. The tuples
- * are stored in an ArrayList, since there is no concern for order or
+ * are stored in a List, since there is no concern for order or
* distinctness.
*/
public class DefaultDataBag extends DefaultAbstractBag {
@@ -295,7 +295,7 @@ public class DefaultDataBag extends Defa
if (mContents.size() == 0) return null;
if (mMemoryPtr mContents.size()) {
-return ((ArrayListTuple)mContents).get(mMemoryPtr++);
+return ((ListTuple)mContents).get(mMemoryPtr++);
} else {
return null;
}
[Pig Wiki] Update of 0102 by 0102
Dear Wiki user, You have subscribed to a wiki page or wiki category on Pig Wiki for change notification. The 0102 page has been changed by 0102. http://wiki.apache.org/pig/0102 -- New page: Third, isolation of the local government departments and the administrative utility of the SAR and the constraints of economic development zones. On a regional reform, opening up and economic development constraints, in addition to the central and provincial government departments from the outside, but also from the management of the development of regional government departments. The special zone administrative system advantages: First, improve administrative efficiency. In the administrative examination and approval of a unified administration, prevented a project Touzi, a Qi Ye registration, etc., needed to many a department for approval and the time very long phenomenon. Second, to prevent government departments and utilities on the enterprise administrative fees and fines. Even some zones, the protection of businesses in the region, does not allow government departments and the administrative utilities to the development zone to the charges and fines. This is why the SAR and the operation and development zone enterprises to invest in an important reason for lower cost. Fourth, the structure and experience, including economic development, industrial growth and development zones for non-SAR, as well as the formation of the national pilot, demonstration, diffusion, lead, and other associated effects. From the SAR, to the Free Trade Zone, to economic and technological development zones, from the national economic and technological development zones, to the provincial and municipal economic and technological development zones, the government in a special area and the park systems and policies, and gradually from point to surface, from the coast to the interior, from the central zone to test and promote local level development zones. This pattern of reform and opening up has greatly liberated the productive forces, increasing the spread of industry and association, due to division of labor, industrial extension, production and supporting, etc., plus the logistics distribution, development led the Pearl River Delta, Yangtze River Delta, Bohai Bay economic development, but to the Midwest industrial and transport development. Fan Gang, On the role of the SAR model in the system when that began to reform a big issue is the lack of information, lack of knowledge and, as the reform and opening of the SAR, the responsibility and act as a rapid absorbing introduction of various relations, systems and information an important mechanism. To clarify relations between various systems, to promote the smooth implementation of reforms, which require a region in all aspects of the reform to get this information. For the pilot reform of the country was full of knowledge, information, experiences and lessons learned, and then used to guide the country's reforms, the country to show the way to do model. This is the significance of the special economic zones and important role in the host [9]. In conclusion, Comrade Deng Xiaoping, the region is to land in China to learn the advanced systems and mechanisms, new a new kind of modern enterprises and government institutions; is the use of foreign capital, technology and advanced management, the formation of a new industrial system, to boost the national economy, greatly emancipated productivity. Opening up of the SAR, bonded, large coastal open economic and technological development, and the subsequent opening up of inland areas and border owe a great deal! [http://www.mbt6shoes.com] Wholesale mbt shoes
