Repository: chukwa
Updated Branches:
refs/heads/master a990fc54a -> 93a1f75c4
CHUKWA-784. Improve CharFileTailingAdaptorUTF8NewLineEscaped and LocalWriter
logic to send proper data chunk. (Eric Yang)
Project: http://git-wip-us.apache.org/repos/asf/chukwa/repo
Commit: http://git-wip-us.apache.org/repos/asf/chukwa/commit/93a1f75c
Tree: http://git-wip-us.apache.org/repos/asf/chukwa/tree/93a1f75c
Diff: http://git-wip-us.apache.org/repos/asf/chukwa/diff/93a1f75c
Branch: refs/heads/master
Commit: 93a1f75c45f7333dcba2980f5ec9ebbdc64bb9d3
Parents: a990fc5
Author: Eric Yang <[email protected]>
Authored: Sun Oct 11 12:42:48 2015 -0700
Committer: Eric Yang <[email protected]>
Committed: Sun Oct 11 12:43:31 2015 -0700
----------------------------------------------------------------------
.../writer/parquet/ChukwaAvroSchema.java | 17 +++
.../writer/TestChukwaWriters.java | 122 -------------------
2 files changed, 17 insertions(+), 122 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/chukwa/blob/93a1f75c/src/main/java/org/apache/hadoop/chukwa/datacollection/writer/parquet/ChukwaAvroSchema.java
----------------------------------------------------------------------
diff --git
a/src/main/java/org/apache/hadoop/chukwa/datacollection/writer/parquet/ChukwaAvroSchema.java
b/src/main/java/org/apache/hadoop/chukwa/datacollection/writer/parquet/ChukwaAvroSchema.java
index eaad1bc..cc113d3 100644
---
a/src/main/java/org/apache/hadoop/chukwa/datacollection/writer/parquet/ChukwaAvroSchema.java
+++
b/src/main/java/org/apache/hadoop/chukwa/datacollection/writer/parquet/ChukwaAvroSchema.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.hadoop.chukwa.datacollection.writer.parquet;
import org.apache.avro.Schema;
http://git-wip-us.apache.org/repos/asf/chukwa/blob/93a1f75c/src/test/java/org/apache/hadoop/chukwa/datacollection/writer/TestChukwaWriters.java
----------------------------------------------------------------------
diff --git
a/src/test/java/org/apache/hadoop/chukwa/datacollection/writer/TestChukwaWriters.java
b/src/test/java/org/apache/hadoop/chukwa/datacollection/writer/TestChukwaWriters.java
index 8d0dd46..28a48d7 100644
---
a/src/test/java/org/apache/hadoop/chukwa/datacollection/writer/TestChukwaWriters.java
+++
b/src/test/java/org/apache/hadoop/chukwa/datacollection/writer/TestChukwaWriters.java
@@ -174,128 +174,6 @@ public class TestChukwaWriters extends TestCase{
return null;
}
-// /**
-// * Test to check if the .chukwa files are closing at the time we expect
them
-// * to close. This test sets the rotateInterval and offsetInterval to small
-// * values, reads the filename of the first .chukwa file, extracts the
-// * timestamp from its name, calculates the timestamp when the next .chukwa
-// * file should be closed, sleeps for some time (enough for producing the
next
-// * .chukwa file), reads the timestamp on the second .chukwa file, and
-// * compares the expected close timestamp with the actual closing timestamp
of
-// * the second file.
-// */
-// public void testParquetWriterFixedCloseInterval() {
-// try {
-// long rotateInterval = 10000;
-// long intervalOffset = 3000;
-//
-// File tempDir = new File(System.getProperty("test.build.data", "/tmp"));
-// if (!tempDir.exists()) {
-// tempDir.mkdirs();
-// }
-//
-// String outputDirectory = tempDir.getPath() +
"/testChukwaWriters_testChukwaParquetWriterFixedCloseInterval_" +
-// System.currentTimeMillis() + "/";
-// String parquetWriterOutputDir = outputDirectory
+"/parquetWriter/parquetOutputDir";
-//
-// File directory = new File(parquetWriterOutputDir);
-// // if some files already exist in this directory then delete them.
Files
-// // may exist due to an old test run.
-// File[] files = directory.listFiles();
-// if (files != null) {
-// for(File file: files) {
-// file.delete();
-// }
-// }
-//
-// Configuration confParquetWriter = new Configuration();
-// confParquetWriter.set("chukwaCollector.rotateInterval",
String.valueOf(rotateInterval));
-// confParquetWriter.set("writer.hdfs.filesystem", "file:///");
-// confParquetWriter.set(ChukwaParquetWriter.OUTPUT_DIR_OPT,
parquetWriterOutputDir );
-// confParquetWriter.set("chukwaCollector.isFixedTimeRotatorScheme",
"true");
-// confParquetWriter.set("chukwaCollector.fixedTimeIntervalOffset",
String.valueOf(intervalOffset));
-//
-// ChukwaWriter parquetWriter = new
ChukwaParquetWriter(confParquetWriter);
-//
-// // we do not want our test to fail due to a lag in calling the
-// // scheduleNextRotation() method and creating of first .chukwa file.
-// // So, we will make sure that the rotation starts in the middle
(approx)
-// // of the rotateInterval
-// long currentTime = System.currentTimeMillis();
-// long currentTimeInSec = currentTime/1000;
-// long timeAfterPrevRotateInterval = currentTimeInSec % rotateInterval;
-// if(timeAfterPrevRotateInterval > (rotateInterval - 2)){
-// Thread.sleep(2000);
-// }
-//
-// String [] fileNames = directory.list();
-// String firstFileName = "";
-// String initialTimestamp = "";
-// // extracting the close time of first .chukwa file. This timestamp can
be
-// // extracted from the file name. An example filename is
-// // 20110531122600002_<host-name>_5f836ece1302899d9a0727e.chukwa
-// for(String file: fileNames) {
-// if ( file.endsWith(".chukwa") ){
-// // set a flag so that later we can identify that this file has been
-// // visited
-// firstFileName = file;
-// // getting just the timestamp part i.e. 20110531122600002 in the
-// // example filename mentioned in the above comment
-// initialTimestamp = file.split("_")[0];
-// // stripping off the millisecond part of timestamp. The timestamp
-// // now becomes 20110531122600
-// initialTimestamp = initialTimestamp.substring(0,
initialTimestamp.length()-3);
-// break;
-// }
-// }
-//
-// SimpleDateFormat formatter = new SimpleDateFormat("yyyyMMddhhmmss");
-// Date initialDate = formatter.parse(initialTimestamp);
-// long initialDateInMillis = initialDate.getTime();
-//
-// // calculate the expected close time of the next .chukwa file.
-// long prevRoundedInterval = initialDateInMillis - (initialDateInMillis %
-// rotateInterval);
-// long expectedNextCloseDate = prevRoundedInterval +
-// rotateInterval + intervalOffset;
-//
-// // sleep for a time interval equal to (rotateInterval +
offsetInterval).
-// // Only one more .chukwa file will be will be produced in this time
-// // interval.
-// long sleepTime = rotateInterval + intervalOffset;
-//
-// Thread.sleep(sleepTime);
-// fileNames = directory.list();
-// String nextTimestamp = "";
-// // extract the timestamp of the second .chukwa file
-// for(String file: fileNames) {
-// if ( file.endsWith(".chukwa") && !file.equals(firstFileName)){
-// nextTimestamp = file.split("_")[0];
-// nextTimestamp = nextTimestamp.substring(0,
nextTimestamp.length()-3);
-// break;
-// }
-// }
-//
-// Date nextDate = formatter.parse(nextTimestamp);
-// System.out.println("initialTimestamp:"+nextTimestamp);
-// long nextDateInMillis = nextDate.getTime();
-//
-// long threshold = 500; //milliseconds
-//
-// // test will be successful only if the timestamp on the second .chukwa
-// // file is very close (differs by < 500 ms) to the expected closing
-// // timestamp we calculated.
-// Assert.assertTrue("File not closed at expected time",
-// (nextDateInMillis - expectedNextCloseDate < threshold));
-// parquetWriter.close();
-//
-// } catch (Throwable e) {
-// e.printStackTrace();
-// Assert.fail("Exception in TestChukwaWriters - " +
-// "testParquetFileFixedCloseInterval()," + e.getMessage());
-// }
-//}
-
/**
* Test to check the calculation of the delay interval for rotation in
* ParquetFileWriter. It uses an array of known currentTimestamps and their
