http://git-wip-us.apache.org/repos/asf/asterixdb/blob/0e21afa7/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/client/SpatialQueryGenerator.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/client/SpatialQueryGenerator.java b/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/client/SpatialQueryGenerator.java deleted file mode 100644 index c046f4e..0000000 --- a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/client/SpatialQueryGenerator.java +++ /dev/null @@ -1,315 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.asterix.experiment.client; - -import java.io.BufferedReader; -import java.io.DataInputStream; -import java.io.DataOutputStream; -import java.io.File; -import java.io.FileOutputStream; -import java.io.FileReader; -import java.io.IOException; -import java.net.Inet4Address; -import java.net.Socket; -import java.util.Random; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Semaphore; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.logging.Level; -import java.util.logging.Logger; - -import org.apache.asterix.experiment.action.base.IAction; -import org.apache.asterix.experiment.action.base.SequentialActionList; -import org.apache.asterix.experiment.action.derived.RunAQLStringAction; -import org.apache.asterix.experiment.action.derived.TimedAction; -import org.apache.http.client.HttpClient; -import org.apache.http.impl.client.DefaultHttpClient; -import org.apache.hyracks.api.util.ExperimentProfilerUtils; - -public class SpatialQueryGenerator { - - private static final int SKIP_LINE_COUNT = 199; - - private final ExecutorService threadPool; - - private final int partitionRangeStart; - - private final int duration; - - private final String restHost; - - private final int restPort; - - private final String orchHost; - - private final int orchPort; - - private final String openStreetMapFilePath; - - private final boolean isIndexOnlyPlan; - - public SpatialQueryGenerator(SpatialQueryGeneratorConfig config) { - threadPool = Executors.newCachedThreadPool(new ThreadFactory() { - - private final AtomicInteger count = new AtomicInteger(); - - @Override - public Thread newThread(Runnable r) { - int tid = count.getAndIncrement(); - Thread t = new Thread(r, "DataGeneratorThread: " + tid); - t.setDaemon(true); - return t; - } - }); - - partitionRangeStart = config.getPartitionRangeStart(); - duration = config.getDuration(); - restHost = config.getRESTHost(); - restPort = config.getRESTPort(); - orchHost = config.getQueryOrchestratorHost(); - orchPort = config.getQueryOrchestratorPort(); - openStreetMapFilePath = config.getOpenStreetMapFilePath(); - isIndexOnlyPlan = config.getIsIndexOnlyPlan(); - } - - public void start() throws Exception { - final Semaphore sem = new Semaphore(0); - threadPool.submit(new QueryGenerator(sem, restHost, restPort, orchHost, orchPort, partitionRangeStart, - duration, openStreetMapFilePath, isIndexOnlyPlan)); - sem.acquire(); - } - - public static class QueryGenerator implements Runnable { - - private static final Logger LOGGER = Logger.getLogger(QueryGenerator.class.getName()); - - private final HttpClient httpClient; - private final Semaphore sem; - private final String restHost; - private final int restPort; - private final String orchHost; - private final int orchPort; - private final int partition; - private final int queryDuration; - private final String openStreetMapFilePath; - private final float[] radiusType = new float[] { 0.00001f, 0.0001f, 0.001f, 0.01f, 0.1f }; - private int radiusIter = 0; - private final Random randGen; - private BufferedReader br; - private long queryCount; - private Random random = new Random(211); - private final boolean isIndexOnlyPlan; - private String outputFilePath; - private FileOutputStream outputFos; - - public QueryGenerator(Semaphore sem, String restHost, int restPort, String orchHost, int orchPort, - int partitionRangeStart, int queryDuration, String openStreetMapFilePath, boolean isIndexOnlyPlan) { - httpClient = new DefaultHttpClient(); - this.sem = sem; - this.restHost = restHost; - this.restPort = restPort; - this.orchHost = orchHost; - this.orchPort = orchPort; - this.partition = partitionRangeStart; - this.queryDuration = queryDuration * 1000; - this.openStreetMapFilePath = openStreetMapFilePath; - this.queryCount = 0; - this.randGen = new Random(partitionRangeStart); - this.isIndexOnlyPlan = isIndexOnlyPlan; - } - - @Override - public void run() { - LOGGER.info("\nQueryGen[" + partition + "] running with the following parameters: \n" - + "queryGenDuration : " + queryDuration + "\n" + "isIndexOnlyPlan : " + isIndexOnlyPlan); - - try { - outputFilePath = openStreetMapFilePath.substring(0, openStreetMapFilePath.lastIndexOf(File.separator)) - + File.separator + "QueryGenResult-" + Inet4Address.getLocalHost().getHostAddress() + ".txt"; - outputFos = ExperimentProfilerUtils.openOutputFile(outputFilePath); - } catch (Exception e) { - e.printStackTrace(); - return; - } - - try { - if (openStreetMapFilePath != null) { - this.br = new BufferedReader(new FileReader(openStreetMapFilePath)); - } - try { - //connect to orchestrator socket - Socket orchSocket = null; - orchSocket = new Socket(orchHost, orchPort); - try { - //send reached message to orchestrator - sendReached(orchSocket); - - //wait until receiving resume message from the orchestrator server - receiveResume(orchSocket); - - if (LOGGER.isLoggable(Level.INFO)) { - LOGGER.info("QueryGen[" + partition + "] starts sending queries..."); - } - //send queries during query duration - long startTS = System.currentTimeMillis(); - long prevTS = startTS; - long curTS = startTS; - while (curTS - startTS < queryDuration) { - sendQuery(); - queryCount++; - curTS = System.currentTimeMillis(); - if (LOGGER.isLoggable(Level.INFO) && queryCount % 100 == 0) { - LOGGER.info("QueryGen[" + partition + "][TimeToQuery100] " + (curTS - prevTS) - + " in milliseconds"); - prevTS = curTS; - } - } - if (LOGGER.isLoggable(Level.INFO)) { - LOGGER.info("QueryGen[" + partition + "][QueryCount] " + queryCount + " in " - + (queryDuration / 1000) + " seconds"); - } - - if (outputFos != null) { - ExperimentProfilerUtils.closeOutputFile(outputFos); - } - - //send reqched message to orchestrator - sendReached(orchSocket); - } finally { - if (orchSocket != null) { - orchSocket.close(); - } - } - } catch (Throwable t) { - t.printStackTrace(); - outputFos.write("Error during sending query\n".getBytes()); - throw t; - } finally { - if (openStreetMapFilePath != null) { - br.close(); - } - if (outputFos != null) { - ExperimentProfilerUtils.closeOutputFile(outputFos); - } - } - } catch (Throwable t) { - System.err.println("Error connecting to rest API server " + restHost + ":" + restPort); - t.printStackTrace(); - } finally { - sem.release(); - } - } - - private void sendQuery() throws IOException { - //prepare radius and center point - int skipLineCount = SKIP_LINE_COUNT; - int lineCount = 0; - String line = null;; - String strPoints[] = null; - float x = 0, y = 0; - int beginX = -180, endX = 180, beginY = -90, endY = 90; - if (openStreetMapFilePath != null) { - while (lineCount < skipLineCount) { - if ((line = br.readLine()) == null) { - //reopen file - br.close(); - br = new BufferedReader(new FileReader(openStreetMapFilePath)); - line = br.readLine(); - } - strPoints = line.split(","); - if (strPoints.length != 2) { - continue; - } - lineCount++; - } - y = Float.parseFloat(strPoints[0]) / 10000000; //latitude (y value) - x = Float.parseFloat(strPoints[1]) / 10000000; //longitude (x value) - } else { - int xMajor = beginX + random.nextInt(endX - beginX); - int xMinor = random.nextInt(100); - x = xMajor + ((float) xMinor) / 100; - - int yMajor = beginY + random.nextInt(endY - beginY); - int yMinor = random.nextInt(100); - y = yMajor + ((float) yMinor) / 100; - } - - //create action - SequentialActionList sAction = new SequentialActionList(); - IAction rangeQueryAction = new TimedAction(new RunAQLStringAction(httpClient, restHost, restPort, - getRangeQueryAQL(radiusType[radiusIter++ % radiusType.length], x, y, isIndexOnlyPlan), outputFos), - outputFos); - sAction.add(rangeQueryAction); - - //perform - sAction.perform(); - } - - private String getRangeQueryAQL(float radius, float x, float y, boolean isIndexOnlyPlan) { - StringBuilder sb = new StringBuilder(); - sb.append("use dataverse experiments; "); - sb.append("count( "); - sb.append("for $x in dataset Tweets").append(" "); - sb.append("let $n := create-circle( "); - sb.append("point(\"").append(x).append(", ").append(y).append("\") "); - sb.append(", "); - sb.append(String.format("%f", radius)); - sb.append(" )"); - if (isIndexOnlyPlan) { - sb.append("where spatial-intersect($x.sender-location, $n) "); - } else { - sb.append("where spatial-intersect($x.sender-location, $n) and $x.btree-extra-field1 <= int32(\"2147483647\") "); - } - sb.append("return $x "); - sb.append(");"); - return sb.toString(); - } - - private void sendReached(Socket s) throws IOException { - new DataOutputStream(s.getOutputStream()).writeInt(OrchestratorDGProtocol.REACHED.ordinal()); - s.getOutputStream().flush(); - if (LOGGER.isLoggable(Level.INFO)) { - LOGGER.info("Sent " + OrchestratorDGProtocol.REACHED + " to " + s.getRemoteSocketAddress()); - } - } - - private void receiveResume(Socket s) throws IOException { - int msg = new DataInputStream(s.getInputStream()).readInt(); - OrchestratorDGProtocol msgType = OrchestratorDGProtocol.values()[msg]; - if (LOGGER.isLoggable(Level.INFO)) { - LOGGER.info("Received " + msgType + " from " + s.getRemoteSocketAddress()); - } - if (msgType != OrchestratorDGProtocol.RESUME) { - throw new IllegalStateException("Unknown protocol message received: " + msgType); - } - } - - private void sendStopped(Socket s) throws IOException { - new DataOutputStream(s.getOutputStream()).writeInt(OrchestratorDGProtocol.STOPPED.ordinal()); - s.getOutputStream().flush(); - if (LOGGER.isLoggable(Level.INFO)) { - LOGGER.info("Sent " + OrchestratorDGProtocol.STOPPED + " to " + s.getRemoteSocketAddress()); - } - } - - } -}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/0e21afa7/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/client/SpatialQueryGeneratorConfig.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/client/SpatialQueryGeneratorConfig.java b/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/client/SpatialQueryGeneratorConfig.java deleted file mode 100644 index 79055aa..0000000 --- a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/client/SpatialQueryGeneratorConfig.java +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.asterix.experiment.client; - -import org.apache.commons.lang3.tuple.Pair; -import org.kohsuke.args4j.CmdLineException; -import org.kohsuke.args4j.CmdLineParser; -import org.kohsuke.args4j.Option; -import org.kohsuke.args4j.OptionDef; -import org.kohsuke.args4j.spi.OptionHandler; -import org.kohsuke.args4j.spi.Parameters; -import org.kohsuke.args4j.spi.Setter; - -public class SpatialQueryGeneratorConfig { - - @Option(name = "-p", aliases = "--partition-range-start", usage = "Starting partition number for the set of data generators (default = 0)") - private int partitionRangeStart = 0; - - public int getPartitionRangeStart() { - return partitionRangeStart; - } - - @Option(name = "-qd", aliases = { "--duration" }, usage = "Duration in seconds to run guery generation") - private int queryGenDuration = -1; - - public int getDuration() { - return queryGenDuration; - } - - @Option(name = "-qc", aliases = { "--query-count" }, usage = "The number of queries to generate") - private int queryCount = -1; - - public int getQueryCount() { - return queryCount; - } - - @Option(name = "-rh", aliases = "--rest-host", usage = "Asterix REST API host address", required = true, metaVar = "HOST") - private String restHost; - - public String getRESTHost() { - return restHost; - } - - @Option(name = "-rp", aliases = "--rest-port", usage = "Asterix REST API port", required = true, metaVar = "PORT") - private int restPort; - - public int getRESTPort() { - return restPort; - } - - @Option(name = "-qoh", aliases = "--query-orchestrator-host", usage = "The host address of query orchestrator") - private String queryOrchHost; - - public String getQueryOrchestratorHost() { - return queryOrchHost; - } - - @Option(name = "-qop", aliases = "--query-orchestrator-port", usage = "The port to be used for the orchestrator server of query orchestrator") - private int queryOrchPort; - - public int getQueryOrchestratorPort() { - return queryOrchPort; - } - - @Option(name = "-of", aliases = "--openstreetmap-filepath", usage = "The open street map gps point data file path") - private String openStreetMapFilePath; - - public String getOpenStreetMapFilePath() { - return openStreetMapFilePath; - } - - @Option(name = "-iop", aliases = "--index-only-plan", usage = "Indicator of whether index only plan or not") - private boolean isIndexOnlyPlan; - - public boolean getIsIndexOnlyPlan() { - return isIndexOnlyPlan; - } - - public static class AddressOptionHandler extends OptionHandler<Pair<String, Integer>> { - - public AddressOptionHandler(CmdLineParser parser, OptionDef option, Setter<? super Pair<String, Integer>> setter) { - super(parser, option, setter); - } - - @Override - public int parseArguments(Parameters params) throws CmdLineException { - int counter = 0; - while (true) { - String param; - try { - param = params.getParameter(counter); - } catch (CmdLineException ex) { - break; - } - - String[] hostPort = param.split(":"); - if (hostPort.length != 2) { - throw new CmdLineException("Invalid address: " + param + ". Expected <host>:<port>"); - } - Integer port = null; - try { - port = Integer.parseInt(hostPort[1]); - } catch (NumberFormatException e) { - throw new CmdLineException("Invalid port " + hostPort[1] + " for address " + param + "."); - } - setter.addValue(Pair.of(hostPort[0], port)); - counter++; - } - return counter; - } - - @Override - public String getDefaultMetaVariable() { - return "addresses"; - } - - } -} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/0e21afa7/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/client/SpatialQueryGeneratorDriver.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/client/SpatialQueryGeneratorDriver.java b/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/client/SpatialQueryGeneratorDriver.java deleted file mode 100644 index ee16340..0000000 --- a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/client/SpatialQueryGeneratorDriver.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.asterix.experiment.client; - -import org.kohsuke.args4j.CmdLineException; -import org.kohsuke.args4j.CmdLineParser; - -public class SpatialQueryGeneratorDriver { - public static void main(String[] args) throws Exception { - SpatialQueryGeneratorConfig clientConfig = new SpatialQueryGeneratorConfig(); - CmdLineParser clp = new CmdLineParser(clientConfig); - try { - clp.parseArgument(args); - } catch (CmdLineException e) { - System.err.println(e.getMessage()); - clp.printUsage(System.err); - System.exit(1); - } - - SpatialQueryGenerator client = new SpatialQueryGenerator(clientConfig); - client.start(); - } -} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/0e21afa7/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/client/SyntheticDataGeneratorConfig.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/client/SyntheticDataGeneratorConfig.java b/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/client/SyntheticDataGeneratorConfig.java deleted file mode 100644 index 8621681..0000000 --- a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/client/SyntheticDataGeneratorConfig.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.asterix.experiment.client; - -import org.kohsuke.args4j.Option; - -public class SyntheticDataGeneratorConfig { - - @Option(name = "-psf", aliases = "--point-source-file", usage = "The point source file") - private String pointSourceFile; - - public String getPointSourceFile() { - return pointSourceFile; - } - - @Option(name = "-psi", aliases = "--point-sampling-interval", usage = "The point sampling interval from the point source file", required = true) - private int pointSamplingInterval; - - public int getpointSamplingInterval() { - return pointSamplingInterval; - } - - @Option(name = "-pid", aliases = "--parition-id", usage = "The partition id in order to avoid key duplication", required = true) - private int partitionId; - - public int getPartitionId() { - return partitionId; - } - - @Option(name = "-of", aliases = "--output-filepath", usage = "The output file path", required = true) - private String outputFilePath; - - public String getOutputFilePath() { - return outputFilePath; - } - - @Option(name = "-rc", aliases = "--record-count", usage = "The record count to generate", required = true) - private long recordCount; - - public long getRecordCount() { - return recordCount; - } -} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/0e21afa7/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/client/SyntheticDataGeneratorForSpatialIndexEvaluation.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/client/SyntheticDataGeneratorForSpatialIndexEvaluation.java b/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/client/SyntheticDataGeneratorForSpatialIndexEvaluation.java deleted file mode 100644 index 25d56a0..0000000 --- a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/client/SyntheticDataGeneratorForSpatialIndexEvaluation.java +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.asterix.experiment.client; - -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; - -import org.apache.asterix.tools.external.data.DataGeneratorForSpatialIndexEvaluation; -import org.apache.asterix.tools.external.data.DataGeneratorForSpatialIndexEvaluation.InitializationInfo; -import org.apache.asterix.tools.external.data.DataGeneratorForSpatialIndexEvaluation.TweetMessageIterator; -import org.apache.asterix.tools.external.data.GULongIDGenerator; -import org.kohsuke.args4j.CmdLineException; -import org.kohsuke.args4j.CmdLineParser; - -public class SyntheticDataGeneratorForSpatialIndexEvaluation { - public static void main(String[] args) throws Exception { - SyntheticDataGeneratorConfig config = new SyntheticDataGeneratorConfig(); - CmdLineParser clp = new CmdLineParser(config); - try { - clp.parseArgument(args); - } catch (CmdLineException e) { - System.err.println(e.getMessage()); - clp.printUsage(System.err); - System.exit(1); - } - - //prepare data generator - GULongIDGenerator uidGenerator = new GULongIDGenerator(config.getPartitionId(), (byte) (0)); - String pointSourceFilePath = config.getPointSourceFile(); - int pointSampleInterval = config.getpointSamplingInterval(); - DataGeneratorForSpatialIndexEvaluation dataGenerator = new DataGeneratorForSpatialIndexEvaluation(new InitializationInfo(), pointSourceFilePath, - pointSampleInterval); - - //get record count to be generated - long maxRecordCount = config.getRecordCount(); - long recordCount = 0; - - //prepare timer - long startTS = System.currentTimeMillis(); - - //prepare tweetIterator which acutally generates tweet - TweetMessageIterator tweetIterator = dataGenerator.new TweetMessageIterator(0, uidGenerator); - - FileOutputStream fos = null; - try { - //prepare output file - fos = openOutputFile(config.getOutputFilePath()); - - while (recordCount < maxRecordCount) { - //get a tweet and append newline at the end - String tweet = tweetIterator.next().toString() + "\n"; - //write to file - fos.write(tweet.getBytes()); - - recordCount++; - if (recordCount % 1000000 == 0) { - System.out.println("... generated " + recordCount + " records"); - } - } - System.out.println("Done: generated " + recordCount + " records in " - + ((System.currentTimeMillis() - startTS) / 1000) + " in seconds!"); - } catch (Exception e) { - e.printStackTrace(); - } finally { - try { - closeOutputFile(fos); - } catch (IOException e) { - e.printStackTrace(); - } - } - } - - public static FileOutputStream openOutputFile(String filepath) throws IOException { - File file = new File(filepath); - if (file.exists()) { - throw new IOException(filepath + "already exists"); - } - file.createNewFile(); - return new FileOutputStream(file); - } - - public static void closeOutputFile(FileOutputStream fos) throws IOException { - fos.flush(); - fos.close(); - fos = null; - } -} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/0e21afa7/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/logging/ExperimentLogFormatter.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/logging/ExperimentLogFormatter.java b/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/logging/ExperimentLogFormatter.java deleted file mode 100644 index 544fbf8..0000000 --- a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/logging/ExperimentLogFormatter.java +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.asterix.experiment.logging; - -import java.io.PrintWriter; -import java.io.StringWriter; -import java.text.MessageFormat; -import java.text.SimpleDateFormat; -import java.util.Date; -import java.util.logging.Formatter; -import java.util.logging.Level; -import java.util.logging.LogRecord; - -public class ExperimentLogFormatter extends Formatter { - - // format string for printing the log record - private static final String format = "{0} {1}\n{2}: {3}\n"; - private final Date dat = new Date(); - private final SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS"); - - /** - * Format the given LogRecord. - * <p> - * The formatting can be customized by specifying the <a href="../Formatter.html#syntax">format string</a> in the <a href="#formatting"> {@code java.util.logging.SimpleFormatter.format}</a> property. The given {@code LogRecord} will be formatted as if by calling: - * - * <pre> - * {@link String#format String.format}(format, date, source, logger, level, message, thrown); - * </pre> - * - * where the arguments are:<br> - * <ol> - * <li>{@code format} - the {@link java.util.Formatter - * java.util.Formatter} format string specified in the {@code java.util.logging.SimpleFormatter.format} property or the default format.</li> - * <li>{@code date} - a {@link Date} object representing {@linkplain LogRecord#getMillis event time} of the log record.</li> - * <li>{@code source} - a string representing the caller, if available; otherwise, the logger's name.</li> - * <li>{@code logger} - the logger's name.</li> - * <li>{@code level} - the {@linkplain Level#getLocalizedName - * log level}.</li> - * <li>{@code message} - the formatted log message returned from the {@link Formatter#formatMessage(LogRecord)} method. It uses {@link java.text.MessageFormat java.text} formatting and does not use the {@code java.util.Formatter format} argument.</li> - * <li>{@code thrown} - a string representing the {@linkplain LogRecord#getThrown throwable} associated with the log record and its backtrace beginning with a newline character, if any; otherwise, an empty string.</li> - * </ol> - * <p> - * Some example formats:<br> - * <ul> - * <li> {@code java.util.logging.SimpleFormatter.format="%4$s: %5$s [%1$tc]%n"} - * <p> - * This prints 1 line with the log level ({@code 4$}), the log message ({@code 5$}) and the timestamp ({@code 1$}) in a square bracket. - * - * <pre> - * WARNING: warning message [Tue Mar 22 13:11:31 PDT 2011] - * </pre> - * - * </li> - * <li> {@code java.util.logging.SimpleFormatter.format="%1$tc %2$s%n%4$s: %5$s%6$s%n"} - * <p> - * This prints 2 lines where the first line includes the timestamp ({@code 1$}) and the source ({@code 2$}); the second line includes the log level ({@code 4$}) and the log message ({@code 5$}) followed with the throwable and its backtrace ({@code 6$}), if any: - * - * <pre> - * Tue Mar 22 13:11:31 PDT 2011 MyClass fatal - * SEVERE: several message with an exception - * java.lang.IllegalArgumentException: invalid argument - * at MyClass.mash(MyClass.java:9) - * at MyClass.crunch(MyClass.java:6) - * at MyClass.main(MyClass.java:3) - * </pre> - * - * </li> - * <li> {@code java.util.logging.SimpleFormatter.format="%1$tb %1$td, %1$tY %1$tl:%1$tM:%1$tS %1$Tp %2$s%n%4$s: %5$s%n"} - * <p> - * This prints 2 lines similar to the example above with a different date/time formatting and does not print the throwable and its backtrace: - * - * <pre> - * Mar 22, 2011 1:11:31 PM MyClass fatal - * SEVERE: several message with an exception - * </pre> - * - * </li> - * </ul> - * <p> - * This method can also be overridden in a subclass. It is recommended to use the {@link Formatter#formatMessage} convenience method to localize and format the message field. - * - * @param record - * the log record to be formatted. - * @return a formatted log record - */ - public synchronized String format(LogRecord record) { - dat.setTime(record.getMillis()); - String source; - if (record.getSourceClassName() != null) { - source = record.getSourceClassName(); - if (record.getSourceMethodName() != null) { - source += " " + record.getSourceMethodName(); - } - } else { - source = record.getLoggerName(); - } - String message = formatMessage(record); - String throwable = ""; - if (record.getThrown() != null) { - StringWriter sw = new StringWriter(); - PrintWriter pw = new PrintWriter(sw); - pw.println(); - record.getThrown().printStackTrace(pw); - pw.close(); - throwable = sw.toString(); - } - return MessageFormat.format(format, sdf.format(dat), source, record.getLevel(), - message, throwable); - } - -} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/0e21afa7/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/report/AbstractDynamicDataEvalReportBuilder.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/report/AbstractDynamicDataEvalReportBuilder.java b/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/report/AbstractDynamicDataEvalReportBuilder.java deleted file mode 100644 index 13d2694..0000000 --- a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/report/AbstractDynamicDataEvalReportBuilder.java +++ /dev/null @@ -1,286 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.asterix.experiment.report; - -import java.io.BufferedReader; -import java.io.File; -import java.io.FileReader; -import java.io.IOException; -import java.text.SimpleDateFormat; -import java.util.List; - -public abstract class AbstractDynamicDataEvalReportBuilder implements IDynamicDataEvalReportBuilder { - - protected final static String INSTANTANEOUS_INSERT_STRING = "[TimeToInsert100000]"; - protected final static int INSTANTAEOUS_INSERT_COUNT = 100000; - protected final static int ROUND_COUNT = 721; - protected final static int ROUND_INTERVAL = 5; - protected final String expHomePath; - protected final String expName; - protected final String runLogFilePath; - protected final String[] ncLogFilePaths; - protected BufferedReader br = null; - protected BufferedReader[] ncLogBrs = null; - protected final int ncLogFileCount; - - protected final StringBuilder dataGenSb; - protected final StringBuilder queryGenSb; - protected final StringBuilder rsb; - - protected AbstractDynamicDataEvalReportBuilder(String expHomePath, String expName, String runLogFileName, - boolean hasStatFile) { - this.expHomePath = expHomePath; - this.expName = expName; - this.runLogFilePath = new String(expHomePath + runLogFileName); - if (expName.contains("1A")) { - ncLogFileCount = 1; - } else if (expName.contains("1B")) { - ncLogFileCount = 2; - } else if (expName.contains("1C")) { - ncLogFileCount = 4; - } else /* if (expName.contains("1D") || other exps) */{ - ncLogFileCount = 8; - } - ncLogFilePaths = new String[ncLogFileCount]; - ncLogBrs = new BufferedReader[ncLogFileCount]; - for (int i = 0; i < ncLogFileCount; i++) { - if (hasStatFile) { - ncLogFilePaths[i] = new String(expHomePath + expName + File.separator + "node" + (i + 1) - + File.separator + "logs" + File.separator + "a1_node" + (i + 1) + ".log"); - } else { - ncLogFilePaths[i] = new String(expHomePath + expName + File.separator + "logs" + File.separator - + "a1_node" + (i + 1) + ".log"); - } - } - dataGenSb = new StringBuilder(); - queryGenSb = new StringBuilder(); - rsb = new StringBuilder(); - } - - protected void openRunLog() throws IOException { - br = new BufferedReader(new FileReader(runLogFilePath)); - for (int i = 0; i < ncLogFileCount; i++) { - ncLogBrs[i] = new BufferedReader(new FileReader(ncLogFilePaths[i])); - } - } - - protected void closeRunLog() throws IOException { - if (br != null) { - br.close(); - } - if (ncLogBrs != null) { - for (int i = 0; i < ncLogFileCount; i++) { - if (ncLogBrs[i] != null) { - ncLogBrs[i].close(); - } - } - } - } - - protected boolean moveToExperimentBegin() throws IOException { - String line; - while ((line = br.readLine()) != null) { - if (line.contains("Running experiment: " + expName)) { - return true; - } - } - return false; - } - - protected void renewStringBuilder() { - dataGenSb.setLength(0); - queryGenSb.setLength(0); - rsb.setLength(0); - } - - @Override - public String getInstantaneousInsertPS(int nodeId, boolean useTimeForX) throws Exception { - renewStringBuilder(); - openRunLog(); - try { - - if (!moveToExperimentBegin()) { - //The experiment run log doesn't exist in this run log file - return null; - } - - int round = 0; - while (round < ROUND_COUNT) { - long IIPS = 0; - String line; - while ((line = ncLogBrs[nodeId].readLine()) != null) { - if (line.contains("IPS")) { - IIPS = ReportBuilderHelper.getLong(line, ", IIPS[", "]"); - break; - } - } - round++; - dataGenSb.append(round * ROUND_INTERVAL).append(",").append(IIPS).append("\n"); - } - - return dataGenSb.toString(); - } finally { - closeRunLog(); - } - } - - @Override - public void getAllNodesAccumulatedInsertPS(int targetRound, List<Long> ipsList) throws Exception { - renewStringBuilder(); - openRunLog(); - ipsList.clear(); - try { - - if (!moveToExperimentBegin()) { - //The experiment run log doesn't exist in this run log file - return; - } - - int round = 0; - while (round < targetRound) { - long IPSPerRound = 0; - for (int i = 0; i < ncLogFileCount; i++) { - String line; - while ((line = ncLogBrs[i].readLine()) != null) { - if (line.contains("IPS")) { - IPSPerRound += ReportBuilderHelper.getLong(line, ", IPS[", "]"); - break; - } - } - } - ipsList.add(IPSPerRound); - round++; - } - return; - } finally { - closeRunLog(); - } - } - - public String getInstantaneousDataGenPS(int genId, boolean useTimeForX) throws Exception { - renewStringBuilder(); - openRunLog(); - try { - if (!moveToExperimentBegin()) { - //The experiment run log doesn't exist in this run log file - return null; - } - - String line; - int dGenId; - int count = 0; - long timeToInsert = 0; - long totalTimeToInsert = 0; - while ((line = br.readLine()) != null) { - if (line.contains(INSTANTANEOUS_INSERT_STRING)) { - dGenId = ReportBuilderHelper.getInt(line, "DataGen[", "]"); - if (dGenId == genId) { - count++; - timeToInsert = ReportBuilderHelper.getLong(line, INSTANTANEOUS_INSERT_STRING, "in"); - totalTimeToInsert += timeToInsert; - if (useTimeForX) { - dataGenSb.append(totalTimeToInsert / 1000).append(",") - .append(INSTANTAEOUS_INSERT_COUNT / ((double) (timeToInsert) / 1000)).append("\n"); - } else { - dataGenSb.append(count).append(",") - .append(INSTANTAEOUS_INSERT_COUNT / ((double) (timeToInsert) / 1000)).append("\n"); - } - } - } - if (line.contains("Running")) { - break; - } - } - System.out.println("GenId[" + genId + "] " + totalTimeToInsert + ", " + (totalTimeToInsert / (1000 * 60))); - return dataGenSb.toString(); - } finally { - closeRunLog(); - } - } - - public long getDataGenStartTimeStamp() throws Exception { - openRunLog(); - try { - String line; - while ((line = br.readLine()) != null) { - if (line.contains("Running experiment: " + expName)) { - while ((line = br.readLine()) != null) { - //2015-10-27 17:18:28,242 INFO [ParallelActionThread 6] transport.TransportImpl (TransportImpl.java:init(155)) - Client identity string: SSH-2.0-SSHJ_0_13_0 - if (line.contains("INFO [ParallelActionThread")) { - //format1 = new SimpleDateFormat("MMM dd, yyyy hh:mm:ss aa"); - //format2 = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss"); - SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss"); - return ReportBuilderHelper.getTimeStampAsLong(line, format); - } - } - } - } - return -1; - } finally { - closeRunLog(); - } - } - - public String getIndexSize(String indexDirPath) throws Exception { - /* - * exmaple - * /mnt/data/sdb/youngsk2/asterix/storage/experiments/Tweets_idx_dhbtreeLocation/device_id_0: - total 211200 - -rw-r--r-- 1 youngsk2 grad 191234048 Jun 29 00:11 2015-06-29-00-09-59-023_2015-06-28-23-51-56-984_b - -rw-r--r-- 1 youngsk2 grad 7864320 Jun 29 00:11 2015-06-29-00-09-59-023_2015-06-28-23-51-56-984_f - -rw-r--r-- 1 youngsk2 grad 4194304 Jun 29 00:10 2015-06-29-00-10-26-997_2015-06-29-00-10-26-997_b - -rw-r--r-- 1 youngsk2 grad 393216 Jun 29 00:10 2015-06-29-00-10-26-997_2015-06-29-00-10-26-997_f - -rw-r--r-- 1 youngsk2 grad 5898240 Jun 29 00:11 2015-06-29-00-10-59-791_2015-06-29-00-10-59-791_b - -rw-r--r-- 1 youngsk2 grad 393216 Jun 29 00:11 2015-06-29-00-10-59-791_2015-06-29-00-10-59-791_f - -rw-r--r-- 1 youngsk2 grad 5898240 Jun 29 00:11 2015-06-29-00-11-30-486_2015-06-29-00-11-30-486_b - -rw-r--r-- 1 youngsk2 grad 393216 Jun 29 00:11 2015-06-29-00-11-30-486_2015-06-29-00-11-30-486_f - - */ - renewStringBuilder(); - openRunLog(); - try { - if (!moveToExperimentBegin()) { - //The experiment run log doesn't exist in this run log file - return null; - } - - String line; - String[] tokens; - long diskSize = 0; - while ((line = br.readLine()) != null) { - if (line.contains(indexDirPath)) { - br.readLine();//discard "total XXXX" line - //read and sum file size - while (!(line = br.readLine()).isEmpty()) { - tokens = line.split("\\s+");; - diskSize += Long.parseLong(tokens[4].trim()); - } - } - if (line.contains("Running")) { - break; - } - } - rsb.append((double) diskSize / (1024 * 1024 * 1024)); - return rsb.toString(); - } finally { - closeRunLog(); - } - } -} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/0e21afa7/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/report/AbstractStaticDataEvalReportBuilder.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/report/AbstractStaticDataEvalReportBuilder.java b/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/report/AbstractStaticDataEvalReportBuilder.java deleted file mode 100644 index fdd1d92..0000000 --- a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/report/AbstractStaticDataEvalReportBuilder.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.asterix.experiment.report; - -public abstract class AbstractStaticDataEvalReportBuilder implements IStaticDataEvalReportBuilder { - - protected final String expName; - protected final String runLogFilePath; - - protected AbstractStaticDataEvalReportBuilder(String expName, String runLogFilePath) { - this.expName = expName; - this.runLogFilePath = runLogFilePath; - } -} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/0e21afa7/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/report/IDynamicDataEvalReportBuilder.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/report/IDynamicDataEvalReportBuilder.java b/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/report/IDynamicDataEvalReportBuilder.java deleted file mode 100644 index 4d5e511..0000000 --- a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/report/IDynamicDataEvalReportBuilder.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.asterix.experiment.report; - -import java.util.List; - -public interface IDynamicDataEvalReportBuilder { - public String getInstantaneousInsertPS(int genId, boolean useTimeForX) throws Exception; - - public String getOverallInsertPS(int minutes) throws Exception; - - public String getInstantaneousQueryPS() throws Exception; - - public String getQueryPS(int minutes) throws Exception; - - void getAllNodesAccumulatedInsertPS(int targetRound, List<Long> ipsList) throws Exception; -} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/0e21afa7/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/report/IStaticDataEvalReportBuilder.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/report/IStaticDataEvalReportBuilder.java b/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/report/IStaticDataEvalReportBuilder.java deleted file mode 100644 index 6c879f4..0000000 --- a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/report/IStaticDataEvalReportBuilder.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.asterix.experiment.report; - -public interface IStaticDataEvalReportBuilder { - public String getSelectQueryTime(); - public String getJoinQueryTime(); -} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/0e21afa7/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/report/NCLogReportBuilder.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/report/NCLogReportBuilder.java b/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/report/NCLogReportBuilder.java deleted file mode 100644 index 5da4463..0000000 --- a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/report/NCLogReportBuilder.java +++ /dev/null @@ -1,212 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.asterix.experiment.report; - -import java.io.BufferedReader; -import java.io.FileReader; -import java.io.IOException; -import java.text.SimpleDateFormat; -import java.util.HashMap; -import java.util.Iterator; -import java.util.Map.Entry; - -public class NCLogReportBuilder { - - private String ncLogFilePath = "/Users/kisskys/workspace/asterix_experiment/run-log/measure-with-balloon/sie1-8dgen/log-1436511417368/SpatialIndexExperiment1ADhbtree/logs/a1_node1.log"; - private BufferedReader br; - private String timeLine; - private String msgLine; - - public NCLogReportBuilder(String filePath) { - if (filePath != null) { - this.ncLogFilePath = filePath; - } - } - - public String getFlushMergeEventAsGanttChartFormat(long testBeginTimeStamp) throws Exception { - openNCLog(); - StringBuilder sb = new StringBuilder(); - long flushStartTimeStamp, flushFinishTimeStamp, mergeStartTimeStamp, mergeFinishTimeStamp; - String indexName; - SimpleDateFormat format = new SimpleDateFormat("MMM dd, yyyy hh:mm:ss aa"); - HashMap<String, Long> flushMap = new HashMap<String, Long>(); - HashMap<String, Long> mergeMap = new HashMap<String, Long>(); - long sTime, fTime; - try { - while ((timeLine = br.readLine()) != null) { - if ((msgLine = br.readLine()) == null) { - break; - } - while (!msgLine.contains("INFO:")) { - timeLine = msgLine; - msgLine = br.readLine(); - if (msgLine == null) { - break; - } - } - if (msgLine == null) { - break; - } - - //flush start - if (msgLine.contains("Started a flush operation for index")) { - flushStartTimeStamp = ReportBuilderHelper.getTimeStampAsLong(timeLine, format); - - //ignore flush op which happened before the data gen started. - if (flushStartTimeStamp < testBeginTimeStamp) { - continue; - } - - indexName = ReportBuilderHelper.getString(msgLine, "experiments/Tweets_idx_", "/]"); - flushMap.put(indexName, flushStartTimeStamp); - } - - //flush finish - if (msgLine.contains("Finished the flush operation for index")) { - flushFinishTimeStamp = ReportBuilderHelper.getTimeStampAsLong(timeLine, format); - - //ignore flush op which happened before the data gen started. - if (flushFinishTimeStamp < testBeginTimeStamp) { - continue; - } - - indexName = ReportBuilderHelper.getString(msgLine, "experiments/Tweets_idx_", "/]"); - - if (flushMap.containsKey(indexName)) { - flushStartTimeStamp = flushMap.remove(indexName); - sTime = (flushStartTimeStamp - testBeginTimeStamp) / 1000; - fTime = (flushFinishTimeStamp - testBeginTimeStamp) / 1000; - if (fTime == sTime) { - ++fTime; - } - //only for sie1 - // if (fTime > 1200) { - // fTime = 1200; - // } - sb.append("f-" + getPrintName(indexName)).append("\t").append(sTime).append("\t").append(fTime) - .append("\t").append(indexName.contains("Tweets") ? "flushPidx" : "flushSidx") - .append("\n"); - } - } - - //merge start - if (msgLine.contains("Started a merge operation for index")) { - mergeStartTimeStamp = ReportBuilderHelper.getTimeStampAsLong(timeLine, format); - - //ignore flush op which happened before the data gen started. - if (mergeStartTimeStamp < testBeginTimeStamp) { - continue; - } - - indexName = ReportBuilderHelper.getString(msgLine, "experiments/Tweets_idx_", "/]"); - mergeMap.put(indexName, mergeStartTimeStamp); - } - - //merge finish - if (msgLine.contains("Finished the merge operation for index")) { - mergeFinishTimeStamp = ReportBuilderHelper.getTimeStampAsLong(timeLine, format); - - //ignore flush op which happened before the data gen started. - if (mergeFinishTimeStamp < testBeginTimeStamp) { - continue; - } - - indexName = ReportBuilderHelper.getString(msgLine, "experiments/Tweets_idx_", "/]"); - - if (mergeMap.containsKey(indexName)) { - mergeStartTimeStamp = mergeMap.remove(indexName); - sTime = (mergeStartTimeStamp - testBeginTimeStamp) / 1000; - fTime = (mergeFinishTimeStamp - testBeginTimeStamp) / 1000; - if (fTime == sTime) { - ++fTime; - } - //only for sie1 - // if (fTime > 1200) { - // fTime = 1200; - // } - sb.append("m-" + getPrintName(indexName)).append("\t").append(sTime).append("\t").append(fTime) - .append("\t").append(indexName.contains("Tweets") ? "mergePidx" : "mergeSidx") - .append("\n"); - } - } - } - - Iterator<Entry<String, Long>> mergeMapIter = mergeMap.entrySet().iterator(); - Entry<String, Long> entry = null; - while (mergeMapIter.hasNext()) { - entry = mergeMapIter.next(); - sb.append("m-" + getPrintName(entry.getKey())).append("\t") - .append((entry.getValue() - testBeginTimeStamp) / 1000).append("\t").append(60 * 20) - .append("\t").append(entry.getKey().contains("Tweets") ? "mergePidx" : "mergeSidx") - .append("\n"); - } - - Iterator<Entry<String, Long>> flushMapIter = mergeMap.entrySet().iterator(); - while (mergeMapIter.hasNext()) { - entry = flushMapIter.next(); - sb.append("f-" + getPrintName(entry.getKey())).append("\t") - .append((entry.getValue() - testBeginTimeStamp) / 1000).append("\t").append(60 * 20) - .append("\t").append(entry.getKey().contains("Tweets") ? "flushPidx" : "flushSidx") - .append("\n"); - } - - return sb.toString(); - } finally { - closeNCLog(); - } - } - - private String getPrintName(String indexName) { - String name = null; - if (indexName.contains("Tweets")) { - if (indexName.contains("0")) { - name = "pidx0"; - } else if (indexName.contains("1")) { - name = "pidx1"; - } else if (indexName.contains("2")) { - name = "pidx2"; - } else if (indexName.contains("3")) { - name = "pidx3"; - } - } else if (indexName.contains("Location")) { - if (indexName.contains("0")) { - name = "sidx0"; //ReportBuilderHelper.getString(indexName, "Location") + "0"; - } else if (indexName.contains("1")) { - name = "sidx1"; //ReportBuilderHelper.getString(indexName, "Location") + "1"; - } else if (indexName.contains("2")) { - name = "sidx2"; //ReportBuilderHelper.getString(indexName, "Location") + "2"; - } else if (indexName.contains("3")) { - name = "sidx3"; //ReportBuilderHelper.getString(indexName, "Location") + "2"; - } - } - return name; - } - - protected void openNCLog() throws IOException { - br = new BufferedReader(new FileReader(ncLogFilePath)); - } - - protected void closeNCLog() throws IOException { - if (br != null) { - br.close(); - } - } - -} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/0e21afa7/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/report/OperatorProfilerReportBuilder.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/report/OperatorProfilerReportBuilder.java b/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/report/OperatorProfilerReportBuilder.java deleted file mode 100644 index 24617bb..0000000 --- a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/report/OperatorProfilerReportBuilder.java +++ /dev/null @@ -1,322 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.asterix.experiment.report; - -import java.io.BufferedReader; -import java.io.FileReader; -import java.io.IOException; -import java.util.HashMap; -import java.util.Iterator; -import java.util.Map.Entry; - -public class OperatorProfilerReportBuilder { - - private static final int INDEX_BUILD_OP_COUNT = 1; - private static final int PIDX_SCAN_OP_COUNT = 1; - private static final int WARM_UP_SELECT_QUERY_COUNT = 500; - private static final int SELECT_QUERY_COUNT = 5000; - private static final int JOIN_QUERY_COUNT = 200; - private static final int JOIN_RADIUS_TYPE_COUNT = 4; - private static final int SELECT_RADIUS_TYPE_COUNT = 5; - private static final int IDX_JOIN_RADIUS_SKIP = JOIN_RADIUS_TYPE_COUNT - 1; - private static final int IDX_SELECT_RADIUS_SKIP = SELECT_RADIUS_TYPE_COUNT - 1; - private static final int IDX_INITIAL_JOIN_SKIP = INDEX_BUILD_OP_COUNT + PIDX_SCAN_OP_COUNT - + WARM_UP_SELECT_QUERY_COUNT + SELECT_QUERY_COUNT; - private static final int IDX_INITIAL_SELECT_SKIP = INDEX_BUILD_OP_COUNT + PIDX_SCAN_OP_COUNT - + WARM_UP_SELECT_QUERY_COUNT; - - private static final int HYRACK_JOB_ELAPSED_TIME_FIELD = 2; - private static final int OP_ELAPSED_TIME_FIELD = 4; - private static final int OP_TASK_ID_FIELD = 2; - private static final int OP_NAME_FIELD = 1; - - private String executionTimeFilePath = null; - private BufferedReader brExecutionTime; - private String line; - private int lineNum; - - public OperatorProfilerReportBuilder(String executionTimeFilePath) { - this.executionTimeFilePath = executionTimeFilePath; - } - - public String getIdxNumber(boolean isJoin, int radiusIdx) throws Exception { - openExecutionTimeFile(); - - StringBuilder sb = new StringBuilder(); - int initialSkip = (isJoin ? IDX_INITIAL_JOIN_SKIP : IDX_INITIAL_SELECT_SKIP) + radiusIdx; - int radiusSkip = isJoin ? IDX_JOIN_RADIUS_SKIP : IDX_SELECT_RADIUS_SKIP; - BufferedReader br = brExecutionTime; - int queryCount = isJoin ? JOIN_QUERY_COUNT / JOIN_RADIUS_TYPE_COUNT : SELECT_QUERY_COUNT - / SELECT_RADIUS_TYPE_COUNT; - lineNum = 0; - JobStat jobStat = new JobStat(); - - try { - - //initial skip - int jobCount = 0; - while ((line = br.readLine()) != null) { - lineNum++; - if (line.contains("TOTAL_HYRACKS_JOB")) { - jobCount++; - if (jobCount > initialSkip) { - break; - } - } - } - - //Reaching Here, line variable contains the first job to be counted - for (int j = 0; j < queryCount; j++) { - - analyzeOperatorExecutionTime(jobStat, br); - - //radius skip - jobCount = 0; - while ((line = br.readLine()) != null) { - lineNum++; - if (line.contains("TOTAL_HYRACKS_JOB")) { - jobCount++; - if (jobCount > radiusSkip) { - break; - } - } - } - } - - //System.out.println("lineNum: " + lineNum); - sb.append("TOTAL_HYRACKS_JOB," + (((double) jobStat.getHyracksJobTimeSum()) / jobStat.getHyracksJobCount()) - + "," + jobStat.getHyracksJobTimeSum() + "," + jobStat.getHyracksJobCount() + "\n"); - sb.append(jobStat.getOperatorsElapsedTimeAsString()); - return sb.toString(); - } finally { - closeExecutionTimeFile(); - } - } - - private void analyzeOperatorExecutionTime(JobStat jobStat, BufferedReader br) throws IOException { - //the line argument contains TOTAL_HYRACKS_JOB string. eg.: - //2015-11-04 19:13:08,003 TOTAL_HYRACKS_JOB a1_node1_JID:3_26202768 TOTAL_HYRACKS_JOB1446660788003 1066 1.066 1066 1.066 - String tokens[] = line.split("\t"); - - if (Long.parseLong(tokens[HYRACK_JOB_ELAPSED_TIME_FIELD]) > 10000) { - System.out.println("[" + lineNum + "] " + line); - } - - jobStat.addHyracksJobTime(Long.parseLong(tokens[HYRACK_JOB_ELAPSED_TIME_FIELD])); - - while ((line = br.readLine()) != null) { - lineNum++; - - if (line.isEmpty()) { - break; - } - - tokens = line.split("\t"); - if (line.contains("DISTRIBUTE_RESULT")) { - jobStat.addDistributeResultTime(Long.parseLong(tokens[OP_ELAPSED_TIME_FIELD])); - continue; - } - if (line.contains("EMPTY_TUPLE_SOURCE")) { - continue; - } - - if (line.contains("TXN_JOB_COMMIT")) { - continue; - } - - jobStat.updateOperatorTime(tokens[OP_TASK_ID_FIELD], tokens[OP_NAME_FIELD], - Long.parseLong(tokens[OP_ELAPSED_TIME_FIELD])); - } - - jobStat.updateTaskForAvgWithSlowestTask(); - } - - protected void openExecutionTimeFile() throws IOException { - brExecutionTime = new BufferedReader(new FileReader(executionTimeFilePath)); - } - - protected void closeExecutionTimeFile() throws IOException { - if (brExecutionTime != null) { - brExecutionTime.close(); - } - } - - class JobStat { - private long hyracksJobElapsedTimeSum; - private int hyracksJobCount; - private long distributeResultTimeSum; - private Task taskForAvg; - private HashMap<String, Task> taskId2TaskMap; - - public JobStat() { - hyracksJobElapsedTimeSum = 0; - hyracksJobCount = 0; - distributeResultTimeSum = 0; - taskForAvg = new Task("TaskForAvg"); - taskId2TaskMap = new HashMap<String, Task>(); - } - - public void reset() { - hyracksJobElapsedTimeSum = 0; - hyracksJobCount = 0; - distributeResultTimeSum = 0; - taskForAvg.reset();; - taskId2TaskMap.clear(); - } - - public void addHyracksJobTime(long elapsedTime) { - hyracksJobElapsedTimeSum += elapsedTime; - hyracksJobCount++; - } - - public void addDistributeResultTime(long elapsedTime) { - distributeResultTimeSum += elapsedTime; - } - - public long getDistributeResultTime() { - return distributeResultTimeSum; - } - - public long getHyracksJobTimeSum() { - return hyracksJobElapsedTimeSum; - } - - public int getHyracksJobCount() { - return hyracksJobCount; - } - - public void updateOperatorTime(String taskId, String operatorName, long elapsedTime) { - Task task = taskId2TaskMap.get(taskId); - if (task == null) { - task = new Task(taskId); - taskId2TaskMap.put(new String(taskId), task); - } - task.updateOperatorTime(operatorName, elapsedTime); - } - - public void updateTaskForAvgWithSlowestTask() { - Iterator<Entry<String, Task>> taskIter = taskId2TaskMap.entrySet().iterator(); - Task slowestTask = null; - Task curTask; - - //get the slowest task - while (taskIter.hasNext()) { - curTask = taskIter.next().getValue(); - if (slowestTask == null) { - slowestTask = curTask; - } else { - if (slowestTask.getElapsedTime() < curTask.getElapsedTime()) { - slowestTask = curTask; - } - } - } - - //update the TaskForAvg with the slowest one - HashMap<String, SumCount> operator2SumCountMap = slowestTask.getOperator2SumCountMap(); - Iterator<Entry<String, SumCount>> operatorIter = operator2SumCountMap.entrySet().iterator(); - while (operatorIter.hasNext()) { - Entry<String, SumCount> entry = operatorIter.next(); - SumCount sc = entry.getValue(); - taskForAvg.updateOperatorTime(entry.getKey(), sc.sum); - } - taskId2TaskMap.clear(); - } - - public String getOperatorsElapsedTimeAsString() { - return "SUM_OF_OPERATORS," + (((double) taskForAvg.getElapsedTime()) / hyracksJobCount) + "," - + taskForAvg.getElapsedTime() + "," + hyracksJobCount + "\n" - + taskForAvg.getOperatorsElapsedTimeAsString() + "DISTRIBUTE_RESULT," - + (((double) distributeResultTimeSum) / hyracksJobCount) + "," + distributeResultTimeSum + "," - + hyracksJobCount + "\n"; - } - } - - class Task { - private String taskId; - private long elapsedTime; - private HashMap<String, SumCount> operator2SumCountMap; - - public Task(String taskId) { - this.taskId = new String(taskId); - elapsedTime = 0; - operator2SumCountMap = new HashMap<String, SumCount>(); - } - - @Override - public int hashCode() { - return taskId.hashCode(); - } - - @Override - public boolean equals(Object o) { - if (o == this) { - return true; - } - if (!(o instanceof Task)) { - return false; - } - return ((Task) o).taskId == taskId; - } - - public long getElapsedTime() { - return elapsedTime; - } - - public void updateOperatorTime(String operatorName, long elapsedTime) { - SumCount sc = operator2SumCountMap.get(operatorName); - if (sc == null) { - sc = new SumCount(); - sc.sum = 0; - sc.count = 0; - operator2SumCountMap.put(new String(operatorName), sc); - } - sc.sum += elapsedTime; - sc.count++; - this.elapsedTime += elapsedTime; - } - - public void reset() { - elapsedTime = 0; - operator2SumCountMap.clear(); - } - - public String getOperatorsElapsedTimeAsString() { - StringBuilder sb = new StringBuilder(); - Iterator<Entry<String, SumCount>> iter = operator2SumCountMap.entrySet().iterator(); - while (iter.hasNext()) { - Entry<String, SumCount> entry = iter.next(); - SumCount sc = entry.getValue(); - sb.append(entry.getKey()).append(",").append(((double) sc.sum) / sc.count).append(",").append(sc.sum) - .append(",").append(sc.count).append("\n"); - } - return sb.toString(); - } - - public HashMap<String, SumCount> getOperator2SumCountMap() { - return operator2SumCountMap; - } - } - - class SumCount { - public long sum; - public int count; - } - -} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/0e21afa7/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/report/ProfilerReportBuilder.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/report/ProfilerReportBuilder.java b/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/report/ProfilerReportBuilder.java deleted file mode 100644 index d5eb539..0000000 --- a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/report/ProfilerReportBuilder.java +++ /dev/null @@ -1,192 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.asterix.experiment.report; - -import java.io.BufferedReader; -import java.io.FileReader; -import java.io.IOException; - -public class ProfilerReportBuilder { - - private static final int INDEX_BUILD_OP_COUNT = 1; - private static final int PIDX_SCAN_OP_COUNT = 1; - private static final int WARM_UP_SELECT_QUERY_COUNT = 500; - private static final int SELECT_QUERY_COUNT = 5000; - private static final int JOIN_QUERY_COUNT = 200; - private static final int PARTITION_COUNT = 4; - - private static final int SELECT_QUERY_INVOLVED_INDEX_COUNT = 2; - private static final int JOIN_QUERY_INVOLVED_INDEX_COUNT = 3; - private static final int JOIN_RADIUS_TYPE_COUNT = 4; - private static final int SELECT_RADIUS_TYPE_COUNT = 5; - private static final int IDX_JOIN_RADIUS_SKIP = JOIN_RADIUS_TYPE_COUNT * JOIN_QUERY_INVOLVED_INDEX_COUNT - * PARTITION_COUNT - PARTITION_COUNT; - private static final int IDX_SELECT_RADIUS_SKIP = SELECT_RADIUS_TYPE_COUNT * SELECT_QUERY_INVOLVED_INDEX_COUNT - * PARTITION_COUNT - PARTITION_COUNT; - private static final int IDX_INITIAL_JOIN_SKIP = (INDEX_BUILD_OP_COUNT + PIDX_SCAN_OP_COUNT + ((WARM_UP_SELECT_QUERY_COUNT + SELECT_QUERY_COUNT) * SELECT_QUERY_INVOLVED_INDEX_COUNT)) - * PARTITION_COUNT; - private static final int IDX_INITIAL_SELECT_SKIP = (INDEX_BUILD_OP_COUNT + PIDX_SCAN_OP_COUNT + (WARM_UP_SELECT_QUERY_COUNT * SELECT_QUERY_INVOLVED_INDEX_COUNT)) - * PARTITION_COUNT; - private static final int FP_JOIN_RADIUS_SKIP = JOIN_RADIUS_TYPE_COUNT * PARTITION_COUNT - PARTITION_COUNT; - private static final int FP_SELECT_RADIUS_SKIP = SELECT_RADIUS_TYPE_COUNT * PARTITION_COUNT - PARTITION_COUNT; - private static final int FP_INITIAL_JOIN_SKIP = (PIDX_SCAN_OP_COUNT + WARM_UP_SELECT_QUERY_COUNT + SELECT_QUERY_COUNT) - * PARTITION_COUNT; - private static final int FP_INITIAL_SELECT_SKIP = (PIDX_SCAN_OP_COUNT + WARM_UP_SELECT_QUERY_COUNT) - * PARTITION_COUNT; - - private String indexSearchTimeFilePath = null; - private String falsePositiveFilePath = null; - private String cacheMissFilePath = null; - private BufferedReader brIndexSearchTime; - private BufferedReader brFalsePositive; - private BufferedReader brCacheMiss; - private String line; - - public ProfilerReportBuilder(String indexSearchTimeFilePath, String falsePositiveFilePath, String cacheMissFilePath) { - this.indexSearchTimeFilePath = indexSearchTimeFilePath; - this.falsePositiveFilePath = falsePositiveFilePath; - this.cacheMissFilePath = cacheMissFilePath; - } - - public String getIdxNumber(boolean getSearchTime, boolean isJoin, int radiusIdx, int indexIdx) throws Exception { - if (getSearchTime) { - openIndexSearchTimeFile(); - } else { - openCacheMissFile(); - } - - StringBuilder sb = new StringBuilder(); - int involvedIndexCount = isJoin ? JOIN_QUERY_INVOLVED_INDEX_COUNT : SELECT_QUERY_INVOLVED_INDEX_COUNT; - int initialSkip = (isJoin ? IDX_INITIAL_JOIN_SKIP : IDX_INITIAL_SELECT_SKIP) + radiusIdx * involvedIndexCount - * PARTITION_COUNT + indexIdx * PARTITION_COUNT; - int radiusSkip = isJoin ? IDX_JOIN_RADIUS_SKIP : IDX_SELECT_RADIUS_SKIP; - long measuredValue = 0; - BufferedReader br = getSearchTime ? brIndexSearchTime : brCacheMiss; - int lineNum = 0; - int queryCount = isJoin ? JOIN_QUERY_COUNT / JOIN_RADIUS_TYPE_COUNT : SELECT_QUERY_COUNT - / SELECT_RADIUS_TYPE_COUNT; - try { - - //initial skip - for (int i = 0; i < initialSkip; i++) { - br.readLine(); - ++lineNum; - } - - for (int j = 0; j < queryCount; j++) { - //get target index numbers - for (int i = 0; i < PARTITION_COUNT; i++) { - line = br.readLine(); - measuredValue += Long.parseLong(line); - ++lineNum; - } - - //radius skip - for (int i = 0; i < radiusSkip; i++) { - br.readLine(); - ++lineNum; - } - } - - //System.out.println("lineNum: " + lineNum); - sb.append((double) measuredValue / (PARTITION_COUNT * queryCount)); - return sb.toString(); - } finally { - if (getSearchTime) { - closeIndexSearchTimeFile(); - } else { - closeCacheMissFile(); - } - } - } - - public String getFalsePositives(boolean isJoin, int radiusIdx) throws Exception { - openFalsePositiveFile(); - - StringBuilder sb = new StringBuilder(); - int initialSkip = (isJoin ? FP_INITIAL_JOIN_SKIP : FP_INITIAL_SELECT_SKIP) + radiusIdx * PARTITION_COUNT; - int radiusSkip = isJoin ? FP_JOIN_RADIUS_SKIP : FP_SELECT_RADIUS_SKIP; - long falsePositives = 0; - BufferedReader br = brFalsePositive; - int lineNum = 0; - int queryCount = isJoin ? JOIN_QUERY_COUNT / JOIN_RADIUS_TYPE_COUNT : SELECT_QUERY_COUNT - / SELECT_RADIUS_TYPE_COUNT; - try { - - //initial skip - for (int i = 0; i < initialSkip; i++) { - br.readLine(); - ++lineNum; - } - - for (int j = 0; j < queryCount; j++) { - //get target index numbers - for (int i = 0; i < PARTITION_COUNT; i++) { - line = br.readLine(); - falsePositives += Long.parseLong(line); - ++lineNum; - } - - //radius skip - for (int i = 0; i < radiusSkip; i++) { - br.readLine(); - ++lineNum; - } - } - - //System.out.println("lineNum: " + lineNum); - sb.append((double) falsePositives / (PARTITION_COUNT * queryCount)); - return sb.toString(); - } finally { - closeFalsePositiveFile(); - } - } - - protected void openIndexSearchTimeFile() throws IOException { - brIndexSearchTime = new BufferedReader(new FileReader(indexSearchTimeFilePath)); - } - - protected void closeIndexSearchTimeFile() throws IOException { - if (brIndexSearchTime != null) { - brIndexSearchTime.close(); - } - } - - protected void openFalsePositiveFile() throws IOException { - brFalsePositive = new BufferedReader(new FileReader(falsePositiveFilePath)); - } - - protected void closeFalsePositiveFile() throws IOException { - if (brFalsePositive != null) { - brFalsePositive.close(); - } - } - - protected void openCacheMissFile() throws IOException { - brCacheMiss = new BufferedReader(new FileReader(cacheMissFilePath)); - } - - protected void closeCacheMissFile() throws IOException { - if (brCacheMiss != null) { - brCacheMiss.close(); - } - } - -}
