Repository: asterixdb
Updated Branches:
  refs/heads/master 9ddf5e071 -> cb92dad73


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/cb92dad7/asterixdb/asterix-tools/src/test/java/org/apache/asterix/tools/external/data/TweetGeneratorForSpatialIndexEvaluation.java
----------------------------------------------------------------------
diff --git 
a/asterixdb/asterix-tools/src/test/java/org/apache/asterix/tools/external/data/TweetGeneratorForSpatialIndexEvaluation.java
 
b/asterixdb/asterix-tools/src/test/java/org/apache/asterix/tools/external/data/TweetGeneratorForSpatialIndexEvaluation.java
new file mode 100644
index 0000000..fcaa2a4
--- /dev/null
+++ 
b/asterixdb/asterix-tools/src/test/java/org/apache/asterix/tools/external/data/TweetGeneratorForSpatialIndexEvaluation.java
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.tools.external.data;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.util.Map;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import 
org.apache.asterix.tools.external.data.DataGeneratorForSpatialIndexEvaluation.InitializationInfo;
+import 
org.apache.asterix.tools.external.data.DataGeneratorForSpatialIndexEvaluation.TweetMessage;
+import 
org.apache.asterix.tools.external.data.DataGeneratorForSpatialIndexEvaluation.TweetMessageIterator;
+
+public class TweetGeneratorForSpatialIndexEvaluation {
+
+    private static Logger LOGGER = 
Logger.getLogger(TweetGeneratorForSpatialIndexEvaluation.class.getName());
+
+    public static final String KEY_DURATION = "duration";
+    public static final String KEY_TPS = "tps";
+    public static final String KEY_GUID_SEED = "guid-seed";
+    public static final String KEY_OPENSTREETMAP_FILEPATH = 
"open-street-map-filepath";
+    public static final String KEY_LOCATION_SAMPLE_INTERVAL = 
"location-sample-interval";
+
+    public static final String OUTPUT_FORMAT = "output-format";
+    public static final String OUTPUT_FORMAT_ARECORD = "arecord";
+    public static final String OUTPUT_FORMAT_ADM_STRING = "adm-string";
+
+    private static final int DEFAULT_DURATION = 60; //seconds
+    private static final int DEFAULT_GUID_SEED = 0;
+    private static final int DEFAULT_SAMPLE_INTERVAL = 1;
+
+    private int duration;
+    private TweetMessageIterator tweetIterator = null;
+    private int partition;
+    private int tweetCount = 0;
+    private int frameTweetCount = 0;
+    private int numFlushedTweets = 0;
+    private OutputStream os;
+    private DataGeneratorForSpatialIndexEvaluation dataGenerator = null;
+    private ByteBuffer outputBuffer = ByteBuffer.allocate(32 * 1024);
+    private GULongIDGenerator uidGenerator;
+    private String openStreetMapFilePath;
+    private int locationSampleInterval;
+
+    public int getTweetCount() {
+        return tweetCount;
+    }
+
+    public TweetGeneratorForSpatialIndexEvaluation(Map<String, String> 
configuration, int partition, String format,
+            OutputStream os) throws Exception {
+        this.partition = partition;
+        String value = configuration.get(KEY_DURATION);
+        this.duration = value != null ? Integer.parseInt(value) : 
DEFAULT_DURATION;
+        openStreetMapFilePath = configuration.get(KEY_OPENSTREETMAP_FILEPATH);
+        String lsi = configuration.get(KEY_LOCATION_SAMPLE_INTERVAL);
+        this.locationSampleInterval = lsi != null ? Integer.parseInt(lsi) : 
DEFAULT_SAMPLE_INTERVAL;
+        int guidSeed = configuration.get(KEY_GUID_SEED) != null ? 
Integer.parseInt(configuration.get(KEY_GUID_SEED))
+                : DEFAULT_GUID_SEED;
+        uidGenerator = new GULongIDGenerator(partition, (byte) (guidSeed));
+        dataGenerator = new DataGeneratorForSpatialIndexEvaluation(new 
InitializationInfo(), openStreetMapFilePath,
+                locationSampleInterval);
+        tweetIterator = dataGenerator.new TweetMessageIterator(duration, 
uidGenerator);
+        this.os = os;
+    }
+
+    private void writeTweetString(TweetMessage tweetMessage) throws 
IOException {
+        String tweet = tweetMessage.toString() + "\n";
+        tweetCount++;
+        byte[] b = tweet.getBytes();
+        if (outputBuffer.position() + b.length > outputBuffer.limit()) {
+            flush();
+            numFlushedTweets += frameTweetCount;
+            frameTweetCount = 0;
+            outputBuffer.put(b);
+        } else {
+            outputBuffer.put(b);
+        }
+        frameTweetCount++;
+    }
+
+    public int getNumFlushedTweets() {
+        return numFlushedTweets;
+    }
+
+    private void flush() throws IOException {
+        outputBuffer.flip();
+        os.write(outputBuffer.array(), 0, outputBuffer.limit());
+        outputBuffer.position(0);
+        outputBuffer.limit(32 * 1024);
+    }
+
+    public boolean setNextRecordBatch(int numTweetsInBatch) throws Exception {
+        boolean moreData = tweetIterator.hasNext();
+        if (!moreData) {
+            if (outputBuffer.position() > 0) {
+                flush();
+                numFlushedTweets += frameTweetCount;
+                frameTweetCount = 0;
+            }
+            if (LOGGER.isLoggable(Level.INFO)) {
+                LOGGER.info("Reached end of batch. Tweet Count: [" + partition 
+ "]" + tweetCount);
+            }
+            return false;
+        } else {
+            int count = 0;
+            while (count < numTweetsInBatch) {
+                writeTweetString(tweetIterator.next());
+                count++;
+            }
+            return true;
+        }
+    }
+
+    public void resetDurationAndFlushedTweetCount(int duration) {
+        tweetIterator.resetDuration(duration);
+        numFlushedTweets = 0;
+        tweetCount = 0;
+
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/cb92dad7/asterixdb/asterix-tools/src/test/java/org/apache/asterix/tools/tbltoadm/TblToAdm.java
----------------------------------------------------------------------
diff --git 
a/asterixdb/asterix-tools/src/test/java/org/apache/asterix/tools/tbltoadm/TblToAdm.java
 
b/asterixdb/asterix-tools/src/test/java/org/apache/asterix/tools/tbltoadm/TblToAdm.java
new file mode 100644
index 0000000..26656f6
--- /dev/null
+++ 
b/asterixdb/asterix-tools/src/test/java/org/apache/asterix/tools/tbltoadm/TblToAdm.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.tools.tbltoadm;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.PrintWriter;
+
+public class TblToAdm {
+
+    private static final String SEPARATOR_STRING = "\\|\\s*";
+    private static final char SEPARATOR_CHAR = '|';
+
+    private static void convertFile(String inputFileName, String 
outputFileName, String scmFileName) throws IOException {
+        File scmFile = new File(scmFileName);
+        File inFile = new File(inputFileName);
+        File outFile = new File(outputFileName);
+
+        BufferedReader scmReader = new BufferedReader(new FileReader(scmFile));
+        String scmStr = scmReader.readLine();
+        String[] fieldInfo = scmStr.split(SEPARATOR_STRING);
+        String[] fieldNames = new String[fieldInfo.length];
+        boolean[] fieldIsString = new boolean[fieldInfo.length];
+        for (int i = 0; i < fieldInfo.length; i++) {
+            String[] f = fieldInfo[i].split(":\\s*");
+            fieldNames[i] = f[0];
+            fieldIsString[i] = f[1].equals("string");
+        }
+        scmReader.close();
+
+        BufferedReader reader = new BufferedReader(new FileReader(inFile));
+        PrintWriter writer = new PrintWriter(outFile);
+        String row;
+        while ((row = reader.readLine()) != null) {
+            // String[] fields = row.split(SEPARATOR);
+            if (row.length() == 0 || row.charAt(0) == ' ' || row.charAt(0) == 
'\n') {
+                continue;
+            }
+            writer.write("{  ");
+            int pos = 0;
+            int strlen = row.length();
+            for (int i = 0; i < fieldNames.length; i++) {
+                if (i > 0) {
+                    writer.write(", ");
+                }
+                writer.print('"');
+                writer.write(fieldNames[i]);
+                writer.write("\": ");
+                if (fieldIsString[i]) {
+                    writer.print('"');
+                }
+                for (; pos < strlen; pos++) {
+                    char c = row.charAt(pos);
+                    if (c != SEPARATOR_CHAR) {
+                        writer.print(c);
+
+                    } else {
+                        ++pos;
+                        break;
+                    }
+                }
+                if (fieldIsString[i]) {
+                    writer.print('"');
+                }
+            }
+            writer.write("  }\n");
+        }
+        writer.close();
+        reader.close();
+    }
+
+    public static void main(String args[]) throws Exception {
+        if (args.length != 3) {
+            System.err.println("Usage: bash tbl2adm <srcfile.tbl> 
<destfile.adm> <schema-file>");
+            return;
+        }
+        convertFile(args[0], args[1], args[2]);
+    }
+}

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/cb92dad7/asterixdb/asterix-tools/src/test/java/org/apache/asterix/tools/translator/ADGenDmlTranslator.java
----------------------------------------------------------------------
diff --git 
a/asterixdb/asterix-tools/src/test/java/org/apache/asterix/tools/translator/ADGenDmlTranslator.java
 
b/asterixdb/asterix-tools/src/test/java/org/apache/asterix/tools/translator/ADGenDmlTranslator.java
new file mode 100644
index 0000000..a525b1b
--- /dev/null
+++ 
b/asterixdb/asterix-tools/src/test/java/org/apache/asterix/tools/translator/ADGenDmlTranslator.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.tools.translator;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.asterix.common.annotations.TypeDataGen;
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.lang.common.base.Statement;
+import org.apache.asterix.lang.common.statement.DataverseDecl;
+import org.apache.asterix.lang.common.statement.TypeDecl;
+import org.apache.asterix.metadata.MetadataException;
+import org.apache.asterix.metadata.MetadataTransactionContext;
+import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.om.types.TypeSignature;
+import org.apache.asterix.translator.AbstractLangTranslator;
+import org.apache.asterix.translator.TypeTranslator;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+
+public class ADGenDmlTranslator extends AbstractLangTranslator {
+
+    private final MetadataTransactionContext mdTxnCtx;
+    private final List<Statement> aqlStatements;
+    private Map<TypeSignature, IAType> types;
+    private Map<TypeSignature, TypeDataGen> typeDataGenMap;
+
+    public ADGenDmlTranslator(MetadataTransactionContext mdTxnCtx, 
List<Statement> aqlStatements) {
+        this.mdTxnCtx = mdTxnCtx;
+        this.aqlStatements = aqlStatements;
+    }
+
+    public void translate() throws AsterixException, MetadataException, 
AlgebricksException {
+        String defaultDataverse = getDefaultDataverse();
+        types = new HashMap<TypeSignature, IAType>();
+        typeDataGenMap = new HashMap<TypeSignature, TypeDataGen>();
+
+        for (Statement stmt : aqlStatements) {
+            if (stmt.getKind() == Statement.Kind.TYPE_DECL) {
+                TypeDecl td = (TypeDecl) stmt;
+                String typeDataverse =
+                        td.getDataverseName() == null ? defaultDataverse : 
td.getDataverseName().getValue();
+
+                Map<TypeSignature, IAType> typeInStmt = 
TypeTranslator.computeTypes(mdTxnCtx, td.getTypeDef(),
+                        td.getIdent().getValue(), typeDataverse, types);
+                types.putAll(typeInStmt);
+
+                TypeSignature signature = new TypeSignature(typeDataverse, 
td.getIdent().getValue());
+                TypeDataGen tdg = td.getDatagenAnnotation();
+                if (tdg != null) {
+                    typeDataGenMap.put(signature, tdg);
+                }
+            }
+        }
+    }
+
+    private String getDefaultDataverse() {
+        for (Statement stmt : aqlStatements) {
+            if (stmt.getKind() == Statement.Kind.DATAVERSE_DECL) {
+                return ((DataverseDecl) stmt).getDataverseName().getValue();
+            }
+        }
+        return null;
+    }
+
+    public Map<TypeSignature, IAType> getTypeMap() {
+        return types;
+    }
+
+    public Map<TypeSignature, TypeDataGen> getTypeDataGenMap() {
+        return typeDataGenMap;
+    }
+
+}

Reply via email to