Repository: incubator-eagle Updated Branches: refs/heads/master 668aea0cb -> 0474d5916 (forced update)
[EAGLE-273][EAGLE-269] Fix exceeding the maximum row size of 65,535 bytes in mysql and comparisons between LONG_VARCHAR not supported https://issues.apache.org/jira/browse/EAGLE-273 https://issues.apache.org/jira/browse/EAGLE-269 The problem is cause the limitation of maximum row size in mysql innodb engine, so need to keep the column size relatively small, but it will cause it can't store large field, it's a little tricky, so may have following possible solutions: * Solution One: Fix mysql innodb size to resolve the limitation. * Solution Two: Keep the field relatively small to avoid exceeding maximum row size and make sure creating table successfully, and modify field size according to actual usage (Which is the approach we take for this problem) Author: Hao Chen <h...@apache.org> Closes #158 from haoch/EAGLE-273-LATEST. Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/9849303e Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/9849303e Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/9849303e Branch: refs/heads/master Commit: 9849303ebf8a63e4f54f0eb035c8060c43e50611 Parents: f90a0a4 Author: Hao Chen <h...@apache.org> Authored: Fri May 27 19:34:32 2016 +0800 Committer: Hao Chen <h...@apache.org> Committed: Fri May 27 19:34:32 2016 +0800 ---------------------------------------------------------------------- .../eagle/storage/jdbc/JdbcConstants.java | 5 ++- .../schema/JdbcEntityDefinitionManager.java | 3 +- .../jdbc/schema/JdbcEntitySchemaManager.java | 6 +-- .../eagle/storage/jdbc/TestJdbcStorage.java | 19 +++++++++ .../bin/quick-start-with-external-webservice.sh | 45 ++++++++++++++++++++ .../resources/safemodecheck-policy-import.sh | 2 +- 6 files changed, 73 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/9849303e/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/JdbcConstants.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/JdbcConstants.java b/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/JdbcConstants.java index a6d0c93..ecf2845 100644 --- a/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/JdbcConstants.java +++ b/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/JdbcConstants.java @@ -28,7 +28,10 @@ public class JdbcConstants { public static final String ROW_KEY_COLUMN_NAME = "uuid"; public static final int DEFAULT_TYPE_FOR_COMPLEX_TYPE = Types.BLOB; - public static final int DEFAULT_VARCHAR_SIZE =30000; + + //MySQL Max Row Size: 21485 + public static final int DEFAULT_FIELD_VARCHAR_SIZE =1000; + public static final int DEFAULT_TAG_VARCHAR_SIZE =254; // Eagle JDBC Storage Configuration public final static String EAGLE_DB_USERNAME = "eagle.service.storage-username"; http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/9849303e/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/schema/JdbcEntityDefinitionManager.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/schema/JdbcEntityDefinitionManager.java b/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/schema/JdbcEntityDefinitionManager.java index 15810b2..afcb31c 100644 --- a/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/schema/JdbcEntityDefinitionManager.java +++ b/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/schema/JdbcEntityDefinitionManager.java @@ -43,7 +43,6 @@ public class JdbcEntityDefinitionManager { public static JdbcEntityDefinition getJdbcEntityDefinition(EntityDefinition entityDefinition){ checkInit(); - Class<? extends TaggedLogAPIEntity> entityClass = entityDefinition.getEntityClass(); JdbcEntityDefinition jdbcEntityDefinition = sqlEntityDefinitionCache.get(entityClass); if(jdbcEntityDefinition == null){ @@ -150,7 +149,7 @@ public class JdbcEntityDefinitionManager { // Intially bind basic java types with SQL types //================================================ static { - registerJdbcType(String.class, Types.LONGVARCHAR); + registerJdbcType(String.class, Types.VARCHAR); registerJdbcType(Integer.class, Types.INTEGER); registerJdbcType(Double.class, Types.DOUBLE); registerJdbcType(Float.class, Types.FLOAT); http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/9849303e/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/schema/JdbcEntitySchemaManager.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/schema/JdbcEntitySchemaManager.java b/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/schema/JdbcEntitySchemaManager.java index 4cd1967..60fd810 100644 --- a/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/schema/JdbcEntitySchemaManager.java +++ b/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/schema/JdbcEntitySchemaManager.java @@ -155,7 +155,7 @@ public class JdbcEntitySchemaManager implements IJdbcEntityDDLManager { tagColumn.setTypeCode(Types.VARCHAR); tagColumn.setJavaName(tagName); // tagColumn.setScale(1024); - tagColumn.setSize(String.valueOf(JdbcConstants.DEFAULT_VARCHAR_SIZE)); + tagColumn.setSize(String.valueOf(JdbcConstants.DEFAULT_FIELD_VARCHAR_SIZE)); tagColumn.setDefaultValue(null); tagColumn.setDescription("eagle entity tag column for "+tagName); return tagColumn; @@ -196,7 +196,7 @@ public class JdbcEntitySchemaManager implements IJdbcEntityDDLManager { // Index index = new UniqueIndex(); for (String tag : entityDefinition.getInternal().getTags()) { Column tagColumn = createTagColumn(tag); - tagColumn.setSize(String.valueOf(JdbcConstants.DEFAULT_VARCHAR_SIZE)); + tagColumn.setSize(String.valueOf(JdbcConstants.DEFAULT_TAG_VARCHAR_SIZE)); table.addColumn(tagColumn); // IndexColumn indexColumn = new IndexColumn(); // indexColumn.setName(tag); @@ -214,7 +214,7 @@ public class JdbcEntitySchemaManager implements IJdbcEntityDDLManager { fieldColumn.setJavaName(entry.getKey()); Integer typeCode = entityDefinition.getJdbcColumnTypeCodeOrNull(entry.getKey()); typeCode = typeCode == null? Types.VARCHAR:typeCode; - if(typeCode == Types.VARCHAR) fieldColumn.setSize(String.valueOf(JdbcConstants.DEFAULT_VARCHAR_SIZE)); + if(typeCode == Types.VARCHAR) fieldColumn.setSize(String.valueOf(JdbcConstants.DEFAULT_FIELD_VARCHAR_SIZE)); fieldColumn.setTypeCode(typeCode); fieldColumn.setDescription("eagle field column "+entry.getKey()+":"+entityDefinition.getColumnTypeOrNull(entry.getKey())); table.addColumn(fieldColumn); http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/9849303e/eagle-core/eagle-query/eagle-storage-jdbc/src/test/java/org/apache/eagle/storage/jdbc/TestJdbcStorage.java ---------------------------------------------------------------------- diff --git a/eagle-core/eagle-query/eagle-storage-jdbc/src/test/java/org/apache/eagle/storage/jdbc/TestJdbcStorage.java b/eagle-core/eagle-query/eagle-storage-jdbc/src/test/java/org/apache/eagle/storage/jdbc/TestJdbcStorage.java index 005a61c..f33bc64 100644 --- a/eagle-core/eagle-query/eagle-storage-jdbc/src/test/java/org/apache/eagle/storage/jdbc/TestJdbcStorage.java +++ b/eagle-core/eagle-query/eagle-storage-jdbc/src/test/java/org/apache/eagle/storage/jdbc/TestJdbcStorage.java @@ -54,6 +54,25 @@ public class TestJdbcStorage extends JdbcStorageTestBase { Assert.assertNotNull(result); } + /** + * To make sure not exception like Comparisons between 'LONG VARCHAR (UCS_BASIC)' and 'LONG VARCHAR (UCS_BASIC)' are not supported + * + * @throws QueryCompileException + * @throws IOException + */ + @Test + public void testReadByStringTypedField() throws QueryCompileException, IOException { + RawQuery rawQuery = new RawQuery(); + rawQuery.setQuery("TestTimeSeriesAPIEntity[@cluster=\"c4ut\" AND @field7 =\"some_field7_value\"]{*}"); + System.out.println(DateTimeUtil.millisecondsToHumanDateWithSeconds(baseTimestamp)); + rawQuery.setStartTime(DateTimeUtil.millisecondsToHumanDateWithSeconds(baseTimestamp)); + rawQuery.setEndTime(DateTimeUtil.millisecondsToHumanDateWithMilliseconds(baseTimestamp+2000)); + rawQuery.setPageSize(1000); + CompiledQuery query = new CompiledQuery(rawQuery); + QueryResult<TestTimeSeriesAPIEntity> result = storage.query(query, entityDefinition); + Assert.assertNotNull(result); + } + @Test public void testReadByNotEqualCondition() throws QueryCompileException, IOException { RawQuery rawQuery = new RawQuery(); http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/9849303e/eagle-examples/eagle-cassandra-example/bin/quick-start-with-external-webservice.sh ---------------------------------------------------------------------- diff --git a/eagle-examples/eagle-cassandra-example/bin/quick-start-with-external-webservice.sh b/eagle-examples/eagle-cassandra-example/bin/quick-start-with-external-webservice.sh new file mode 100755 index 0000000..e0646fe --- /dev/null +++ b/eagle-examples/eagle-cassandra-example/bin/quick-start-with-external-webservice.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +export EAGLE_CURRENT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +export EAGLE_BASE_DIR=$EAGLE_CURRENT_DIR/../../.. +export EAGLE_ASSEMBLY_DIR=${EAGLE_BASE_DIR}/eagle-assembly/target/eagle-*-bin/eagle-*/ + +ls ${EAGLE_ASSEMBLY_DIR} 1>/dev/null 2>/dev/null +if [ "$?" != "0" ];then + echo "$EAGLE_ASSEMBLY_DIR not exist, build now" + cd $EAGLE_BASE_DIR + mvn package -DskipTests +fi + +cd $EAGLE_ASSEMBLY_DIR/ + +bin/eagle-topology-init.sh + +echo "Starting zookeeper" +bin/zookeeper-server-start.sh -daemon conf/zookeeper-server.properties +sleep 1 + +echo "Starting kafka" +bin/kafka-server-start.sh -daemon conf/kafka-server.properties +sleep 1 + +echo "Creating kafka topic: cassandra_querylog_local" +bin/kafka-topics.sh --zookeeper localhost:2181 --create --topic cassandra_querylog_local --partitions 3 --replication-factor 1 + +$EAGLE_CURRENT_DIR/init.sh +$EAGLE_CURRENT_DIR/send-sample-querylog.sh +$EAGLE_ASSEMBLY_DIR/bin/kafka-stream-monitor.sh cassandraQueryLogStream cassandraQueryLogExecutor $EAGLE_CURRENT_DIR/../conf/cassandra-security-local.conf \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/9849303e/eagle-hadoop-metric/src/main/resources/safemodecheck-policy-import.sh ---------------------------------------------------------------------- diff --git a/eagle-hadoop-metric/src/main/resources/safemodecheck-policy-import.sh b/eagle-hadoop-metric/src/main/resources/safemodecheck-policy-import.sh index 32a6bee..1fb3831 100644 --- a/eagle-hadoop-metric/src/main/resources/safemodecheck-policy-import.sh +++ b/eagle-hadoop-metric/src/main/resources/safemodecheck-policy-import.sh @@ -48,4 +48,4 @@ curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:a echo "" echo "Finished initialization for eagle topology" -exit 0 +exit 0 \ No newline at end of file
