http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/5ae2318d/src/blur-gui/src/main/webapps/controller/functions.jsp ---------------------------------------------------------------------- diff --git a/src/blur-gui/src/main/webapps/controller/functions.jsp b/src/blur-gui/src/main/webapps/controller/functions.jsp new file mode 100644 index 0000000..bbc6e4c --- /dev/null +++ b/src/blur-gui/src/main/webapps/controller/functions.jsp @@ -0,0 +1,25 @@ + +<%!public String row(String... col) { + String ret = "<tr>"; + for (String c : col) + ret += "<td>" + c + "</td>"; + return ret + "</tr>\n"; + } + + public String row(int colspan, String col) { + return "<tr><td colspan=" + colspan + ">" + col + "</td></tr>"; + } + + public String shardListLink(String cluster, String display) { + return "<a href='shardList.jsp?clusterName=" + cluster + + "' title='view shard servers'>" + display + "</a>"; + } + + public String table(String content, String... headers) { + String ret = "<table class='statTable'><tr>\n"; + for(String h : headers) + ret += "<td class='statTableTitle'>" + h + "</td>\n"; + return ret + "</tr>" + content + "</table>\n"; + } + + %> \ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/5ae2318d/src/blur-gui/src/main/webapps/controller/index.html ---------------------------------------------------------------------- diff --git a/src/blur-gui/src/main/webapps/controller/index.html b/src/blur-gui/src/main/webapps/controller/index.html new file mode 100644 index 0000000..b661c3b --- /dev/null +++ b/src/blur-gui/src/main/webapps/controller/index.html @@ -0,0 +1,13 @@ +<meta HTTP-EQUIV="REFRESH" content="0;url=controller.jsp"/> +<html> +<head> +<title>Blur</title> +</head> +<body> +<h1>Blur</h1> + +<ul> +<li><a href="controller.jsp">Blur Controller Health/Status</a></li> +</ul> +</body> +</html> http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/5ae2318d/src/blur-gui/src/main/webapps/controller/shardList.jsp ---------------------------------------------------------------------- diff --git a/src/blur-gui/src/main/webapps/controller/shardList.jsp b/src/blur-gui/src/main/webapps/controller/shardList.jsp new file mode 100644 index 0000000..f539343 --- /dev/null +++ b/src/blur-gui/src/main/webapps/controller/shardList.jsp @@ -0,0 +1,55 @@ +<%@ page contentType="text/html; charset=UTF-8" isThreadSafe="false" + import="javax.servlet.*" import="javax.servlet.http.*" + import="java.io.*" import="java.util.*" import="java.text.DateFormat" + import="java.lang.Math" import="java.net.URLEncoder" + import="com.nearinfinity.blur.thrift.*" + import="com.nearinfinity.blur.thrift.generated.*" + import="com.nearinfinity.blur.thrift.generated.Blur.*"%> +<%@ include file="functions.jsp"%> +<%! + public String shards(Iface client) throws Exception { + String ret = ""; + List<String> servers = client.shardServerList("default"); + + for(String s : servers) { + ret += row(s,"",""); + } + return ret; + } +%> + +<% + //TODO: prop file the port + String hostName = request.getServerName() + ":40010"; + + Iface client = BlurClient.getClient(hostName); + + String clusterName = request.getParameter("clusterName"); +%> + +<html> +<head> +<title>Blur Cluster Shard List '<%=hostName%>' +</title> +<link href="style.css" rel="stylesheet" type="text/css" /> +</head> +<body> + <% + if (clusterName == null) { + %> + No cluster specified, go home. + <% + } else { + %> + <h1> + Blur Shard List for Cluster '<%=clusterName%>' + </h1> + <br /> + <%=table(shards(client),"Shard") %> + + <% + } + %> + <br/> +</body> +</html> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/5ae2318d/src/blur-gui/src/main/webapps/controller/style.css ---------------------------------------------------------------------- diff --git a/src/blur-gui/src/main/webapps/controller/style.css b/src/blur-gui/src/main/webapps/controller/style.css new file mode 100644 index 0000000..cda4fb7 --- /dev/null +++ b/src/blur-gui/src/main/webapps/controller/style.css @@ -0,0 +1,53 @@ +body { + font-family: sans-serif; + font-size: 11px; + line-height: 17px; +} + +a { + color: #0044FF !important; + text-decoration: none !important; +} + +h2 { + margin: 5px 5px 0px 0px; +} + +hr { + -moz-border-bottom-colors: none; + -moz-border-image: none; + -moz-border-left-colors: none; + -moz-border-right-colors: none; + -moz-border-top-colors: none; + border-color: -moz-use-text-color #DDDDDD #DDDDDD; + border-right: 1px solid #DDDDDD; + border-style: none solid solid; + border-width: 0 1px 1px; +} + +table { + font-size: 12px; + +} + +td { + align: left; + font-size: 12px; + padding-left: 10px; + padding-right: 10px; + font-family: Verdana; +} + +.statTable { + width: 500px; +} + +.statTableTitle { + color: 55555; + align: left; + font-weight: bold; + font-size:14pt; + padding: 10px 5px 0px 5px; + font-family: Helvetica; + background-color: #DDDDDD; +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/5ae2318d/src/blur-gui/src/main/webapps/controller/table.jsp ---------------------------------------------------------------------- diff --git a/src/blur-gui/src/main/webapps/controller/table.jsp b/src/blur-gui/src/main/webapps/controller/table.jsp new file mode 100644 index 0000000..4aa15b9 --- /dev/null +++ b/src/blur-gui/src/main/webapps/controller/table.jsp @@ -0,0 +1,147 @@ +<%@page import="java.text.DecimalFormat"%> +<%@page import="java.text.NumberFormat"%> +<%@ page contentType="text/html; charset=UTF-8" isThreadSafe="false" + import="javax.servlet.*" import="javax.servlet.http.*" + import="java.io.*" import="java.util.*" import="java.text.DateFormat" + import="java.lang.Math" import="java.net.URLEncoder" + import="com.nearinfinity.blur.thrift.*" + import="com.nearinfinity.blur.thrift.generated.*" + import="com.nearinfinity.blur.thrift.generated.Blur.*"%> +<%@ include file="functions.jsp"%> +<%!public boolean tableInSafeMode(Iface client, String clusterName) throws Exception { + return client.isInSafeMode(clusterName); + } + + public String getStats(Iface client, String tableName) throws Exception { + DecimalFormat df = new DecimalFormat("#,###,###,###0.00"); + String ret = ""; + + TableStats ts = client.getTableStats(tableName); + String size = ""; + //bytes + if(ts.bytes < 1000) + size = ts.bytes + " bytes"; + //kb + else if(ts.bytes < 1048576) + size = df.format(ts.bytes/1000.0) + " KB"; + else if(ts.bytes < 1073741824) + size = df.format(ts.bytes/1000.0/1000) + " MB"; + else if(ts.bytes < 137438953472l) + size = df.format(ts.bytes/1000.0/1000/1000) + " GB"; + else if(ts.bytes < 1099511627776l) + size = df.format(ts.bytes/1000.0/1000/1000/1000) + " TB"; + ret += row("size", size); + ret += row("Queries", ts.queries + ""); + ret += row("Rows", ts.rowCount + ""); + ret += row("Records", ts.recordCount + ""); + + return ret; + } + + public String getSchema(Iface client, String tableName) throws Exception { + String ret = ""; + + Schema s = client.schema(tableName); + for (String fam : s.columnFamilies.keySet()) { + String tmp = ""; + for (String c : s.columnFamilies.get(fam)) + tmp += c + ", "; + if (!"".equals(tmp)) + tmp = tmp.substring(0, tmp.length() - 2); + ret += row(fam, tmp); + } + + return ret; + } + + public String getAD(Iface client, String tableName) throws Exception { + String ret = ""; + + TableDescriptor td = client.describe(tableName); + AnalyzerDefinition ad = td.analyzerDefinition; + Map<String, ColumnFamilyDefinition> cfds = ad.columnFamilyDefinitions; + for (String cf : cfds.keySet()) { + ColumnFamilyDefinition cfd = cfds.get(cf); + if (cfd.defaultDefinition != null) + ret += row(cf, "default", + cfd.defaultDefinition.analyzerClassName); + else + ret += row(cf, "default", "none set"); + for (String col : cfd.columnDefinitions.keySet()) { + ret += row("", col, + cfd.columnDefinitions.get(col).analyzerClassName); + } + } + + return ret; + }%> +<% + final String NONE = "none given"; + + String hostName = request.getServerName() + ":40010"; + + Iface client = BlurClient.getClient(hostName); + + String tableName = request.getParameter("tableName"); + String clusterName = request.getParameter("clusterName"); + + if (tableName == null || tableName.length() < 1) { + tableName = NONE; + } + + if (clusterName == null || clusterName.length() < 1) { + clusterName = NONE; + } +%> + + +<html> +<head> +<title>Table '<%=tableName%>' +</title> + +<link href="style.css" rel="stylesheet" type="text/css" /> + +</head> + +<body> + <% + if (NONE.equals(clusterName) || NONE.equals(tableName)) { + %> + Dont have a cluster and tableName specified, go home. + <% + } else { + %> + <h1> + Table '<%=tableName%>' + </h1> + <% + if (tableInSafeMode(client, clusterName)) { + %> + Cluster + <%=clusterName%> + is in safe mode, cannot retrieve table information yet. + <% + } else { + %> + + <h2>Stats</h2> + <%=table(getStats(client, tableName), "Stat", + "Value")%> + <br /> + <h2>Schema</h2> + <%=table(getSchema(client, tableName), + "ColumnFamily", "Column")%> + <br /> + <h2>Field Definitions</h2> + <%=table(getAD(client, tableName), "ColumnFamily", + "Column", "Analyzer")%> + + <% + } + } + %> + +</body> +</html> + http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/5ae2318d/src/blur-testsuite/pom.xml ---------------------------------------------------------------------- diff --git a/src/blur-testsuite/pom.xml b/src/blur-testsuite/pom.xml index 9bf9741..c5085fd 100644 --- a/src/blur-testsuite/pom.xml +++ b/src/blur-testsuite/pom.xml @@ -34,6 +34,12 @@ <scope>test</scope> </dependency> <dependency> + <groupId>org.apache.zookeeper</groupId> + <artifactId>zookeeper</artifactId> + <version>3.3.4</version> + <scope>provided</scope> + </dependency> + <dependency> <groupId>log4j</groupId> <artifactId>log4j</artifactId> <version>1.2.15</version> @@ -71,39 +77,41 @@ </repositories> <build> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-compiler-plugin</artifactId> - <configuration> - <source>1.6</source> - <target>1.6</target> - </configuration> - </plugin> + <pluginManagement> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + <configuration> + <source>1.6</source> + <target>1.6</target> + </configuration> + </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-dependency-plugin</artifactId> - <executions> - <execution> - <id>copy-dependencies</id> - <phase>package</phase> - <goals> - <goal>copy-dependencies</goal> - </goals> - <configuration> - <outputDirectory>${project.build.directory}/../../../lib - </outputDirectory> - <overWriteReleases>false</overWriteReleases> - <overWriteSnapshots>false</overWriteSnapshots> - <overWriteIfNewer>true</overWriteIfNewer> - <excludeTransitive>true</excludeTransitive> - <excludeArtifactIds>junit,commons-cli,commons-logging,hadoop-core,slf4j-api,slf4j-log4j12 - </excludeArtifactIds> - </configuration> - </execution> - </executions> - </plugin> - </plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-dependency-plugin</artifactId> + <executions> + <execution> + <id>copy-dependencies</id> + <phase>package</phase> + <goals> + <goal>copy-dependencies</goal> + </goals> + <configuration> + <outputDirectory>${project.build.directory}/../../../lib + </outputDirectory> + <overWriteReleases>false</overWriteReleases> + <overWriteSnapshots>false</overWriteSnapshots> + <overWriteIfNewer>true</overWriteIfNewer> + <excludeTransitive>true</excludeTransitive> + <excludeArtifactIds>junit,commons-cli,commons-logging,hadoop-core,slf4j-api,slf4j-log4j12 + </excludeArtifactIds> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </pluginManagement> </build> </project> http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/5ae2318d/src/blur-testsuite/src/main/java/com/nearinfinity/blur/testsuite/CreateInsertQueryRepeating.java ---------------------------------------------------------------------- diff --git a/src/blur-testsuite/src/main/java/com/nearinfinity/blur/testsuite/CreateInsertQueryRepeating.java b/src/blur-testsuite/src/main/java/com/nearinfinity/blur/testsuite/CreateInsertQueryRepeating.java new file mode 100644 index 0000000..7e6b44c --- /dev/null +++ b/src/blur-testsuite/src/main/java/com/nearinfinity/blur/testsuite/CreateInsertQueryRepeating.java @@ -0,0 +1,246 @@ +package com.nearinfinity.blur.testsuite; + +import java.io.IOException; +import java.text.DecimalFormat; +import java.util.ArrayList; +import java.util.List; +import java.util.Random; +import java.util.UUID; + +import org.apache.thrift.TException; +import org.apache.thrift.async.AsyncMethodCallback; + +import com.nearinfinity.blur.thrift.AsyncClientPool; +import com.nearinfinity.blur.thrift.BlurClient; +import com.nearinfinity.blur.thrift.generated.AnalyzerDefinition; +import com.nearinfinity.blur.thrift.generated.Blur; +import com.nearinfinity.blur.thrift.generated.Blur.AsyncClient.mutateBatch_call; +import com.nearinfinity.blur.thrift.generated.Blur.Iface; +import com.nearinfinity.blur.thrift.generated.BlurException; +import com.nearinfinity.blur.thrift.generated.BlurQuery; +import com.nearinfinity.blur.thrift.generated.ColumnDefinition; +import com.nearinfinity.blur.thrift.generated.ColumnFamilyDefinition; +import com.nearinfinity.blur.thrift.generated.RowMutation; +import com.nearinfinity.blur.thrift.generated.RowMutationType; +import com.nearinfinity.blur.thrift.generated.SimpleQuery; +import com.nearinfinity.blur.thrift.generated.TableDescriptor; + +import static com.nearinfinity.blur.utils.BlurUtil.*; + +public class CreateInsertQueryRepeating { + + private static DecimalFormat df = new DecimalFormat("#,###,000.00"); + private static final char[] symbols = new char[36]; + + static { + for (int idx = 0; idx < 10; ++idx) + symbols[idx] = (char) ('0' + idx); + for (int idx = 10; idx < 36; ++idx) + symbols[idx] = (char) ('a' + idx - 10); + } + + private static final Random random = new Random(); + + public static String randomString(int length) { + char[] buf = new char[length]; + + for (int idx = 0; idx < buf.length; ++idx) + buf[idx] = symbols[random.nextInt(symbols.length)]; + return new String(buf); + } + + public static void getClusters(Iface client) { + try { + List<String> shardClusterList = client.shardClusterList(); + for (String cluster : shardClusterList) + System.out.println("cluster: " + cluster); + } catch (BlurException e) { + e.printStackTrace(); + } catch (TException e) { + e.printStackTrace(); + } + } + + public static void createTable(Iface client, String tableName, + String cluster) throws BlurException, TException { + TableDescriptor td = new TableDescriptor(); + td.analyzerDefinition = new AnalyzerDefinition(); + ColumnFamilyDefinition cfd = new ColumnFamilyDefinition(); + ColumnDefinition c = new ColumnDefinition( + "org.apache.lucene.analysis.standard.StandardAnalyzer", true, + null); + cfd.putToColumnDefinitions("cf1", c); + td.analyzerDefinition.putToColumnFamilyDefinitions("cf1", cfd); + + td.name = tableName; + // TODO: doc doesnt say required, yet it barfs without it? + td.cluster = cluster == null ? "default" : cluster; + td.isEnabled = true; + + td.shardCount = 2; + td.readOnly = false; + // TODO: hardcodes bad + td.tableUri = "hdfs://localhost:8020/" + tableName; + client.createTable(td); + // client.enableTable(tableName); + System.out.println("table created"); + } + + /** + * @param args + * @throws TException + * @throws BlurException + * @throws IOException + */ + public static void main(String[] args) throws BlurException, TException, IOException { + String connectionStr = "localhost:40010"; + final String cluster = "default"; + String uri = "hdfs://localhost:8020/test1"; + int shardCount = 1; + Iface client = BlurClient.getClient(connectionStr); + + List<String> tableList = client.tableList(); + if (tableList == null || !tableList.contains("test1")) + createTable(client, "test1", cluster); + + System.out.println("Starting load"); + loadupTable(client, "test1", 1000); + System.out.println("Finshed load"); + + System.out.println("query time!"); + queryTable(client, "test1", 10000); + System.out.println("query done!"); + + System.out.println("going into auto create test: "); + + System.exit(0); + int i = 0; + while (i++ < 50) { + String tableName = UUID.randomUUID().toString(); + System.out.println("Creating [" + tableName + "]"); + createTable(client, cluster, uri, shardCount, tableName); + System.out.println("Loading [" + tableName + "]"); + loadTable(client, tableName); + System.out.println("Disabling [" + tableName + "]"); + disable(client, tableName); + System.out.println("Removing [" + tableName + "]"); + delete(client, tableName); + } + } + + // really only useful against the table that was filled via loadupTable + private static void queryTable(Iface client, String tableName, int times) + throws BlurException, TException { + long start = System.currentTimeMillis(); + BlurQuery bq = new BlurQuery(); + bq.fetch = 10; + for (int i = 1; i <= times; i++) { + SimpleQuery sq = new SimpleQuery(); + sq.queryStr = "numberField:" + random.nextInt(1000); + sq.superQueryOn = true; + bq.simpleQuery = sq; + client.query(tableName, bq); + if (i % 1000 == 0) { + System.out + .println("queries: " + + i + + " times " + + df.format((i / ((System.currentTimeMillis() + - start + 0.0)/1000))) + " queries/s"); + } + } + System.out + .println("queries: " + + times + + " times " + + df.format((times / ((System.currentTimeMillis() + - start + 0.0)/1000))) + " queries/s"); + + } + + private static void loadupTable(Iface client, String tableName, int rows) + throws BlurException, TException, IOException { + AsyncClientPool pool = new AsyncClientPool(10, 30000); + + + long start = System.currentTimeMillis(); + + List<RowMutation> mutates = new ArrayList<RowMutation>(); + + for (int i = 1; i <= rows; i++) { + RowMutation mutation = new RowMutation(); + mutation.table = tableName; + mutation.waitToBeVisible = true; + mutation.rowId = UUID.randomUUID().toString(); + mutation.addToRecordMutations(newRecordMutation("test", + "test-" + i, + newColumn("uuidField", UUID.randomUUID().toString()), + newColumn("numberField", i + ""), + newColumn("fatTextField",randomString(1000)))); + mutation.rowMutationType = RowMutationType.REPLACE_ROW; + mutates.add(mutation); + + if(mutates.size() == 10) { + pool.getClient(Blur.AsyncIface.class, "localhost:40010").mutateBatch(mutates, + new AsyncMethodCallback<Blur.AsyncClient.mutateBatch_call>() { + + @Override + public void onError(Exception exception) { + exception.printStackTrace(); + } + + @Override + public void onComplete(mutateBatch_call response) { + } + }); + mutates.clear(); + } + if (i % 50 == 0) { + System.out.println("loaded: " + i + " around " + + df.format((i / ((System.currentTimeMillis() - start+0.0)/1000) )) + + " rows/s"); + } + } + System.out.println("loaded: " + rows + " around " + + df.format((rows / ((System.currentTimeMillis() - start+0.0)/1000))) + " rows/s"); + } + + private static void disable(Iface client, String tableName) + throws BlurException, TException { + client.disableTable(tableName); + } + + private static void delete(Iface client, String tableName) + throws BlurException, TException { + client.removeTable(tableName, true); + } + + private static void loadTable(Iface client, String tableName) + throws BlurException, TException { + RowMutation mutation = new RowMutation(); + mutation.table = tableName; + mutation.waitToBeVisible = true; + mutation.rowId = "test"; + mutation.addToRecordMutations(newRecordMutation("test", "test", + newColumn("test", "test"))); + mutation.rowMutationType = RowMutationType.REPLACE_ROW; + client.mutate(mutation); + } + + private static void createTable(Iface client, final String cluster, + String uri, int shardCount, String tableName) throws BlurException, + TException { + final TableDescriptor tableDescriptor = new TableDescriptor(); + tableDescriptor.analyzerDefinition = new AnalyzerDefinition(); + tableDescriptor.cluster = cluster; + + tableDescriptor.name = tableName; + tableDescriptor.readOnly = false; + + tableDescriptor.shardCount = shardCount; + tableDescriptor.tableUri = uri + "/" + tableName; + + client.createTable(tableDescriptor); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/5ae2318d/src/blur-util/src/main/resources/blur-default.properties ---------------------------------------------------------------------- diff --git a/src/blur-util/src/main/resources/blur-default.properties b/src/blur-util/src/main/resources/blur-default.properties index 2b0784a..54eb197 100644 --- a/src/blur-util/src/main/resources/blur-default.properties +++ b/src/blur-util/src/main/resources/blur-default.properties @@ -44,3 +44,6 @@ blur.query.max.record.fetch=1000 blur.zookeeper.system.time.tolerance=3000 +blur.gui.controller.port=40080 +blur.gui.shard.port=40090 +
