Author: koji
Date: Thu Dec 20 14:32:46 2012
New Revision: 1424493
URL: http://svn.apache.org/viewvc?rev=1424493&view=rev
Log:
make more general
Added:
labs/alike/trunk/src/java/org/apache/alike/HistogramMatcher.java
labs/alike/trunk/src/java/org/apache/alike/LeastSquaresHistogramMatcher.java
labs/alike/trunk/src/java/org/apache/alike/LuceneIndexerBase.java
labs/alike/trunk/src/java/org/apache/alike/SolrStandardXMLIndexer.java
Modified:
labs/alike/trunk/demo/demo-conf.xml
labs/alike/trunk/src/java/org/apache/alike/AlikeConfig.java
labs/alike/trunk/src/java/org/apache/alike/QuantizeVectors.java
labs/alike/trunk/src/test/org/apache/alike/AlikeConfigTest.java
labs/alike/trunk/src/test/test-files/valid-conf.xml
Modified: labs/alike/trunk/demo/demo-conf.xml
URL:
http://svn.apache.org/viewvc/labs/alike/trunk/demo/demo-conf.xml?rev=1424493&r1=1424492&r2=1424493&view=diff
==============================================================================
--- labs/alike/trunk/demo/demo-conf.xml (original)
+++ labs/alike/trunk/demo/demo-conf.xml Thu Dec 20 14:32:46 2012
@@ -38,13 +38,13 @@
</clustering>
<vectorQuantization>
+ <histogramMatcher class="org.apache.alike.LeastSquaresHistogramMatcher"/>
<fieldNames>
<imageFileFieldName>imgFile</imageFileFieldName>
<queryFieldName>query</queryFieldName>
<histogramFieldName>histogram</histogramFieldName>
</fieldNames>
<indexer class="org.apache.alike.SolrStandardXMLIndexer">
- <histogramMatcher class="org.apache.alike.LeastSquaresHistogramMatcher"/>
<param name="file">solr-demo-data.xml</param>
</indexer>
</vectorQuantization>
Modified: labs/alike/trunk/src/java/org/apache/alike/AlikeConfig.java
URL:
http://svn.apache.org/viewvc/labs/alike/trunk/src/java/org/apache/alike/AlikeConfig.java?rev=1424493&r1=1424492&r2=1424493&view=diff
==============================================================================
--- labs/alike/trunk/src/java/org/apache/alike/AlikeConfig.java (original)
+++ labs/alike/trunk/src/java/org/apache/alike/AlikeConfig.java Thu Dec 20
14:32:46 2012
@@ -17,6 +17,9 @@
package org.apache.alike;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+
import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathExpressionException;
@@ -85,11 +88,37 @@ public final class AlikeConfig {
return getStringValue("/config/clustering/dump/@file");
}
- public String getIndexerSolrXMLFile(){
- return
getStringValue("/config/vectorQuantization/indexer/param[@name='file']/text()");
+ public String getHistogramMatcherClass(){
+ return
getStringValue("/config/vectorQuantization/histogramMatcher/@class");
+ }
+
+ public HistogramMatcher getHistogramMatcher(){
+ ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
+ try {
+ Class<HistogramMatcher> clazz = (Class<HistogramMatcher>)
classLoader.loadClass(getHistogramMatcherClass());
+ Constructor<HistogramMatcher> cnst = clazz.getConstructor();
+ return cnst.newInstance();
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public String getLuceneIndexerClass(){
+ return getStringValue("/config/vectorQuantization/indexer/@class");
+ }
+
+ public LuceneIndexerBase getLuceneIndexer(){
+ ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
+ try {
+ Class<LuceneIndexerBase> clazz = (Class<LuceneIndexerBase>)
classLoader.loadClass(getLuceneIndexerClass());
+ Constructor<LuceneIndexerBase> cnst =
clazz.getConstructor(AlikeConfig.class);
+ return cnst.newInstance(this);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
}
- private String getStringValue(String exp){
+ public String getStringValue(String exp){
try {
return ((String)xpath.evaluate(exp, is, XPathConstants.STRING)).trim();
} catch (XPathExpressionException e) {
Added: labs/alike/trunk/src/java/org/apache/alike/HistogramMatcher.java
URL:
http://svn.apache.org/viewvc/labs/alike/trunk/src/java/org/apache/alike/HistogramMatcher.java?rev=1424493&view=auto
==============================================================================
--- labs/alike/trunk/src/java/org/apache/alike/HistogramMatcher.java (added)
+++ labs/alike/trunk/src/java/org/apache/alike/HistogramMatcher.java Thu Dec 20
14:32:46 2012
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.alike;
+
+public abstract class HistogramMatcher {
+
+ public abstract double computeSimilarity(double[] centroid, double[] desc);
+}
Added:
labs/alike/trunk/src/java/org/apache/alike/LeastSquaresHistogramMatcher.java
URL:
http://svn.apache.org/viewvc/labs/alike/trunk/src/java/org/apache/alike/LeastSquaresHistogramMatcher.java?rev=1424493&view=auto
==============================================================================
---
labs/alike/trunk/src/java/org/apache/alike/LeastSquaresHistogramMatcher.java
(added)
+++
labs/alike/trunk/src/java/org/apache/alike/LeastSquaresHistogramMatcher.java
Thu Dec 20 14:32:46 2012
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.alike;
+
+public class LeastSquaresHistogramMatcher extends HistogramMatcher {
+
+ public double computeSimilarity(double[] centroid, double[] desc) {
+ int D = centroid.length;
+ double sum = 0;
+ for(int i = 0; i < D; i++){
+ sum += (centroid[i] - desc[i]) * (centroid[i] - desc[i]);
+ }
+ return Math.sqrt(sum);
+ }
+}
Added: labs/alike/trunk/src/java/org/apache/alike/LuceneIndexerBase.java
URL:
http://svn.apache.org/viewvc/labs/alike/trunk/src/java/org/apache/alike/LuceneIndexerBase.java?rev=1424493&view=auto
==============================================================================
--- labs/alike/trunk/src/java/org/apache/alike/LuceneIndexerBase.java (added)
+++ labs/alike/trunk/src/java/org/apache/alike/LuceneIndexerBase.java Thu Dec
20 14:32:46 2012
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.alike;
+
+import java.util.Map;
+
+public abstract class LuceneIndexerBase {
+
+ protected AlikeConfig config;
+
+ public LuceneIndexerBase(AlikeConfig config){
+ this.config = config;
+ }
+
+ public abstract void createIndex(Map<String, int[]> histograms) throws
Exception;
+
+ protected String getImageFileFieldName(){
+ return
config.getStringValue("/config/vectorQuantization/fieldNames/imageFileFieldName/text()");
+ }
+
+ protected String getQueryFieldName(){
+ return
config.getStringValue("/config/vectorQuantization/fieldNames/queryFieldName/text()");
+ }
+
+ protected String getHistogramFieldName(){
+ return
config.getStringValue("/config/vectorQuantization/fieldNames/histogramFieldName/text()");
+ }
+}
Modified: labs/alike/trunk/src/java/org/apache/alike/QuantizeVectors.java
URL:
http://svn.apache.org/viewvc/labs/alike/trunk/src/java/org/apache/alike/QuantizeVectors.java?rev=1424493&r1=1424492&r2=1424493&view=diff
==============================================================================
--- labs/alike/trunk/src/java/org/apache/alike/QuantizeVectors.java (original)
+++ labs/alike/trunk/src/java/org/apache/alike/QuantizeVectors.java Thu Dec 20
14:32:46 2012
@@ -21,7 +21,6 @@ import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
-import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
@@ -49,7 +48,7 @@ public class QuantizeVectors {
*
* @see {@link AlikeConfig}
*/
- public static void main(String[] args) throws IOException {
+ public static void main(String[] args) throws Exception {
if(args.length != 1){
printUsage(1);
}
@@ -58,17 +57,18 @@ public class QuantizeVectors {
K = config.getNumOfClusters();
D = config.getNumOfDimensions();
+ LuceneIndexerBase indexer = config.getLuceneIndexer();
// read cluster centroids
double[][] centroids = getCentroids(config.getClusterDumpFile());
// make histograms
- HistogramExecutor executor = new HistogramExecutor(centroids);
+ HistogramExecutor executor =
+ new HistogramExecutor(centroids, config.getHistogramMatcher());
FileUtil.executeRecursively(executor, config.getDescNormalFSDir());
Map<String, int[]> histograms = executor.getHistograms();
- // create Solr "standard" XML file
- createForSolr(config.getIndexerSolrXMLFile(), histograms);
+ indexer.createIndex(histograms);
}
static void printUsage(int exit){
@@ -125,10 +125,12 @@ public class QuantizeVectors {
private double[][] centroids;
Map<String, int[]> histogramMap;
+ private HistogramMatcher histogramMatcher;
- public HistogramExecutor(double[][] centroids){
+ public HistogramExecutor(double[][] centroids, HistogramMatcher
histogramMatcher){
this.centroids = centroids;
histogramMap = new HashMap<String, int[]>();
+ this.histogramMatcher = histogramMatcher;
}
@Override
@@ -167,7 +169,7 @@ public class QuantizeVectors {
double minDistance = Double.MAX_VALUE;
int pos = Integer.MAX_VALUE;
for(int i = 0; i < K; i++){
- double distance = computeSimilarity(centroids[i], desc);
+ double distance = histogramMatcher.computeSimilarity(centroids[i],
desc);
if(minDistance > distance){
minDistance = distance;
pos = i;
@@ -183,14 +185,6 @@ public class QuantizeVectors {
}
}
- static double computeSimilarity(double[] centroid, double[] desc){
- double sum = 0;
- for(int i = 0; i < D; i++){
- sum += (centroid[i] - desc[i]) * (centroid[i] - desc[i]);
- }
- return Math.sqrt(sum);
- }
-
static void printHistograms(Map<String, int[]> histograms){
for(String key : histograms.keySet()){
System.out.println("\n------------------------------------------------------------");
@@ -206,57 +200,6 @@ public class QuantizeVectors {
}
}
- static void createForSolr(String ofile, Map<String, int[]> histograms)
throws IOException {
- PrintWriter pw = null;
- try{
- pw = new PrintWriter(ofile);
- pw.println("<add>");
- for(String key : histograms.keySet()){
- pw.println("<doc>");
- printImageFileNameField(pw, key);
- printAssembledQueryField(pw, histograms.get(key));
- printHistogramField(pw, histograms.get(key));
- pw.println("</doc>");
- }
- pw.println("</add>");
- }
- finally{
- IOUtils.closeQuietly(pw);
- }
- }
-
- private static void printImageFileNameField(PrintWriter pw, String key)
throws IOException {
- printField(pw, "imgFile", key);
- }
-
- private static void printAssembledQueryField(PrintWriter pw, int[]
histogram) throws IOException {
- StringBuilder sb = new StringBuilder();
- for(int i = 0; i < histogram.length; i++){
- if(histogram[i] > 0){
- String q = Integer.toString(i) + "^" + Integer.toString(histogram[i]);
- sb.append(q).append(' ');
- }
- }
-
- printField(pw, "query", sb.toString().trim());
- }
-
- private static void printHistogramField(PrintWriter pw, int[] histogram)
throws IOException {
- StringBuilder sb = new StringBuilder();
- for(int i = 0; i < histogram.length; i++){
- int v = histogram[i];
- for(int j = 0; j < v; j++){
- sb.append(Integer.toString(i)).append(' ');
- }
- }
-
- printField(pw, "histogram", sb.toString().trim());
- }
-
- static void printField(PrintWriter pw, String name, String value) throws
IOException {
- pw.printf(" <field name=\"%s\">%s</field>\n", name, value);
- }
-
static void test(Map<String, int[]> histograms){
Comparator<KeyScorePair> c = new KeyScorePairComparator();
for(String key : histograms.keySet()){
Added: labs/alike/trunk/src/java/org/apache/alike/SolrStandardXMLIndexer.java
URL:
http://svn.apache.org/viewvc/labs/alike/trunk/src/java/org/apache/alike/SolrStandardXMLIndexer.java?rev=1424493&view=auto
==============================================================================
--- labs/alike/trunk/src/java/org/apache/alike/SolrStandardXMLIndexer.java
(added)
+++ labs/alike/trunk/src/java/org/apache/alike/SolrStandardXMLIndexer.java Thu
Dec 20 14:32:46 2012
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.alike;
+
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.Map;
+
+import org.apache.commons.io.IOUtils;
+
+/**
+ * Create Solr "standard" XML file. Uses need to "post" the output XML file
+ * to Solr in order to make an index.
+ *
+ */
+public class SolrStandardXMLIndexer extends LuceneIndexerBase {
+
+ public SolrStandardXMLIndexer(AlikeConfig config) {
+ super(config);
+ }
+
+ public void createIndex(Map<String, int[]> histograms) throws Exception {
+ PrintWriter pw = null;
+ try{
+ pw = new
PrintWriter(config.getStringValue("/config/vectorQuantization/indexer/param[@name='file']/text()"));
+ pw.println("<add>");
+ for(String key : histograms.keySet()){
+ pw.println("<doc>");
+ printImageFileNameField(pw, key);
+ printAssembledQueryField(pw, histograms.get(key));
+ printHistogramField(pw, histograms.get(key));
+ pw.println("</doc>");
+ }
+ pw.println("</add>");
+ }
+ finally{
+ IOUtils.closeQuietly(pw);
+ }
+ }
+
+ private void printImageFileNameField(PrintWriter pw, String key) throws
IOException {
+ printField(pw, getImageFileFieldName(), key);
+ }
+
+ private void printAssembledQueryField(PrintWriter pw, int[] histogram)
throws IOException {
+ StringBuilder sb = new StringBuilder();
+ for(int i = 0; i < histogram.length; i++){
+ if(histogram[i] > 0){
+ String q = Integer.toString(i) + "^" + Integer.toString(histogram[i]);
+ sb.append(q).append(' ');
+ }
+ }
+
+ printField(pw, getQueryFieldName(), sb.toString().trim());
+ }
+
+ private void printHistogramField(PrintWriter pw, int[] histogram) throws
IOException {
+ StringBuilder sb = new StringBuilder();
+ for(int i = 0; i < histogram.length; i++){
+ int v = histogram[i];
+ for(int j = 0; j < v; j++){
+ sb.append(Integer.toString(i)).append(' ');
+ }
+ }
+
+ printField(pw, getHistogramFieldName(), sb.toString().trim());
+ }
+
+ void printField(PrintWriter pw, String name, String value) throws
IOException {
+ pw.printf(" <field name=\"%s\">%s</field>\n", name, value);
+ }
+}
Modified: labs/alike/trunk/src/test/org/apache/alike/AlikeConfigTest.java
URL:
http://svn.apache.org/viewvc/labs/alike/trunk/src/test/org/apache/alike/AlikeConfigTest.java?rev=1424493&r1=1424492&r2=1424493&view=diff
==============================================================================
--- labs/alike/trunk/src/test/org/apache/alike/AlikeConfigTest.java (original)
+++ labs/alike/trunk/src/test/org/apache/alike/AlikeConfigTest.java Thu Dec 20
14:32:46 2012
@@ -18,6 +18,7 @@
package org.apache.alike;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -87,7 +88,22 @@ public class AlikeConfigTest {
}
@Test
- public void testGetIndexerSolrXMLFile() throws Exception {
- assertEquals("solr-demo-data.xml", config.getIndexerSolrXMLFile());
+ public void testGetHistogramMatcherClass() throws Exception {
+ assertEquals("org.apache.alike.LeastSquaresHistogramMatcher",
config.getHistogramMatcherClass());
+ }
+
+ @Test
+ public void testGetHistogramMatcher() throws Exception {
+ assertTrue(config.getHistogramMatcher() instanceof
LeastSquaresHistogramMatcher);
+ }
+
+ @Test
+ public void testGetLuceneIndexerClass() throws Exception {
+ assertEquals("org.apache.alike.SolrStandardXMLIndexer",
config.getLuceneIndexerClass());
+ }
+
+ @Test
+ public void testGetLuceneIndexer() throws Exception {
+ assertTrue(config.getLuceneIndexer() instanceof SolrStandardXMLIndexer);
}
}
Modified: labs/alike/trunk/src/test/test-files/valid-conf.xml
URL:
http://svn.apache.org/viewvc/labs/alike/trunk/src/test/test-files/valid-conf.xml?rev=1424493&r1=1424492&r2=1424493&view=diff
==============================================================================
--- labs/alike/trunk/src/test/test-files/valid-conf.xml (original)
+++ labs/alike/trunk/src/test/test-files/valid-conf.xml Thu Dec 20 14:32:46 2012
@@ -38,13 +38,13 @@
</clustering>
<vectorQuantization>
+ <histogramMatcher class="org.apache.alike.LeastSquaresHistogramMatcher"/>
<fieldNames>
<imageFileFieldName>imgFile</imageFileFieldName>
<queryFieldName>query</queryFieldName>
<histogramFieldName>histogram</histogramFieldName>
</fieldNames>
<indexer class="org.apache.alike.SolrStandardXMLIndexer">
- <histogramMatcher class="org.apache.alike.LeastSquaresHistogramMatcher"/>
<param name="file">solr-demo-data.xml</param>
</indexer>
</vectorQuantization>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
