This is an automated email from the ASF dual-hosted git repository.
lewismc pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-mudrod.git
The following commit(s) were added to refs/heads/master by this push:
new 74d3a70 SDAP-185: Clean up code base (#43)
74d3a70 is described below
commit 74d3a709dde55e1df60286d7e637e2c119aa64c2
Author: Kevin Ratnasekera <[email protected]>
AuthorDate: Wed Mar 6 22:11:56 2019 +0530
SDAP-185: Clean up code base (#43)
* SDAP-185: Clean up code base
* Improve logging for PODAACMetadata
---
.../mudrod/discoveryengine/RecommendEngine.java | 13 ++
.../discoveryengine/WeblogDiscoveryEngine.java | 10 +-
.../org/apache/sdap/mudrod/driver/ESDriver.java | 12 +-
.../mudrod/integration/LinkageIntegration.java | 14 +-
.../org/apache/sdap/mudrod/main/MudrodEngine.java | 2 +-
.../sdap/mudrod/metadata/pre/ApiHarvester.java | 9 +-
.../sdap/mudrod/metadata/pre/MatrixGenerator.java | 2 +-
.../sdap/mudrod/metadata/structure/Metadata.java | 5 -
.../metadata/structure/MetadataExtractor.java | 5 +-
.../mudrod/metadata/structure/PODAACMetadata.java | 210 ++++++++++-----------
.../org/apache/sdap/mudrod/ontology/Ontology.java | 10 +-
.../sdap/mudrod/ontology/pre/AggregateTriples.java | 27 ++-
.../mudrod/ontology/process/OntologyLinkCal.java | 2 +-
.../mudrod/ontology/process/OntologyParser.java | 4 +-
.../sdap/mudrod/ontology/process/OwlParser.java | 4 +-
.../mudrod/recommendation/pre/ImportMetadata.java | 12 +-
.../recommendation/pre/MetadataTFIDFGenerator.java | 2 +-
.../recommendation/pre/NormalizeFeatures.java | 8 +-
.../recommendation/pre/SessionCooccurence.java | 6 +-
.../process/AbstractBasedSimilarity.java | 2 +-
.../process/FeatureBasedSimilarity.java | 10 +-
.../recommendation/process/SessionBasedCF.java | 6 +-
.../structure/HybridRecommendation.java | 18 +-
.../recommendation/structure/MetadataFeature.java | 6 +-
.../structure/MetadataTokenizer.java | 16 +-
.../structure/PODAACMetadataFeature.java | 18 +-
.../sdap/mudrod/semantics/SemanticAnalyzer.java | 8 +-
.../sdap/mudrod/ssearch/ClickStreamImporter.java | 12 +-
.../org/apache/sdap/mudrod/ssearch/Ranker.java | 6 +-
.../org/apache/sdap/mudrod/ssearch/Searcher.java | 12 +-
.../sdap/mudrod/ssearch/ranking/DataGenerator.java | 29 +--
.../sdap/mudrod/ssearch/ranking/Learner.java | 5 +-
.../mudrod/ssearch/ranking/SparkFormatter.java | 35 +++-
.../mudrod/ssearch/ranking/TrainingImporter.java | 2 +-
.../sdap/mudrod/ssearch/structure/SResult.java | 10 +-
.../apache/sdap/mudrod/tools/EONETIngester.java | 9 +-
.../org/apache/sdap/mudrod/utils/HttpRequest.java | 2 +-
.../apache/sdap/mudrod/utils/LinkageTriple.java | 8 +-
.../org/apache/sdap/mudrod/utils/MatrixUtil.java | 20 +-
.../java/org/apache/sdap/mudrod/utils/SVDUtil.java | 8 +-
.../apache/sdap/mudrod/utils/SimilarityUtil.java | 2 +-
.../weblog/partition/KGreedyPartitionSolver.java | 25 ++-
.../partition/ThePartitionProblemSolver.java | 16 +-
.../mudrod/weblog/partition/logPartitioner.java | 17 +-
.../sdap/mudrod/weblog/pre/CrawlerDetection.java | 2 +-
.../sdap/mudrod/weblog/pre/HistoryGenerator.java | 11 +-
.../sdap/mudrod/weblog/pre/ImportLogFile.java | 25 +--
.../apache/sdap/mudrod/weblog/pre/LogAbstract.java | 19 +-
.../weblog/pre/RankingTrainDataGenerator.java | 15 +-
.../sdap/mudrod/weblog/pre/SessionGenerator.java | 30 +--
.../sdap/mudrod/weblog/pre/SessionStatistic.java | 16 +-
.../weblog/structure/log/ApacheAccessLog.java | 10 +-
.../mudrod/weblog/structure/log/Coordinates.java | 1 +
.../mudrod/weblog/structure/log/RequestUrl.java | 8 +-
.../weblog/structure/session/ClickStream.java | 7 +-
.../weblog/structure/session/RankingTrainData.java | 13 ++
.../mudrod/weblog/structure/session/Session.java | 3 +-
.../weblog/structure/session/SessionExtractor.java | 8 +-
.../weblog/structure/session/SessionNode.java | 13 +-
.../mudrod/ontology/process/TestLocalOntology.java | 6 +-
.../mudrod/services/MudrodContextListener.java | 2 +-
.../autocomplete/AutoCompleteResource.java | 6 +-
.../search/SearchDatasetDetailResource.java | 6 +-
.../services/search/SearchMetadataResource.java | 6 +-
.../services/search/SearchVocabResource.java | 6 +-
.../services/search/SessionDetailResource.java | 7 +-
66 files changed, 535 insertions(+), 344 deletions(-)
diff --git
a/core/src/main/java/org/apache/sdap/mudrod/discoveryengine/RecommendEngine.java
b/core/src/main/java/org/apache/sdap/mudrod/discoveryengine/RecommendEngine.java
index b664c8e..4239e3e 100644
---
a/core/src/main/java/org/apache/sdap/mudrod/discoveryengine/RecommendEngine.java
+++
b/core/src/main/java/org/apache/sdap/mudrod/discoveryengine/RecommendEngine.java
@@ -1,3 +1,16 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.sdap.mudrod.discoveryengine;
import org.apache.sdap.mudrod.driver.ESDriver;
diff --git
a/core/src/main/java/org/apache/sdap/mudrod/discoveryengine/WeblogDiscoveryEngine.java
b/core/src/main/java/org/apache/sdap/mudrod/discoveryengine/WeblogDiscoveryEngine.java
index b2cdb9f..59510db 100644
---
a/core/src/main/java/org/apache/sdap/mudrod/discoveryengine/WeblogDiscoveryEngine.java
+++
b/core/src/main/java/org/apache/sdap/mudrod/discoveryengine/WeblogDiscoveryEngine.java
@@ -20,7 +20,13 @@ import org.apache.hadoop.fs.Path;
import org.apache.sdap.mudrod.driver.ESDriver;
import org.apache.sdap.mudrod.driver.SparkDriver;
import org.apache.sdap.mudrod.main.MudrodConstants;
-import org.apache.sdap.mudrod.weblog.pre.*;
+import org.apache.sdap.mudrod.weblog.pre.ClickStreamGenerator;
+import org.apache.sdap.mudrod.weblog.pre.CrawlerDetection;
+import org.apache.sdap.mudrod.weblog.pre.HistoryGenerator;
+import org.apache.sdap.mudrod.weblog.pre.ImportLogFile;
+import org.apache.sdap.mudrod.weblog.pre.SessionGenerator;
+import org.apache.sdap.mudrod.weblog.pre.SessionStatistic;
+import org.apache.sdap.mudrod.weblog.pre.RemoveRawLog;
import org.apache.sdap.mudrod.weblog.process.ClickStreamAnalyzer;
import org.apache.sdap.mudrod.weblog.process.UserHistoryAnalyzer;
import org.slf4j.Logger;
@@ -44,7 +50,7 @@ public class WeblogDiscoveryEngine extends
DiscoveryEngineAbstract {
*/
private static final long serialVersionUID = 1L;
private static final Logger LOG =
LoggerFactory.getLogger(WeblogDiscoveryEngine.class);
- public String timeSuffix = null;
+ private String timeSuffix = null;
public WeblogDiscoveryEngine(Properties props, ESDriver es, SparkDriver
spark) {
super(props, es, spark);
diff --git a/core/src/main/java/org/apache/sdap/mudrod/driver/ESDriver.java
b/core/src/main/java/org/apache/sdap/mudrod/driver/ESDriver.java
index dc1b4c8..b761d3b 100644
--- a/core/src/main/java/org/apache/sdap/mudrod/driver/ESDriver.java
+++ b/core/src/main/java/org/apache/sdap/mudrod/driver/ESDriver.java
@@ -64,8 +64,15 @@ import java.time.Instant;
import java.time.LocalDate;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
import java.util.Map.Entry;
+import java.util.Properties;
+import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
@@ -588,7 +595,6 @@ public class ESDriver implements Serializable {
public int getDocCount(String[] index, String[] type, QueryBuilder
filterSearch) {
SearchRequestBuilder countSrBuilder =
getClient().prepareSearch(index).setTypes(type).setQuery(filterSearch).setSize(0);
SearchResponse countSr = countSrBuilder.execute().actionGet();
- int docCount = (int) countSr.getHits().getTotalHits();
- return docCount;
+ return (int) countSr.getHits().getTotalHits();
}
}
diff --git
a/core/src/main/java/org/apache/sdap/mudrod/integration/LinkageIntegration.java
b/core/src/main/java/org/apache/sdap/mudrod/integration/LinkageIntegration.java
index a8cc723..21aa5ba 100644
---
a/core/src/main/java/org/apache/sdap/mudrod/integration/LinkageIntegration.java
+++
b/core/src/main/java/org/apache/sdap/mudrod/integration/LinkageIntegration.java
@@ -29,8 +29,15 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.text.DecimalFormat;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.LinkedHashMap;
+import java.util.Map;
import java.util.Map.Entry;
+import java.util.Properties;
import java.util.concurrent.ExecutionException;
import java.util.stream.Collectors;
@@ -41,8 +48,8 @@ public class LinkageIntegration extends DiscoveryStepAbstract
{
private static final Logger LOG =
LoggerFactory.getLogger(LinkageIntegration.class);
private static final long serialVersionUID = 1L;
- transient List<LinkedTerm> termList = new ArrayList<>();
- DecimalFormat df = new DecimalFormat("#.00");
+ private transient List<LinkedTerm> termList = new ArrayList<>();
+ private DecimalFormat df = new DecimalFormat("#.00");
private static final String INDEX_NAME = MudrodConstants.ES_INDEX_NAME;
private static final String WEIGHT = "weight";
@@ -296,7 +303,6 @@ public class LinkageIntegration extends
DiscoveryStepAbstract {
/**
* Method of converting hashmap to JSON
*
- * @param word input query
* @param wordweights a map from related terms to weights
* @return converted JSON object
*/
diff --git a/core/src/main/java/org/apache/sdap/mudrod/main/MudrodEngine.java
b/core/src/main/java/org/apache/sdap/mudrod/main/MudrodEngine.java
index 02cf1e6..0ecb43c 100644
--- a/core/src/main/java/org/apache/sdap/mudrod/main/MudrodEngine.java
+++ b/core/src/main/java/org/apache/sdap/mudrod/main/MudrodEngine.java
@@ -391,7 +391,7 @@ public class MudrodEngine {
me.end();
} catch (Exception e) {
HelpFormatter formatter = new HelpFormatter();
- formatter.printHelp("MudrodEngine: 'dataDir' argument is mandatory. " +
"User must also provide an ingest method.", new Options());
+ formatter.printHelp("MudrodEngine: 'dataDir' argument is mandatory. User
must also provide an ingest method.", new Options());
LOG.error("Error whilst parsing command line.", e);
}
}
diff --git
a/core/src/main/java/org/apache/sdap/mudrod/metadata/pre/ApiHarvester.java
b/core/src/main/java/org/apache/sdap/mudrod/metadata/pre/ApiHarvester.java
index f492ecf..48127ad 100644
--- a/core/src/main/java/org/apache/sdap/mudrod/metadata/pre/ApiHarvester.java
+++ b/core/src/main/java/org/apache/sdap/mudrod/metadata/pre/ApiHarvester.java
@@ -28,7 +28,14 @@ import org.elasticsearch.action.index.IndexRequest;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.*;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.InputStream;
+
import java.util.Properties;
/**
diff --git
a/core/src/main/java/org/apache/sdap/mudrod/metadata/pre/MatrixGenerator.java
b/core/src/main/java/org/apache/sdap/mudrod/metadata/pre/MatrixGenerator.java
index a8d3d22..bdbd04d 100644
---
a/core/src/main/java/org/apache/sdap/mudrod/metadata/pre/MatrixGenerator.java
+++
b/core/src/main/java/org/apache/sdap/mudrod/metadata/pre/MatrixGenerator.java
@@ -74,7 +74,7 @@ public class MatrixGenerator extends DiscoveryStepAbstract {
MatrixUtil.exportToCSV(wordDocMatrix.rowMatrix, wordDocMatrix.rowkeys,
wordDocMatrix.colkeys, metadataMatrixFile);
} catch (Exception e) {
- LOG.error("Error during Metadata matrix generaion: {}", e);
+ LOG.error("Error during Metadata matrix generaion: ", e);
}
endTime = System.currentTimeMillis();
diff --git
a/core/src/main/java/org/apache/sdap/mudrod/metadata/structure/Metadata.java
b/core/src/main/java/org/apache/sdap/mudrod/metadata/structure/Metadata.java
index c420c94..59dc576 100644
--- a/core/src/main/java/org/apache/sdap/mudrod/metadata/structure/Metadata.java
+++ b/core/src/main/java/org/apache/sdap/mudrod/metadata/structure/Metadata.java
@@ -14,12 +14,7 @@
package org.apache.sdap.mudrod.metadata.structure;
import java.io.Serializable;
-import java.util.ArrayList;
import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ExecutionException;
-
-import org.apache.sdap.mudrod.driver.ESDriver;
public abstract class Metadata implements Serializable {
diff --git
a/core/src/main/java/org/apache/sdap/mudrod/metadata/structure/MetadataExtractor.java
b/core/src/main/java/org/apache/sdap/mudrod/metadata/structure/MetadataExtractor.java
index 5f84206..dcec82a 100644
---
a/core/src/main/java/org/apache/sdap/mudrod/metadata/structure/MetadataExtractor.java
+++
b/core/src/main/java/org/apache/sdap/mudrod/metadata/structure/MetadataExtractor.java
@@ -29,7 +29,6 @@ import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
-import java.util.concurrent.ExecutionException;
public class MetadataExtractor implements Serializable {
@@ -67,7 +66,7 @@ public class MetadataExtractor implements Serializable {
* @param type metadata type name
* @return metadata list
*/
- protected List<Metadata> loadMetadataFromES(ESDriver es, String index,
String type) {
+ private List<Metadata> loadMetadataFromES(ESDriver es, String index, String
type) {
List<Metadata> metadatas = new ArrayList<>();
SearchResponse scrollResp =
es.getClient().prepareSearch(index).setTypes(type).setQuery(QueryBuilders.matchAllQuery()).setScroll(new
TimeValue(60000)).setSize(100).execute().actionGet();
@@ -97,7 +96,7 @@ public class MetadataExtractor implements Serializable {
* @return PairRDD, in each pair key is metadata short name and value is term
* list extracted from metadata variables.
*/
- protected JavaPairRDD<String, List<String>> buildMetadataRDD(ESDriver es,
JavaSparkContext sc, String index, List<Metadata> metadatas) {
+ private JavaPairRDD<String, List<String>> buildMetadataRDD(ESDriver es,
JavaSparkContext sc, String index, List<Metadata> metadatas) {
JavaRDD<Metadata> metadataRDD = sc.parallelize(metadatas);
JavaPairRDD<String, List<String>> metadataTermsRDD =
metadataRDD.mapToPair(new PairFunction<Metadata, String, List<String>>() {
/**
diff --git
a/core/src/main/java/org/apache/sdap/mudrod/metadata/structure/PODAACMetadata.java
b/core/src/main/java/org/apache/sdap/mudrod/metadata/structure/PODAACMetadata.java
index a0c91d1..8f614ca 100644
---
a/core/src/main/java/org/apache/sdap/mudrod/metadata/structure/PODAACMetadata.java
+++
b/core/src/main/java/org/apache/sdap/mudrod/metadata/structure/PODAACMetadata.java
@@ -14,6 +14,8 @@
package org.apache.sdap.mudrod.metadata.structure;
import org.apache.sdap.mudrod.driver.ESDriver;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.List;
@@ -24,113 +26,107 @@ import java.util.Map;
*/
public class PODAACMetadata extends Metadata {
- /**
- *
- */
- private static final long serialVersionUID = 1L;
- // shortname: data set short name
- // private String shortname;
- // abstractStr: data set abstract
- private String abstractStr;
- // isoTopic: data set topic
- private String isoTopic;
- // sensor: sensor
- private String sensor;
- // source: data source
- private String source;
- // project: data project
- private String project;
- // hasAbstarct: whether data set has abstract
- boolean hasAbstarct;
-
- // longnameList: data set long name list
- private List<String> longnameList;
- // keywordList:data set key word list
- private List<String> keywordList;
- // termList: data set term list
- private List<String> termList;
- // topicList: data set topic list
- private List<String> topicList;
- // variableList: data set variable list
- private List<String> variableList;
- // abstractList: data set abstract term list
- private List<String> abstractList;
- // isotopicList: data set iso topic list
- private List<String> isotopicList;
- // sensorList: data set sensor list
- private List<String> sensorList;
- // sourceList: data set source list
- private List<String> sourceList;
- // projectList: data set project list
- private List<String> projectList;
- // regionList: data set region list
- private List<String> regionList;
-
- public PODAACMetadata() {
- // Default constructor
- }
-
- /**
- * Creates a new instance of PODAACMetadata.
- *
- * @param shortname
- * data set short name
- * @param longname
- * data set long name
- * @param topics
- * data set topics
- * @param terms
- * data set terms
- * @param variables
- * data set variables
- * @param keywords
- * data set keywords
- * @param region
- * list of regions
- */
- public PODAACMetadata(String shortname, List<String> longname,
List<String> topics, List<String> terms,
- List<String> variables, List<String> keywords,
List<String> region) {
- this.shortname = shortname;
- this.longnameList = longname;
- this.keywordList = keywords;
- this.termList = terms;
- this.topicList = topics;
- this.variableList = variables;
- this.regionList = region;
- }
-
- public PODAACMetadata(Map<String, Object> result, ESDriver es, String
index) {
-
- String shortname = (String) result.get("Dataset-ShortName");
- List<String> topic = (List<String>)
result.get("DatasetParameter-Topic");
- List<String> term = (List<String>)
result.get("DatasetParameter-Term");
- List<String> keyword = (List<String>)
result.get("Dataset-Metadata");
- List<String> variable = (List<String>)
result.get("DatasetParameter-Variable");
- List<String> longname = (List<String>)
result.get("DatasetProject-Project-LongName");
- List<String> region = (List<String>)
result.get("DatasetRegion-Region");
-
- this.shortname = shortname;
- this.longnameList = longname;
- try {
- this.keywordList = es.customAnalyzing(index, keyword);
- this.termList = es.customAnalyzing(index, term);
- this.topicList = es.customAnalyzing(index, topic);
- this.variableList = es.customAnalyzing(index, variable);
- this.regionList = es.customAnalyzing(index, region);
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
-
- /**
- * setTerms: set term of data set
- *
- * @param termstr
- * data set terms
- */
- public void setTerms(String termstr) {
- this.splitString(termstr, this.termList);
- }
+ /**
+ *
+ */
+ private static final long serialVersionUID = 1L;
+ // shortname: data set short name
+ // private String shortname;
+ // abstractStr: data set abstract
+ private String abstractStr;
+ // isoTopic: data set topic
+ private String isoTopic;
+ // sensor: sensor
+ private String sensor;
+ // source: data source
+ private String source;
+ // project: data project
+ private String project;
+ // hasAbstarct: whether data set has abstract
+ boolean hasAbstarct;
+
+ // longnameList: data set long name list
+ private List<String> longnameList;
+ // keywordList:data set key word list
+ private List<String> keywordList;
+ // termList: data set term list
+ private List<String> termList;
+ // topicList: data set topic list
+ private List<String> topicList;
+ // variableList: data set variable list
+ private List<String> variableList;
+ // abstractList: data set abstract term list
+ private List<String> abstractList;
+ // isotopicList: data set iso topic list
+ private List<String> isotopicList;
+ // sensorList: data set sensor list
+ private List<String> sensorList;
+ // sourceList: data set source list
+ private List<String> sourceList;
+ // projectList: data set project list
+ private List<String> projectList;
+ // regionList: data set region list
+ private List<String> regionList;
+
+ private static final Logger LOG =
LoggerFactory.getLogger(PODAACMetadata.class);
+
+ public PODAACMetadata() {
+ // Default constructor
+ }
+
+ /**
+ * Creates a new instance of PODAACMetadata.
+ *
+ * @param shortname data set short name
+ * @param longname data set long name
+ * @param topics data set topics
+ * @param terms data set terms
+ * @param variables data set variables
+ * @param keywords data set keywords
+ * @param region list of regions
+ */
+ public PODAACMetadata(String shortname, List<String> longname, List<String>
topics, List<String> terms,
+ List<String> variables, List<String> keywords,
List<String> region) {
+ this.shortname = shortname;
+ this.longnameList = longname;
+ this.keywordList = keywords;
+ this.termList = terms;
+ this.topicList = topics;
+ this.variableList = variables;
+ this.regionList = region;
+ }
+
+ public PODAACMetadata(Map<String, Object> result, ESDriver es, String index)
{
+
+ String shortname = (String) result.get("Dataset-ShortName");
+ List<String> topic = (List<String>) result.get("DatasetParameter-Topic");
+ List<String> term = (List<String>) result.get("DatasetParameter-Term");
+ List<String> keyword = (List<String>) result.get("Dataset-Metadata");
+ List<String> variable = (List<String>)
result.get("DatasetParameter-Variable");
+ List<String> longname = (List<String>)
result.get("DatasetProject-Project-LongName");
+ List<String> region = (List<String>) result.get("DatasetRegion-Region");
+
+ this.shortname = shortname;
+ this.longnameList = longname;
+ try {
+ this.keywordList = es.customAnalyzing(index, keyword);
+ this.termList = es.customAnalyzing(index, term);
+ this.topicList = es.customAnalyzing(index, topic);
+ this.variableList = es.customAnalyzing(index, variable);
+ this.regionList = es.customAnalyzing(index, region);
+ } catch (Exception e) {
+ LOG.error("Constructor initialization failed : ", e);
+ }
+ }
+
+ /**
+ * setTerms: set term of data set
+ *
+ * @param termstr data set terms
+ */
+ public void setTerms(String termstr) {
+ this.splitString(termstr, this.termList);
+ }
/**
* setKeywords: set key word of data set
diff --git a/core/src/main/java/org/apache/sdap/mudrod/ontology/Ontology.java
b/core/src/main/java/org/apache/sdap/mudrod/ontology/Ontology.java
index 70116de..9b6f3c7 100644
--- a/core/src/main/java/org/apache/sdap/mudrod/ontology/Ontology.java
+++ b/core/src/main/java/org/apache/sdap/mudrod/ontology/Ontology.java
@@ -30,12 +30,12 @@ public interface Ontology {
*
* @param urls a {@link java.lang.String} containing ontology URIs.
*/
- public void load(String[] urls);
+ void load(String[] urls);
/**
* Load a collection of default ontology resources.
*/
- public void load() ;
+ void load() ;
/**
* merge relevant ontology subgraphs into a new subgraph which can
@@ -44,7 +44,7 @@ public interface Ontology {
* @param o an ontology to merge with the current ontology held
* within Mudrod.
*/
- public void merge(Ontology o);
+ void merge(Ontology o);
/**
* Retreive all subclasses for a particular entity provided within the
@@ -53,7 +53,7 @@ public interface Ontology {
* @param entitySearchTerm an input search term
* @return an {@link java.util.Iterator} object containing subClass entries.
*/
- public Iterator<String> subclasses(String entitySearchTerm);
+ Iterator<String> subclasses(String entitySearchTerm);
/**
* Retreive all synonyms for a particular entity provided within the
@@ -62,6 +62,6 @@ public interface Ontology {
* @param queryKeyPhrase a phrase to undertake synonym expansion on.
* @return an {@link java.util.Iterator} object containing synonym entries.
*/
- public Iterator<String> synonyms(String queryKeyPhrase);
+ Iterator<String> synonyms(String queryKeyPhrase);
}
diff --git
a/core/src/main/java/org/apache/sdap/mudrod/ontology/pre/AggregateTriples.java
b/core/src/main/java/org/apache/sdap/mudrod/ontology/pre/AggregateTriples.java
index e32c0f7..f92d423 100644
---
a/core/src/main/java/org/apache/sdap/mudrod/ontology/pre/AggregateTriples.java
+++
b/core/src/main/java/org/apache/sdap/mudrod/ontology/pre/AggregateTriples.java
@@ -59,7 +59,7 @@ public class AggregateTriples extends DiscoveryStepAbstract {
try {
file.createNewFile();
} catch (IOException e2) {
- e2.printStackTrace();
+ LOG.error("File creation failed: ", e2);
}
try(FileWriter fw = new FileWriter(file.getAbsoluteFile())){
@@ -71,12 +71,9 @@ public class AggregateTriples extends DiscoveryStepAbstract {
try {
loadXml(fileIn.getAbsolutePath());
getAllClass();
- } catch (JDOMException e1) {
- e1.printStackTrace();
- } catch (IOException e1) {
- e1.printStackTrace();
+ } catch (JDOMException | IOException e1) {
+ LOG.error("Couln't load Owl file into memory: ", e1);
}
-
}
}
} catch (IOException e) {
@@ -91,11 +88,11 @@ public class AggregateTriples extends DiscoveryStepAbstract
{
return null;
}
- public Document document;
- public Element rootNode = null;
- final static String owl_namespace = "http://www.w3.org/2002/07/owl#";;
- final static String rdf_namespace =
"http://www.w3.org/1999/02/22-rdf-syntax-ns#";;
- final static String rdfs_namespace = "http://www.w3.org/2000/01/rdf-schema#";;
+ private Document document;
+ private Element rootNode = null;
+ private final static String owl_namespace = "http://www.w3.org/2002/07/owl#";;
+ private final static String rdf_namespace =
"http://www.w3.org/1999/02/22-rdf-syntax-ns#";;
+ private final static String rdfs_namespace =
"http://www.w3.org/2000/01/rdf-schema#";;
BufferedWriter bw = null;
@@ -106,7 +103,7 @@ public class AggregateTriples extends DiscoveryStepAbstract
{
* @throws JDOMException JDOMException
* @throws IOException IOException
*/
- public void loadXml(String filePathName) throws JDOMException, IOException {
+ private void loadXml(String filePathName) throws JDOMException, IOException {
SAXBuilder saxBuilder = new SAXBuilder();
File file = new File(filePathName);
@@ -140,7 +137,7 @@ public class AggregateTriples extends DiscoveryStepAbstract
{
* @param ele parent element
* @return the element of child
*/
- public Element findChild(String str, Element ele) {
+ private Element findChild(String str, Element ele) {
Iterator<?> processDescendants = ele.getDescendants(new ElementFilter());
String name = "";
Element result = null;
@@ -162,7 +159,7 @@ public class AggregateTriples extends DiscoveryStepAbstract
{
*
* @throws IOException IOException
*/
- public void getAllClass() throws IOException {
+ private void getAllClass() throws IOException {
List<?> classElements = rootNode.getChildren("Class",
Namespace.getNamespace("owl", owl_namespace));
for (Object classElement1 : classElements) {
@@ -208,7 +205,7 @@ public class AggregateTriples extends DiscoveryStepAbstract
{
* @param str String needed to be processed
* @return the processed string
*/
- public String cutString(String str) {
+ private String cutString(String str) {
str = str.substring(str.indexOf("#") + 1);
String[] strArray = str.split("(?=[A-Z])");
str = Arrays.toString(strArray);
diff --git
a/core/src/main/java/org/apache/sdap/mudrod/ontology/process/OntologyLinkCal.java
b/core/src/main/java/org/apache/sdap/mudrod/ontology/process/OntologyLinkCal.java
index 23fa348..1cf1376 100644
---
a/core/src/main/java/org/apache/sdap/mudrod/ontology/process/OntologyLinkCal.java
+++
b/core/src/main/java/org/apache/sdap/mudrod/ontology/process/OntologyLinkCal.java
@@ -43,7 +43,7 @@ public class OntologyLinkCal extends DiscoveryStepAbstract {
/**
* Method of adding mapping for triples extracted from SWEET
*/
- public void addSWEETMapping() {
+ private void addSWEETMapping() {
XContentBuilder Mapping;
try {
Mapping =
jsonBuilder().startObject().startObject(MudrodConstants.ONTOLOGY_LINKAGE_TYPE).startObject("properties").startObject("concept_A").field("type",
"string")
diff --git
a/core/src/main/java/org/apache/sdap/mudrod/ontology/process/OntologyParser.java
b/core/src/main/java/org/apache/sdap/mudrod/ontology/process/OntologyParser.java
index 6e2a5f2..f23bc70 100644
---
a/core/src/main/java/org/apache/sdap/mudrod/ontology/process/OntologyParser.java
+++
b/core/src/main/java/org/apache/sdap/mudrod/ontology/process/OntologyParser.java
@@ -32,7 +32,7 @@ public interface OntologyParser {
* implementation processing the ontology operation(s).
* @param ontModel the {@link org.apache.jena.ontology.OntModel}
*/
- public void parse(Ontology ont, OntModel ontModel);
+ void parse(Ontology ont, OntModel ontModel);
/**
* An ontology model (RDF graph) for which to obtain an
@@ -41,6 +41,6 @@ public interface OntologyParser {
* @param ontModel the {@link org.apache.jena.ontology.OntModel}
* @return an {@link java.util.Iterator} instance containing all root
classes.
*/
- public Iterator<OntClass> rootClasses(OntModel ontModel);
+ Iterator<OntClass> rootClasses(OntModel ontModel);
}
diff --git
a/core/src/main/java/org/apache/sdap/mudrod/ontology/process/OwlParser.java
b/core/src/main/java/org/apache/sdap/mudrod/ontology/process/OwlParser.java
index 3f12cd6..0fffcd8 100644
--- a/core/src/main/java/org/apache/sdap/mudrod/ontology/process/OwlParser.java
+++ b/core/src/main/java/org/apache/sdap/mudrod/ontology/process/OwlParser.java
@@ -59,7 +59,7 @@ public class OwlParser implements OntologyParser {
}
}
- protected void parseClass(OntClass cls, List<Object> occurs, int depth) {
+ private void parseClass(OntClass cls, List<Object> occurs, int depth) {
//dont deal with anonymous classes
if (cls.isAnon()) {
return;
@@ -155,7 +155,7 @@ public class OwlParser implements OntologyParser {
}
}
- public String rdfidToLabel(String idString) {
+ private String rdfidToLabel(String idString) {
Pattern p = Pattern.compile("([a-z0-9])([A-Z])");
Matcher m = p.matcher(idString);
diff --git
a/core/src/main/java/org/apache/sdap/mudrod/recommendation/pre/ImportMetadata.java
b/core/src/main/java/org/apache/sdap/mudrod/recommendation/pre/ImportMetadata.java
index 15e6a03..f953974 100644
---
a/core/src/main/java/org/apache/sdap/mudrod/recommendation/pre/ImportMetadata.java
+++
b/core/src/main/java/org/apache/sdap/mudrod/recommendation/pre/ImportMetadata.java
@@ -26,7 +26,11 @@ import org.elasticsearch.action.index.IndexRequest;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.*;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
import java.util.Properties;
/**
@@ -61,7 +65,7 @@ public class ImportMetadata extends DiscoveryStepAbstract {
* addMetadataMapping: Add mapping to index metadata in Elasticsearch. Please
* invoke this method before import metadata to Elasticsearch.
*/
- public void addMetadataMapping() {
+ private void addMetadataMapping() {
String mappingJson = "{\r\n \"dynamic_templates\": " + "[\r\n " +
"{\r\n \"strings\": " + "{\r\n \"match_mapping_type\":
\"string\","
+ "\r\n \"mapping\": {\r\n \"type\":
\"string\"," + "\r\n \"analyzer\": \"csv\"\r\n }" +
"\r\n }\r\n }\r\n ]\r\n}";
@@ -91,10 +95,10 @@ public class ImportMetadata extends DiscoveryStepAbstract {
es.getBulkProcessor().add(ir);
} catch (IOException e) {
- e.printStackTrace();
+ LOG.error("Error reading File Content: ", e);
}
} catch (FileNotFoundException e) {
- e.printStackTrace();
+ LOG.error("Error finding file: ", e);
}
}
diff --git
a/core/src/main/java/org/apache/sdap/mudrod/recommendation/pre/MetadataTFIDFGenerator.java
b/core/src/main/java/org/apache/sdap/mudrod/recommendation/pre/MetadataTFIDFGenerator.java
index 3486607..763eca6 100644
---
a/core/src/main/java/org/apache/sdap/mudrod/recommendation/pre/MetadataTFIDFGenerator.java
+++
b/core/src/main/java/org/apache/sdap/mudrod/recommendation/pre/MetadataTFIDFGenerator.java
@@ -70,7 +70,7 @@ public class MetadataTFIDFGenerator extends
DiscoveryStepAbstract {
return null;
}
- public LabeledRowMatrix generateWordBasedTFIDF() throws Exception {
+ private LabeledRowMatrix generateWordBasedTFIDF() throws Exception {
MetadataTokenizer opt = new MetadataTokenizer(props);
diff --git
a/core/src/main/java/org/apache/sdap/mudrod/recommendation/pre/NormalizeFeatures.java
b/core/src/main/java/org/apache/sdap/mudrod/recommendation/pre/NormalizeFeatures.java
index 458ccbb..741cd8a 100644
---
a/core/src/main/java/org/apache/sdap/mudrod/recommendation/pre/NormalizeFeatures.java
+++
b/core/src/main/java/org/apache/sdap/mudrod/recommendation/pre/NormalizeFeatures.java
@@ -11,10 +11,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-/**
- * This package includes the preprocessing, processing, and data structure used
- * by recommendation module.
- */
+
package org.apache.sdap.mudrod.recommendation.pre;
import org.apache.sdap.mudrod.discoveryengine.DiscoveryStepAbstract;
@@ -35,7 +32,6 @@ import org.slf4j.LoggerFactory;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
-import java.util.regex.Pattern;
public class NormalizeFeatures extends DiscoveryStepAbstract {
@@ -80,7 +76,7 @@ public class NormalizeFeatures extends DiscoveryStepAbstract {
return null;
}
- public void normalizeMetadataVariables(ESDriver es) {
+ private void normalizeMetadataVariables(ESDriver es) {
es.createBulkProcessor();
diff --git
a/core/src/main/java/org/apache/sdap/mudrod/recommendation/pre/SessionCooccurence.java
b/core/src/main/java/org/apache/sdap/mudrod/recommendation/pre/SessionCooccurence.java
index 4a5d4bb..61492ca 100644
---
a/core/src/main/java/org/apache/sdap/mudrod/recommendation/pre/SessionCooccurence.java
+++
b/core/src/main/java/org/apache/sdap/mudrod/recommendation/pre/SessionCooccurence.java
@@ -30,7 +30,11 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import scala.Tuple2;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
/**
* ClassName: SessionCooccurenceMatrix Function: Generate metadata session
diff --git
a/core/src/main/java/org/apache/sdap/mudrod/recommendation/process/AbstractBasedSimilarity.java
b/core/src/main/java/org/apache/sdap/mudrod/recommendation/process/AbstractBasedSimilarity.java
index f2de90a..94b68fd 100644
---
a/core/src/main/java/org/apache/sdap/mudrod/recommendation/process/AbstractBasedSimilarity.java
+++
b/core/src/main/java/org/apache/sdap/mudrod/recommendation/process/AbstractBasedSimilarity.java
@@ -57,7 +57,7 @@ public class AbstractBasedSimilarity extends
DiscoveryStepAbstract {
svd.saveToES(tripleList,
props.getProperty(MudrodConstants.ES_INDEX_NAME),
MudrodConstants.METADATA_WORD_SIM_TYPE, true, true);
} catch (Exception e) {
- e.printStackTrace();
+ LOG.error("Abstract similarity calculation failed: ", e);
}
endTime = System.currentTimeMillis();
diff --git
a/core/src/main/java/org/apache/sdap/mudrod/recommendation/process/FeatureBasedSimilarity.java
b/core/src/main/java/org/apache/sdap/mudrod/recommendation/process/FeatureBasedSimilarity.java
index 3a18795..4b2bfeb 100644
---
a/core/src/main/java/org/apache/sdap/mudrod/recommendation/process/FeatureBasedSimilarity.java
+++
b/core/src/main/java/org/apache/sdap/mudrod/recommendation/process/FeatureBasedSimilarity.java
@@ -11,10 +11,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-/**
- * This package includes the preprocessing, processing, and data structure used
- * by recommendation module.
- */
package org.apache.sdap.mudrod.recommendation.process;
import org.apache.sdap.mudrod.discoveryengine.DiscoveryStepAbstract;
@@ -104,7 +100,7 @@ public class FeatureBasedSimilarity extends
DiscoveryStepAbstract implements Ser
return null;
}
- public void featureSimilarity(ESDriver es) {
+ private void featureSimilarity(ESDriver es) {
es.createBulkProcessor();
@@ -156,7 +152,7 @@ public class FeatureBasedSimilarity extends
DiscoveryStepAbstract implements Ser
es.destroyBulkProcessor();
}
- public static void addMapping(ESDriver es, String index, String type) {
+ private static void addMapping(ESDriver es, String index, String type) {
XContentBuilder Mapping;
try {
Mapping =
jsonBuilder().startObject().startObject(type).startObject("properties").startObject("concept_A").field("type",
"string").field("index", "not_analyzed").endObject()
@@ -170,7 +166,7 @@ public class FeatureBasedSimilarity extends
DiscoveryStepAbstract implements Ser
}
}
- public void normalizeVariableWeight(ESDriver es) {
+ private void normalizeVariableWeight(ESDriver es) {
es.createBulkProcessor();
diff --git
a/core/src/main/java/org/apache/sdap/mudrod/recommendation/process/SessionBasedCF.java
b/core/src/main/java/org/apache/sdap/mudrod/recommendation/process/SessionBasedCF.java
index e58e6d5..32db2ba 100644
---
a/core/src/main/java/org/apache/sdap/mudrod/recommendation/process/SessionBasedCF.java
+++
b/core/src/main/java/org/apache/sdap/mudrod/recommendation/process/SessionBasedCF.java
@@ -50,7 +50,7 @@ public class SessionBasedCF extends DiscoveryStepAbstract {
@Override
public Object execute() {
- LOG.info("Session based metadata similarity starts.");
+ LOG.info("Session based metadata similarity calculation starts.");
startTime = System.currentTimeMillis();
try {
@@ -63,11 +63,11 @@ public class SessionBasedCF extends DiscoveryStepAbstract {
}
} catch (Exception e) {
- e.printStackTrace();
+ LOG.error("Session based metadata similarity calculation failed: ", e);
}
endTime = System.currentTimeMillis();
- LOG.info("Session based metadata similarity ends. Took {}s", (endTime -
startTime) / 1000);
+ LOG.info("Session based metadata similarity calculation ends. Took {}s",
(endTime - startTime) / 1000);
return null;
}
diff --git
a/core/src/main/java/org/apache/sdap/mudrod/recommendation/structure/HybridRecommendation.java
b/core/src/main/java/org/apache/sdap/mudrod/recommendation/structure/HybridRecommendation.java
index 2b29c03..6d915e5 100644
---
a/core/src/main/java/org/apache/sdap/mudrod/recommendation/structure/HybridRecommendation.java
+++
b/core/src/main/java/org/apache/sdap/mudrod/recommendation/structure/HybridRecommendation.java
@@ -42,11 +42,11 @@ public class HybridRecommendation extends
DiscoveryStepAbstract {
*/
private static final long serialVersionUID = 1L;
// recommended metadata list
- protected transient List<LinkedTerm> termList = new ArrayList<>();
+ private transient List<LinkedTerm> termList = new ArrayList<>();
// format decimal
- DecimalFormat df = new DecimalFormat("#.00");
+ private DecimalFormat df = new DecimalFormat("#.00");
// index name
- protected static final String INDEX_NAME = MudrodConstants.ES_INDEX_NAME;
+ private static final String INDEX_NAME = MudrodConstants.ES_INDEX_NAME;
private static final String WEIGHT = "weight";
/**
@@ -145,7 +145,7 @@ public class HybridRecommendation extends
DiscoveryStepAbstract {
* @param num the number of converted elements
* @return converted JSON object
*/
- protected JsonElement mapToJson(Map<String, Double> wordweights, int num) {
+ private JsonElement mapToJson(Map<String, Double> wordweights, int num) {
Gson gson = new Gson();
List<JsonObject> nodes = new ArrayList<>();
@@ -164,9 +164,7 @@ public class HybridRecommendation extends
DiscoveryStepAbstract {
}
String nodesJson = gson.toJson(nodes);
- JsonElement nodesElement = gson.fromJson(nodesJson, JsonElement.class);
-
- return nodesElement;
+ return gson.fromJson(nodesJson, JsonElement.class);
}
/**
@@ -178,7 +176,7 @@ public class HybridRecommendation extends
DiscoveryStepAbstract {
* @return recommended dataset map, key is dataset name, value is similarity
* value
*/
- public Map<String, Double> getRelatedData(String type, String input, int
num) {
+ private Map<String, Double> getRelatedData(String type, String input, int
num) {
termList = new ArrayList<>();
Map<String, Double> termsMap = new HashMap<>();
Map<String, Double> sortedMap = new HashMap<>();
@@ -204,7 +202,7 @@ public class HybridRecommendation extends
DiscoveryStepAbstract {
* @param num the number of recommended dataset
* @return recommended dataset list
*/
- public List<LinkedTerm> getRelatedDataFromES(String type, String input, int
num) {
+ private List<LinkedTerm> getRelatedDataFromES(String type, String input, int
num) {
SearchRequestBuilder builder =
es.getClient().prepareSearch(props.getProperty(INDEX_NAME)).setTypes(type).setQuery(QueryBuilders.termQuery("concept_A",
input)).addSort(WEIGHT, SortOrder.DESC)
.setSize(num);
@@ -230,7 +228,7 @@ public class HybridRecommendation extends
DiscoveryStepAbstract {
* @param passedMap input map
* @return sorted map
*/
- public Map<String, Double> sortMapByValue(Map<String, Double> passedMap) {
+ private Map<String, Double> sortMapByValue(Map<String, Double> passedMap) {
List<String> mapKeys = new ArrayList<>(passedMap.keySet());
List<Double> mapValues = new ArrayList<>(passedMap.values());
Collections.sort(mapValues, Collections.reverseOrder());
diff --git
a/core/src/main/java/org/apache/sdap/mudrod/recommendation/structure/MetadataFeature.java
b/core/src/main/java/org/apache/sdap/mudrod/recommendation/structure/MetadataFeature.java
index ca5b6d4..0dfffcf 100644
---
a/core/src/main/java/org/apache/sdap/mudrod/recommendation/structure/MetadataFeature.java
+++
b/core/src/main/java/org/apache/sdap/mudrod/recommendation/structure/MetadataFeature.java
@@ -11,10 +11,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-/**
- * This package includes the preprocessing, processing, and data structure used
- * by recommendation module.
- */
package org.apache.sdap.mudrod.recommendation.structure;
import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -25,6 +21,8 @@ import java.util.Map;
public abstract class MetadataFeature implements Serializable {
+ private static final long serialVersionUID = 1L;
+
protected static final Integer VAR_SPATIAL = 1;
protected static final Integer VAR_TEMPORAL = 2;
protected static final Integer VAR_CATEGORICAL = 3;
diff --git
a/core/src/main/java/org/apache/sdap/mudrod/recommendation/structure/MetadataTokenizer.java
b/core/src/main/java/org/apache/sdap/mudrod/recommendation/structure/MetadataTokenizer.java
index b56c25d..0f6e1b7 100644
---
a/core/src/main/java/org/apache/sdap/mudrod/recommendation/structure/MetadataTokenizer.java
+++
b/core/src/main/java/org/apache/sdap/mudrod/recommendation/structure/MetadataTokenizer.java
@@ -11,10 +11,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-/**
- * This package includes the preprocessing, processing, and data structure used
- * by recommendation module.
- */
package org.apache.sdap.mudrod.recommendation.structure;
import org.apache.sdap.mudrod.driver.ESDriver;
@@ -45,8 +41,8 @@ public class MetadataTokenizer implements Serializable {
private String metadataType;
private List<String> variables;
- public static final String SPLIT_BLANK = " ";
- public static final String SPLIT_COMMA = ",";
+ private static final String SPLIT_BLANK = " ";
+ private static final String SPLIT_COMMA = ",";
public MetadataTokenizer(Properties props) {
indexName = props.getProperty(MudrodConstants.ES_INDEX_NAME);
@@ -103,7 +99,7 @@ public class MetadataTokenizer implements Serializable {
}
- public List<String> getTokens(String str, String splitter) throws Exception {
+ private List<String> getTokens(String str, String splitter) throws Exception
{
String[] tokens = null;
if (splitter.equals(SPLIT_BLANK)) {
tokens = str.split(" ");
@@ -113,7 +109,7 @@ public class MetadataTokenizer implements Serializable {
return java.util.Arrays.asList(tokens);
}
- public List<Tuple2<String, String>> loadMetadataFromES(ESDriver es,
List<String> variables, String metadataName) throws Exception {
+ private List<Tuple2<String, String>> loadMetadataFromES(ESDriver es,
List<String> variables, String metadataName) throws Exception {
SearchResponse scrollResp =
es.getClient().prepareSearch(indexName).setTypes(metadataType).setQuery(QueryBuilders.matchAllQuery()).setScroll(new
TimeValue(60000)).setSize(100).execute()
.actionGet();
@@ -154,9 +150,7 @@ public class MetadataTokenizer implements Serializable {
RowMatrix docwordMatrix = labelMatrix.rowMatrix;
- RowMatrix docwordTFIDFMatrix = MatrixUtil.createTFIDFMatrix(docwordMatrix);
-
- labelMatrix.rowMatrix = docwordTFIDFMatrix;
+ labelMatrix.rowMatrix = MatrixUtil.createTFIDFMatrix(docwordMatrix);
return labelMatrix;
}
diff --git
a/core/src/main/java/org/apache/sdap/mudrod/recommendation/structure/PODAACMetadataFeature.java
b/core/src/main/java/org/apache/sdap/mudrod/recommendation/structure/PODAACMetadataFeature.java
index d2b4e28..541c166 100644
---
a/core/src/main/java/org/apache/sdap/mudrod/recommendation/structure/PODAACMetadataFeature.java
+++
b/core/src/main/java/org/apache/sdap/mudrod/recommendation/structure/PODAACMetadataFeature.java
@@ -11,13 +11,11 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-/**
- * This package includes the preprocessing, processing, and data structure used
- * by recommendation module.
- */
package org.apache.sdap.mudrod.recommendation.structure;
import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.regex.Pattern;
@@ -27,6 +25,8 @@ import java.util.Map;
public class PODAACMetadataFeature extends MetadataFeature {
+ private static final Logger LOG =
LoggerFactory.getLogger(PODAACMetadataFeature.class);
+
public void normalizeSpatialVariables(Map<String, Object> metadata,
Map<String, Object> updatedValues) {
// get spatial resolution
@@ -121,7 +121,7 @@ public class PODAACMetadataFeature extends MetadataFeature {
return 0;
}
- public Double getProLevelNum(String pro) {
+ private Double getProLevelNum(String pro) {
if (pro == null) {
return 1.0;
}
@@ -236,7 +236,7 @@ public class PODAACMetadataFeature extends MetadataFeature {
contentBuilder.field("Spatial_Covergae_Sim", similarity);
} catch (IOException e) {
// TODO Auto-generated catch block
- e.printStackTrace();
+ LOG.error("Spatial similarity calculation failed : ", e);
}
}
@@ -280,7 +280,7 @@ public class PODAACMetadataFeature extends MetadataFeature {
contentBuilder.field("Temporal_Covergae_Sim", similarity);
} catch (IOException e) {
// TODO Auto-generated catch block
- e.printStackTrace();
+ LOG.error("Temporal similarity calculation failed : ", e);
}
}
@@ -319,7 +319,7 @@ public class PODAACMetadataFeature extends MetadataFeature {
contentBuilder.field(variable + "_Sim", similarity);
} catch (IOException e) {
// TODO Auto-generated catch block
- e.printStackTrace();
+ LOG.error("Categorical variables similarity calculation failed : ", e);
}
}
}
@@ -352,7 +352,7 @@ public class PODAACMetadataFeature extends MetadataFeature {
contentBuilder.field(variable + "_Sim", similarity);
} catch (IOException e) {
// TODO Auto-generated catch block
- e.printStackTrace();
+ LOG.error("Ordinal similarity calculation failed : ", e);
}
}
}
diff --git
a/core/src/main/java/org/apache/sdap/mudrod/semantics/SemanticAnalyzer.java
b/core/src/main/java/org/apache/sdap/mudrod/semantics/SemanticAnalyzer.java
index a4ecdc8..fb32303 100644
--- a/core/src/main/java/org/apache/sdap/mudrod/semantics/SemanticAnalyzer.java
+++ b/core/src/main/java/org/apache/sdap/mudrod/semantics/SemanticAnalyzer.java
@@ -23,6 +23,8 @@ import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.mllib.linalg.Vector;
import org.apache.spark.mllib.linalg.distributed.CoordinateMatrix;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
@@ -38,6 +40,7 @@ public class SemanticAnalyzer extends MudrodAbstract {
*
*/
private static final long serialVersionUID = 1L;
+ private static final Logger LOG =
LoggerFactory.getLogger(SemanticAnalyzer.class);
/**
* Creates a new instance of SemanticAnalyzer.
@@ -119,7 +122,7 @@ public class SemanticAnalyzer extends MudrodAbstract {
try {
LinkageTriple.insertTriples(es, tripleList, index, type);
} catch (IOException e) {
- e.printStackTrace();
+ LOG.error("Saving to Elastic Search failed : ", e);
}
}
@@ -141,8 +144,7 @@ public class SemanticAnalyzer extends MudrodAbstract {
try {
LinkageTriple.insertTriples(es, tripleList, index, type, bTriple,
bSymmetry);
} catch (IOException e) {
- e.printStackTrace();
-
+ LOG.error("Saving to Elastic Search failed : ", e);
}
}
}
diff --git
a/core/src/main/java/org/apache/sdap/mudrod/ssearch/ClickStreamImporter.java
b/core/src/main/java/org/apache/sdap/mudrod/ssearch/ClickStreamImporter.java
index 546fae5..5694236 100644
--- a/core/src/main/java/org/apache/sdap/mudrod/ssearch/ClickStreamImporter.java
+++ b/core/src/main/java/org/apache/sdap/mudrod/ssearch/ClickStreamImporter.java
@@ -19,9 +19,10 @@ import org.apache.sdap.mudrod.driver.SparkDriver;
import org.apache.sdap.mudrod.main.MudrodConstants;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.BufferedReader;
-import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.util.Properties;
@@ -37,6 +38,7 @@ public class ClickStreamImporter extends MudrodAbstract {
*
*/
private static final long serialVersionUID = 1L;
+ private static final Logger LOG =
LoggerFactory.getLogger(ClickStreamImporter.class);
public ClickStreamImporter(Properties props, ESDriver es, SparkDriver spark)
{
super(props, es, spark);
@@ -65,7 +67,7 @@ public class ClickStreamImporter extends MudrodAbstract {
clickStreamMatrixType).setSource(
mapping).execute().actionGet();
} catch (IOException e) {
- e.printStackTrace();
+ LOG.error("Add Elasticsearch mapping for click stream data failed : ",
e);
}
}
@@ -97,17 +99,15 @@ public class ClickStreamImporter extends MudrodAbstract {
}
}
}
- } catch (FileNotFoundException e) {
- e.printStackTrace();
} catch (IOException e) {
- e.printStackTrace();
+ LOG.error("Import click stream CSV into Elasticsearch failed : ", e);
} finally {
if (br != null) {
try {
br.close();
es.destroyBulkProcessor();
} catch (IOException e) {
- e.printStackTrace();
+ //Ignore
}
}
}
diff --git a/core/src/main/java/org/apache/sdap/mudrod/ssearch/Ranker.java
b/core/src/main/java/org/apache/sdap/mudrod/ssearch/Ranker.java
index af7e6a9..9215859 100644
--- a/core/src/main/java/org/apache/sdap/mudrod/ssearch/Ranker.java
+++ b/core/src/main/java/org/apache/sdap/mudrod/ssearch/Ranker.java
@@ -24,7 +24,11 @@ import org.apache.spark.mllib.regression.LabeledPoint;
import java.io.Serializable;
import java.text.DecimalFormat;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Properties;
/**
* Supports the ability to calculating ranking score
diff --git a/core/src/main/java/org/apache/sdap/mudrod/ssearch/Searcher.java
b/core/src/main/java/org/apache/sdap/mudrod/ssearch/Searcher.java
index ce0183a..adec951 100644
--- a/core/src/main/java/org/apache/sdap/mudrod/ssearch/Searcher.java
+++ b/core/src/main/java/org/apache/sdap/mudrod/ssearch/Searcher.java
@@ -32,7 +32,11 @@ import org.elasticsearch.search.sort.SortOrder;
import java.io.Serializable;
import java.text.DecimalFormat;
import java.text.SimpleDateFormat;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
import java.util.regex.Pattern;
/**
@@ -56,7 +60,7 @@ public class Searcher extends MudrodAbstract implements
Serializable {
* @param pro processing level string
* @return processing level number
*/
- public Double getProLevelNum(String pro) {
+ private Double getProLevelNum(String pro) {
if (pro == null) {
return 1.0;
}
@@ -73,7 +77,7 @@ public class Searcher extends MudrodAbstract implements
Serializable {
return proNum;
}
- public Double getPop(Double pop) {
+ private Double getPop(Double pop) {
if (pop > 1000) {
pop = 1000.0;
}
@@ -91,7 +95,7 @@ public class Searcher extends MudrodAbstract implements
Serializable {
* @return a list of search result
*/
@SuppressWarnings("unchecked")
- public List<SResult> searchByQuery(String index, String type, String query,
String queryOperator, String rankOption) {
+ private List<SResult> searchByQuery(String index, String type, String query,
String queryOperator, String rankOption) {
boolean exists = es.getClient()
.admin()
.indices()
diff --git
a/core/src/main/java/org/apache/sdap/mudrod/ssearch/ranking/DataGenerator.java
b/core/src/main/java/org/apache/sdap/mudrod/ssearch/ranking/DataGenerator.java
index ed69b6b..30abde5 100644
---
a/core/src/main/java/org/apache/sdap/mudrod/ssearch/ranking/DataGenerator.java
+++
b/core/src/main/java/org/apache/sdap/mudrod/ssearch/ranking/DataGenerator.java
@@ -15,8 +15,14 @@ package org.apache.sdap.mudrod.ssearch.ranking;
import au.com.bytecode.opencsv.CSVReader;
import au.com.bytecode.opencsv.CSVWriter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-import java.io.*;
+
+import java.io.File;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -37,7 +43,8 @@ public class DataGenerator {
private static List<List<String>> myMasterList = new ArrayList<>();
// HashMap used for comparing evaluation classes
- public static final Map<String, Integer> map1 = new HashMap<>();
+ private static final Map<String, Integer> map1 = new HashMap<>();
+ private static final Logger LOG =
LoggerFactory.getLogger(DataGenerator.class);
static {
map1.put("Excellent", 7);
@@ -78,7 +85,7 @@ public class DataGenerator {
* Parses the original user-specified CSV file, storing the contents for
future calculations
* and formatting.
*/
- public static void parseFile() {
+ private static void parseFile() {
String[][] dataArr = null;
try {
String sourceDir = mySourceDir;
@@ -118,10 +125,8 @@ public class DataGenerator {
}
}
}
- } catch (FileNotFoundException e) {
- e.printStackTrace();
} catch (IOException e) {
- e.printStackTrace();
+ LOG.error("User specified CSV file parse failed : ", e);
}
}
@@ -131,7 +136,7 @@ public class DataGenerator {
*
* @param arr the parsed contents of the original CSV file
*/
- public static void calculateVec(String[][] arr) {
+ private static void calculateVec(String[][] arr) {
List<List<String>> listofLists = new ArrayList<>(); // Holds calculations
int rowStart = 1;
@@ -182,7 +187,7 @@ public class DataGenerator {
* @return 1 if first evaluation is greater than the second, -1 if first
evaluation is less than the second, and
* 10 in the case that the two are equal
*/
- public static int compareEvaluation(String eval1, String eval2) {
+ private static int compareEvaluation(String eval1, String eval2) {
int evalNum1 = map1.get(eval1);
int evalNum2 = map1.get(eval2);
@@ -203,7 +208,7 @@ public class DataGenerator {
* @param rawList originally calculated data from the input CSV file
* @return data that has an equal distribution of evaluation values
*/
- public static List<List<String>> equalizeList(List<List<String>> rawList) {
+ private static List<List<String>> equalizeList(List<List<String>> rawList) {
// Create two sets - one containing row index for +1 and the other for -1
List<Integer> pos1List = new ArrayList<>();
List<Integer> neg1List = new ArrayList<>();
@@ -269,7 +274,7 @@ public class DataGenerator {
*
* @param arr 2D array containing the parsed information from input file
*/
- public static void storeHead(String[][] arr) {
+ private static void storeHead(String[][] arr) {
myHeader = new String[arr[0].length]; // Reside private variable
System.arraycopy(arr[0], 0, myHeader, 0, arr[0].length);
@@ -280,7 +285,7 @@ public class DataGenerator {
*
* @param list finalized vector data to write to user specified output file
*/
- public static void writeCSVfile(List<List<String>> list) {
+ private static void writeCSVfile(List<List<String>> list) {
String outputFile = myResultDir;
boolean alreadyExists = new File(outputFile).exists();
@@ -299,7 +304,7 @@ public class DataGenerator {
csvOutput.close(); // Close csvWriter
} catch (IOException e) {
- e.printStackTrace();
+ LOG.error("Writing to CSV file failed : ", e);
}
}
diff --git
a/core/src/main/java/org/apache/sdap/mudrod/ssearch/ranking/Learner.java
b/core/src/main/java/org/apache/sdap/mudrod/ssearch/ranking/Learner.java
index 8a752a3..0e5ffbc 100644
--- a/core/src/main/java/org/apache/sdap/mudrod/ssearch/ranking/Learner.java
+++ b/core/src/main/java/org/apache/sdap/mudrod/ssearch/ranking/Learner.java
@@ -28,13 +28,12 @@ public class Learner implements Serializable {
*
*/
private static final long serialVersionUID = 1L;
- SVMModel model = null;
- transient SparkContext sc = null;
+ private SVMModel model = null;
+ private transient SparkContext sc = null;
/**
* Constructor to load in spark SVM classifier
*
- * @param classifierName classifier type
* @param skd an instance of spark driver
* @param svmSgdModel path to a trained model
*/
diff --git
a/core/src/main/java/org/apache/sdap/mudrod/ssearch/ranking/SparkFormatter.java
b/core/src/main/java/org/apache/sdap/mudrod/ssearch/ranking/SparkFormatter.java
index 8c7fa7f..0db16f4 100644
---
a/core/src/main/java/org/apache/sdap/mudrod/ssearch/ranking/SparkFormatter.java
+++
b/core/src/main/java/org/apache/sdap/mudrod/ssearch/ranking/SparkFormatter.java
@@ -1,10 +1,33 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.sdap.mudrod.ssearch.ranking;
-import java.io.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
import java.text.DecimalFormat;
public class SparkFormatter {
- DecimalFormat NDForm = new DecimalFormat("#.###");
+
+ private DecimalFormat NDForm = new DecimalFormat("#.###");
+ private static final Logger LOG =
LoggerFactory.getLogger(SparkFormatter.class);
public SparkFormatter() {
}
@@ -17,9 +40,9 @@ public class SparkFormatter {
try {
file.createNewFile();
- try (FileWriter fw = new FileWriter(outputTXTFileName);
- BufferedWriter bw = new BufferedWriter(fw);
- BufferedReader br = new BufferedReader(new
FileReader(inputCSVFileName));) {
+ try (FileWriter fw = new FileWriter(outputTXTFileName);
+ BufferedWriter bw = new BufferedWriter(fw);
+ BufferedReader br = new BufferedReader(new
FileReader(inputCSVFileName));) {
String line = null;
line = br.readLine(); //header
@@ -41,7 +64,7 @@ public class SparkFormatter {
}
}
} catch (IOException e) {
- e.printStackTrace();
+ LOG.error("Writing to output text file failed : ", e);
}
}
}
diff --git
a/core/src/main/java/org/apache/sdap/mudrod/ssearch/ranking/TrainingImporter.java
b/core/src/main/java/org/apache/sdap/mudrod/ssearch/ranking/TrainingImporter.java
index 12fdecb..ebe4628 100644
---
a/core/src/main/java/org/apache/sdap/mudrod/ssearch/ranking/TrainingImporter.java
+++
b/core/src/main/java/org/apache/sdap/mudrod/ssearch/ranking/TrainingImporter.java
@@ -47,7 +47,7 @@ public class TrainingImporter extends MudrodAbstract {
/**
* Method of adding mapping to traning set type
*/
- public void addMapping() {
+ private void addMapping() {
XContentBuilder Mapping;
try {
Mapping =
jsonBuilder().startObject().startObject("trainingranking").startObject("properties").startObject("query").field("type",
"string").field("index", "not_analyzed").endObject()
diff --git
a/core/src/main/java/org/apache/sdap/mudrod/ssearch/structure/SResult.java
b/core/src/main/java/org/apache/sdap/mudrod/ssearch/structure/SResult.java
index 81de2b4..8f7a904 100644
--- a/core/src/main/java/org/apache/sdap/mudrod/ssearch/structure/SResult.java
+++ b/core/src/main/java/org/apache/sdap/mudrod/ssearch/structure/SResult.java
@@ -21,11 +21,11 @@ import java.lang.reflect.Field;
public class SResult {
public static final String rlist[] = { "term_score", "releaseDate_score",
/*"versionNum_score",*/
"processingL_score", "allPop_score", "monthPop_score",
"userPop_score"/*, "termAndv_score"*/ };
- String shortName = null;
- String longName = null;
- String topic = null;
- String description = null;
- String relase_date = null;
+ private String shortName = null;
+ private String longName = null;
+ private String topic = null;
+ private String description = null;
+ private String relase_date = null;
public Double final_score = 0.0;
public Double term_score = 0.0;
diff --git a/core/src/main/java/org/apache/sdap/mudrod/tools/EONETIngester.java
b/core/src/main/java/org/apache/sdap/mudrod/tools/EONETIngester.java
index f3d36b3..4ec5698 100644
--- a/core/src/main/java/org/apache/sdap/mudrod/tools/EONETIngester.java
+++ b/core/src/main/java/org/apache/sdap/mudrod/tools/EONETIngester.java
@@ -50,7 +50,6 @@ import com.google.gson.JsonParser;
* Entry point providing ingestion logic of <a
href="https://eonet.sci.gsfc.nasa.gov/";>
* Earth Observatory Natural Event Tracker (EONET)</a> data into
* the SDAP search server.
- * @param <E>
*/
public class EONETIngester extends MudrodAbstract {
@@ -134,7 +133,7 @@ public class EONETIngester extends MudrodAbstract {
try {
updateResponse = esDriver.getClient().update(updateRequest).get();
} catch (InterruptedException | ExecutionException e) {
- e.printStackTrace();
+ LOG.error("Failed to execute bulk Index request : ", e);
}
if (updateResponse != null) {
result = updateResponse.getGetResult();
@@ -162,7 +161,7 @@ public class EONETIngester extends MudrodAbstract {
.field("geometries", event.get("geometries").getAsJsonArray())
.endObject();
} catch (IOException e) {
- e.printStackTrace();
+ LOG.error("Failed to create event mapping : ", e);
}
} else {
try {
@@ -177,7 +176,7 @@ public class EONETIngester extends MudrodAbstract {
.field("geometries", event.get("geometries").getAsJsonArray())
.endObject();
} catch (IOException e) {
- e.printStackTrace();
+ LOG.error("Failed to create event mapping : ", e);
}
}
return eventMapping;
@@ -200,7 +199,7 @@ public class EONETIngester extends MudrodAbstract {
eventArray.add(jsonElement);
}
} catch (IOException e) {
- e.printStackTrace();
+ LOG.error("Failed to execute HTTP GET " + request.toString() + " : ",
e);
}
}
return eventArray;
diff --git a/core/src/main/java/org/apache/sdap/mudrod/utils/HttpRequest.java
b/core/src/main/java/org/apache/sdap/mudrod/utils/HttpRequest.java
index 6b623f6..ed596f6 100644
--- a/core/src/main/java/org/apache/sdap/mudrod/utils/HttpRequest.java
+++ b/core/src/main/java/org/apache/sdap/mudrod/utils/HttpRequest.java
@@ -54,7 +54,7 @@ public class HttpRequest {
}
} catch (Exception e) {
line = "{\"exception\":\"No service was found\"}";
- LOG.error(line);
+ LOG.error(line, e);
}
return line;
}
diff --git a/core/src/main/java/org/apache/sdap/mudrod/utils/LinkageTriple.java
b/core/src/main/java/org/apache/sdap/mudrod/utils/LinkageTriple.java
index 1ebae17..9f50c98 100644
--- a/core/src/main/java/org/apache/sdap/mudrod/utils/LinkageTriple.java
+++ b/core/src/main/java/org/apache/sdap/mudrod/utils/LinkageTriple.java
@@ -25,6 +25,8 @@ import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.bucket.terms.Terms;
import org.elasticsearch.search.sort.SortOrder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.io.Serializable;
@@ -54,7 +56,9 @@ public class LinkageTriple implements Serializable {
// keyB: TermB
public String keyB;
// df: Format number
- public static DecimalFormat df = new DecimalFormat("#.00");
+ private static DecimalFormat df = new DecimalFormat("#.00");
+
+ private static final Logger LOG =
LoggerFactory.getLogger(LinkageTriple.class);
public LinkageTriple() {
}
@@ -133,7 +137,7 @@ public class LinkageTriple implements Serializable {
es.getClient().admin().indices().preparePutMapping(index).setType(type).setSource(mapping).execute().actionGet();
} catch (IOException e) {
- e.printStackTrace();
+ LOG.error("Failed to add mapping : ", e);
}
}
diff --git a/core/src/main/java/org/apache/sdap/mudrod/utils/MatrixUtil.java
b/core/src/main/java/org/apache/sdap/mudrod/utils/MatrixUtil.java
index ec27a29..e058abe 100644
--- a/core/src/main/java/org/apache/sdap/mudrod/utils/MatrixUtil.java
+++ b/core/src/main/java/org/apache/sdap/mudrod/utils/MatrixUtil.java
@@ -18,13 +18,23 @@ import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.Optional;
-import org.apache.spark.api.java.function.*;
+import org.apache.spark.api.java.function.Function;
+import org.apache.spark.api.java.function.Function2;
+import org.apache.spark.api.java.function.FlatMapFunction;
+import org.apache.spark.api.java.function.PairFlatMapFunction;
+import org.apache.spark.api.java.function.PairFunction;
import org.apache.spark.mllib.feature.IDF;
import org.apache.spark.mllib.feature.IDFModel;
-import org.apache.spark.mllib.linalg.*;
+import org.apache.spark.mllib.linalg.Matrices;
+import org.apache.spark.mllib.linalg.Matrix;
+import org.apache.spark.mllib.linalg.SingularValueDecomposition;
+import org.apache.spark.mllib.linalg.Vector;
+import org.apache.spark.mllib.linalg.Vectors;
import org.apache.spark.mllib.linalg.distributed.IndexedRow;
import org.apache.spark.mllib.linalg.distributed.IndexedRowMatrix;
import org.apache.spark.mllib.linalg.distributed.RowMatrix;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import scala.Tuple2;
import java.io.BufferedWriter;
@@ -42,6 +52,8 @@ import java.util.stream.Stream;
*/
public class MatrixUtil {
+ private static final Logger LOG = LoggerFactory.getLogger(MatrixUtil.class);
+
private MatrixUtil() {
}
@@ -458,7 +470,7 @@ public class MatrixUtil {
try {
file.createNewFile();
} catch (IOException e1) {
- e1.printStackTrace();
+ LOG.error("Failed to create file : ", e1);
}
try (FileWriter fw = new FileWriter(file.getAbsoluteFile());
BufferedWriter bw = new BufferedWriter(fw);){
@@ -479,7 +491,7 @@ public class MatrixUtil {
bw.write(row + "\n");
}
} catch (IOException e) {
- e.printStackTrace();
+ LOG.error("Failed to write to the file : ", e);
}
}
}
diff --git a/core/src/main/java/org/apache/sdap/mudrod/utils/SVDUtil.java
b/core/src/main/java/org/apache/sdap/mudrod/utils/SVDUtil.java
index 1cd4e00..2b07504 100644
--- a/core/src/main/java/org/apache/sdap/mudrod/utils/SVDUtil.java
+++ b/core/src/main/java/org/apache/sdap/mudrod/utils/SVDUtil.java
@@ -21,6 +21,8 @@ import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.mllib.linalg.Vector;
import org.apache.spark.mllib.linalg.distributed.CoordinateMatrix;
import org.apache.spark.mllib.linalg.distributed.RowMatrix;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.List;
@@ -36,12 +38,14 @@ public class SVDUtil extends MudrodAbstract {
*/
private static final long serialVersionUID = 1L;
// wordRDD: terms extracted from all documents
- JavaRDD<String> wordRDD;
+ private JavaRDD<String> wordRDD;
// svdMatrix: svd matrix
private RowMatrix svdMatrix;
// simMatrix: similarity matrix
private CoordinateMatrix simMatrix;
+ private static final Logger LOG = LoggerFactory.getLogger(SVDUtil.class);
+
/**
* Creates a new instance of SVDUtil.
*
@@ -111,7 +115,7 @@ public class SVDUtil extends MudrodAbstract {
try {
LinkageTriple.insertTriples(es, triples, index, type);
} catch (IOException e) {
- e.printStackTrace();
+ LOG.error("Linkage triples insert to elasticsearch failed : ", e);
}
}
diff --git
a/core/src/main/java/org/apache/sdap/mudrod/utils/SimilarityUtil.java
b/core/src/main/java/org/apache/sdap/mudrod/utils/SimilarityUtil.java
index 71179dc..7591e83 100644
--- a/core/src/main/java/org/apache/sdap/mudrod/utils/SimilarityUtil.java
+++ b/core/src/main/java/org/apache/sdap/mudrod/utils/SimilarityUtil.java
@@ -38,7 +38,7 @@ public class SimilarityUtil {
/**
* CalSimilarityFromMatrix: Calculate term similarity from matrix.
*
- * @param svdMatrix. Each row is corresponding to a term, and each column is
+ * @param svdMatrix Each row is corresponding to a term, and each column is
* corresponding to a dimension of feature
* @return CoordinateMatrix, each row is corresponding to a term, and each
* column is also a term, the cell value is the similarity between the
diff --git
a/core/src/main/java/org/apache/sdap/mudrod/weblog/partition/KGreedyPartitionSolver.java
b/core/src/main/java/org/apache/sdap/mudrod/weblog/partition/KGreedyPartitionSolver.java
index 1e25cec..2980307 100644
---
a/core/src/main/java/org/apache/sdap/mudrod/weblog/partition/KGreedyPartitionSolver.java
+++
b/core/src/main/java/org/apache/sdap/mudrod/weblog/partition/KGreedyPartitionSolver.java
@@ -1,10 +1,29 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.sdap.mudrod.weblog.partition;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
public class KGreedyPartitionSolver implements ThePartitionProblemSolver {
- public boolean bsorted = false;
+ private boolean bsorted = false;
public KGreedyPartitionSolver() {
// default constructor
@@ -76,7 +95,7 @@ public class KGreedyPartitionSolver implements
ThePartitionProblemSolver {
return labelGroups;
}
- public Map<String, Double> sortMapByValue(Map<String, Double> passedMap) {
+ private Map<String, Double> sortMapByValue(Map<String, Double> passedMap) {
List<String> mapKeys = new ArrayList<>(passedMap.keySet());
List<Double> mapValues = new ArrayList<>(passedMap.values());
Collections.sort(mapValues, Collections.reverseOrder());
diff --git
a/core/src/main/java/org/apache/sdap/mudrod/weblog/partition/ThePartitionProblemSolver.java
b/core/src/main/java/org/apache/sdap/mudrod/weblog/partition/ThePartitionProblemSolver.java
index 507140f..28972b9 100644
---
a/core/src/main/java/org/apache/sdap/mudrod/weblog/partition/ThePartitionProblemSolver.java
+++
b/core/src/main/java/org/apache/sdap/mudrod/weblog/partition/ThePartitionProblemSolver.java
@@ -1,8 +1,22 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.sdap.mudrod.weblog.partition;
import java.util.Map;
public interface ThePartitionProblemSolver {
- public Map<String, Integer> solve(Map<String, Double> labelNums, int k);
+ Map<String, Integer> solve(Map<String, Double> labelNums, int k);
+
}
diff --git
a/core/src/main/java/org/apache/sdap/mudrod/weblog/partition/logPartitioner.java
b/core/src/main/java/org/apache/sdap/mudrod/weblog/partition/logPartitioner.java
index 7ff2181..615f4b6 100644
---
a/core/src/main/java/org/apache/sdap/mudrod/weblog/partition/logPartitioner.java
+++
b/core/src/main/java/org/apache/sdap/mudrod/weblog/partition/logPartitioner.java
@@ -1,3 +1,16 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.sdap.mudrod.weblog.partition;
import org.apache.spark.Partitioner;
@@ -6,8 +19,8 @@ import java.util.Map;
public class logPartitioner extends Partitioner {
- int num;
- Map<String, Integer> UserGroups;
+ private int num;
+ private Map<String, Integer> UserGroups;
public logPartitioner(int num) {
this.num = num;
diff --git
a/core/src/main/java/org/apache/sdap/mudrod/weblog/pre/CrawlerDetection.java
b/core/src/main/java/org/apache/sdap/mudrod/weblog/pre/CrawlerDetection.java
index 2f37af9..38e7d29 100644
--- a/core/src/main/java/org/apache/sdap/mudrod/weblog/pre/CrawlerDetection.java
+++ b/core/src/main/java/org/apache/sdap/mudrod/weblog/pre/CrawlerDetection.java
@@ -107,7 +107,7 @@ public class CrawlerDetection extends LogAbstract {
return false;
}
- void checkByRateInParallel() throws InterruptedException, IOException {
+ private void checkByRateInParallel() throws InterruptedException,
IOException {
JavaRDD<String> userRDD = getUserRDD(this.httpType);
LOG.info("Original User count: {}", userRDD.count());
diff --git
a/core/src/main/java/org/apache/sdap/mudrod/weblog/pre/HistoryGenerator.java
b/core/src/main/java/org/apache/sdap/mudrod/weblog/pre/HistoryGenerator.java
index e120765..08c8c6c 100644
--- a/core/src/main/java/org/apache/sdap/mudrod/weblog/pre/HistoryGenerator.java
+++ b/core/src/main/java/org/apache/sdap/mudrod/weblog/pre/HistoryGenerator.java
@@ -28,7 +28,12 @@ import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
-import java.util.*;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
/**
* Supports ability to generate search history (queries) for each individual
@@ -58,7 +63,7 @@ public class HistoryGenerator extends LogAbstract {
* Method to generate a binary user*query matrix (stored in temporary .csv
* file)
*/
- public void generateBinaryMatrix() {
+ private void generateBinaryMatrix() {
try {
File file = new
File(props.getProperty(MudrodConstants.USER_HISTORY_PATH));
if (file.exists()) {
@@ -149,7 +154,7 @@ public class HistoryGenerator extends LogAbstract {
}
fw.close();
} catch (IOException e) {
- e.printStackTrace();
+ LOG.error("Failed to generate Binary Matrix : ", e);
}
}
diff --git
a/core/src/main/java/org/apache/sdap/mudrod/weblog/pre/ImportLogFile.java
b/core/src/main/java/org/apache/sdap/mudrod/weblog/pre/ImportLogFile.java
index 6e5f473..7eb33d7 100644
--- a/core/src/main/java/org/apache/sdap/mudrod/weblog/pre/ImportLogFile.java
+++ b/core/src/main/java/org/apache/sdap/mudrod/weblog/pre/ImportLogFile.java
@@ -19,24 +19,15 @@ import org.apache.sdap.mudrod.main.MudrodConstants;
import org.apache.sdap.mudrod.weblog.structure.log.ApacheAccessLog;
import org.apache.sdap.mudrod.weblog.structure.log.FtpLog;
import org.apache.spark.api.java.JavaRDD;
-import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.spark.rdd.api.java.JavaEsSpark;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.BufferedReader;
import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FileReader;
-import java.io.IOException;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.Date;
import java.util.Properties;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
/**
* Supports ability to parse and process FTP and HTTP log files
@@ -50,11 +41,11 @@ public class ImportLogFile extends LogAbstract {
*/
private static final long serialVersionUID = 1L;
- String logEntryPattern = "^([\\d.]+) (\\S+) (\\S+)
\\[([\\w:/]+\\s[+\\-]\\d{4})\\] " + "\"(.+?)\" (\\d{3}) (\\d+|-)
\"((?:[^\"]|\")+)\" \"([^\"]+)\"";
+ private String logEntryPattern = "^([\\d.]+) (\\S+) (\\S+)
\\[([\\w:/]+\\s[+\\-]\\d{4})\\] " + "\"(.+?)\" (\\d{3}) (\\d+|-)
\"((?:[^\"]|\")+)\" \"([^\"]+)\"";
- public static final int NUM_FIELDS = 9;
- Pattern p = Pattern.compile(logEntryPattern);
- transient Matcher matcher;
+ private static final int NUM_FIELDS = 9;
+ private Pattern p = Pattern.compile(logEntryPattern);
+ private transient Matcher matcher;
@Override
public Object execute(Object o) {
@@ -123,7 +114,7 @@ public class ImportLogFile extends LogAbstract {
return newTime;
}
- public void readFile() {
+ private void readFile() {
String httplogpath = null;
String ftplogpath = null;
@@ -161,18 +152,18 @@ public class ImportLogFile extends LogAbstract {
* @param httplogpath path to the parent directory containing http logs
* @param ftplogpath path to the parent directory containing ftp logs
*/
- public void readFileInParallel(String httplogpath, String ftplogpath) {
+ private void readFileInParallel(String httplogpath, String ftplogpath) {
importHttpfile(httplogpath);
importFtpfile(ftplogpath);
}
- public void importHttpfile(String httplogpath) {
+ private void importHttpfile(String httplogpath) {
// import http logs
JavaRDD<String> accessLogs = spark.sc.textFile(httplogpath,
this.partition).map(s -> ApacheAccessLog.parseFromLogLine(s,
props)).filter(ApacheAccessLog::checknull);
JavaEsSpark.saveJsonToEs(accessLogs, logIndex + "/" + this.httpType);
}
- public void importFtpfile(String ftplogpath) {
+ private void importFtpfile(String ftplogpath) {
// import ftp logs
JavaRDD<String> ftpLogs = spark.sc.textFile(ftplogpath,
this.partition).map(s -> FtpLog.parseFromLogLine(s,
props)).filter(FtpLog::checknull);
JavaEsSpark.saveJsonToEs(ftpLogs, logIndex + "/" + this.ftpType);
diff --git
a/core/src/main/java/org/apache/sdap/mudrod/weblog/pre/LogAbstract.java
b/core/src/main/java/org/apache/sdap/mudrod/weblog/pre/LogAbstract.java
index 1d04902..a489844 100644
--- a/core/src/main/java/org/apache/sdap/mudrod/weblog/pre/LogAbstract.java
+++ b/core/src/main/java/org/apache/sdap/mudrod/weblog/pre/LogAbstract.java
@@ -1,3 +1,16 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.sdap.mudrod.weblog.pre;
import org.apache.commons.io.IOUtils;
@@ -27,7 +40,11 @@ import scala.Tuple2;
import java.io.IOException;
import java.io.InputStream;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
public class LogAbstract extends DiscoveryStepAbstract {
diff --git
a/core/src/main/java/org/apache/sdap/mudrod/weblog/pre/RankingTrainDataGenerator.java
b/core/src/main/java/org/apache/sdap/mudrod/weblog/pre/RankingTrainDataGenerator.java
index de41d56..85d86ec 100644
---
a/core/src/main/java/org/apache/sdap/mudrod/weblog/pre/RankingTrainDataGenerator.java
+++
b/core/src/main/java/org/apache/sdap/mudrod/weblog/pre/RankingTrainDataGenerator.java
@@ -1,3 +1,16 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.sdap.mudrod.weblog.pre;
import org.apache.sdap.mudrod.discoveryengine.DiscoveryStepAbstract;
@@ -37,7 +50,7 @@ public class RankingTrainDataGenerator extends
DiscoveryStepAbstract {
rankingTrainData_JsonRDD.coalesce(1,
true).saveAsTextFile(rankingTrainFile);
} catch (Exception e) {
- e.printStackTrace();
+ LOG.error("Train data ranking failed : ", e);
}
endTime = System.currentTimeMillis();
diff --git
a/core/src/main/java/org/apache/sdap/mudrod/weblog/pre/SessionGenerator.java
b/core/src/main/java/org/apache/sdap/mudrod/weblog/pre/SessionGenerator.java
index 6d779e8..dcdbec1 100644
--- a/core/src/main/java/org/apache/sdap/mudrod/weblog/pre/SessionGenerator.java
+++ b/core/src/main/java/org/apache/sdap/mudrod/weblog/pre/SessionGenerator.java
@@ -42,7 +42,13 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
@@ -72,7 +78,7 @@ public class SessionGenerator extends LogAbstract {
return null;
}
- public void generateSession() {
+ private void generateSession() {
try {
es.createBulkProcessor();
genSessionByReferer(Integer.parseInt(props.getProperty(MudrodConstants.REQUEST_TIME_GAP)));
@@ -86,18 +92,18 @@ public class SessionGenerator extends LogAbstract {
} catch (IOException e) {
LOG.error("Error whilst reading configuration.", e);
} catch (NumberFormatException e) {
- e.printStackTrace();
+ LOG.error("Error whilst parsing Integer configuration parameter.", e);
} catch (InterruptedException e) {
- e.printStackTrace();
+ LOG.error("Current running thread interrupted.", e);
Thread.currentThread().interrupt();
}
}
- public void genSessionByReferer(int timeThres) throws InterruptedException,
IOException {
+ private void genSessionByReferer(int timeThres) throws InterruptedException,
IOException {
genSessionByRefererInParallel(timeThres);
}
- public void combineShortSessions(int timeThres) throws InterruptedException,
IOException {
+ private void combineShortSessions(int timeThres) throws
InterruptedException, IOException {
combineShortSessionsInParallel(timeThres);
}
@@ -109,7 +115,7 @@ public class SessionGenerator extends LogAbstract {
* @throws ElasticsearchException ElasticsearchException
* @throws IOException IOException
*/
- public void deleteInvalid(ESDriver es, String ip) throws IOException {
+ private void deleteInvalid(ESDriver es, String ip) throws IOException {
BoolQueryBuilder filterAll = new BoolQueryBuilder();
filterAll.must(QueryBuilders.termQuery("IP", ip));
@@ -141,7 +147,7 @@ public class SessionGenerator extends LogAbstract {
/**
* Method to update a Elasticsearch record/document by id, field, and value
*
- * @param es
+ * @param es Elasticsearch Driver
* @param index index name is Elasticsearch
* @param type type name
* @param id ID of the document that needs to be updated
@@ -156,7 +162,7 @@ public class SessionGenerator extends LogAbstract {
es.getBulkProcessor().add(ur);
}
- public void genSessionByRefererInParallel(int timeThres) throws
InterruptedException, IOException {
+ private void genSessionByRefererInParallel(int timeThres) throws
InterruptedException, IOException {
JavaRDD<String> userRDD = getUserRDD(this.cleanupType);
int sessionCount = 0;
@@ -195,7 +201,7 @@ public class SessionGenerator extends LogAbstract {
LOG.info("Initial Session count: {}", Integer.toString(sessionCount));
}
- public int genSessionByReferer(ESDriver es, String user, int timeThres)
throws ElasticsearchException, IOException {
+ private int genSessionByReferer(ESDriver es, String user, int timeThres)
throws ElasticsearchException, IOException {
String startTime = null;
int sessionCountIn = 0;
@@ -316,7 +322,7 @@ public class SessionGenerator extends LogAbstract {
return sessionCountIn;
}
- public void combineShortSessionsInParallel(int timeThres) throws
InterruptedException, IOException {
+ private void combineShortSessionsInParallel(int timeThres) throws
InterruptedException, IOException {
JavaRDD<String> userRDD = getUserRDD(this.cleanupType);
@@ -341,7 +347,7 @@ public class SessionGenerator extends LogAbstract {
LOG.info("Final Session count (after combining short sessions): {}",
Long.toString(userRDD.count()));
}
- public void combineShortSessions(ESDriver es, String user, int timeThres)
throws ElasticsearchException, IOException {
+ private void combineShortSessions(ESDriver es, String user, int timeThres)
throws ElasticsearchException, IOException {
BoolQueryBuilder filterSearch = new BoolQueryBuilder();
filterSearch.must(QueryBuilders.termQuery("IP", user));
diff --git
a/core/src/main/java/org/apache/sdap/mudrod/weblog/pre/SessionStatistic.java
b/core/src/main/java/org/apache/sdap/mudrod/weblog/pre/SessionStatistic.java
index c89160f..2846979 100644
--- a/core/src/main/java/org/apache/sdap/mudrod/weblog/pre/SessionStatistic.java
+++ b/core/src/main/java/org/apache/sdap/mudrod/weblog/pre/SessionStatistic.java
@@ -71,13 +71,11 @@ public class SessionStatistic extends LogAbstract {
startTime = System.currentTimeMillis();
try {
processSession();
- } catch (IOException e) {
- e.printStackTrace();
+ } catch (IOException | ExecutionException e) {
+ LOG.error("Session summarization process failed.", e);
} catch (InterruptedException e) {
- e.printStackTrace();
+ LOG.error("Current running thread interrupted.", e);
Thread.currentThread().interrupt();
- } catch (ExecutionException e) {
- e.printStackTrace();
}
endTime = System.currentTimeMillis();
es.refreshIndex();
@@ -85,7 +83,7 @@ public class SessionStatistic extends LogAbstract {
return null;
}
- public void processSession() throws InterruptedException, IOException,
ExecutionException {
+ private void processSession() throws InterruptedException, IOException,
ExecutionException {
processSessionInParallel();
}
@@ -95,7 +93,7 @@ public class SessionStatistic extends LogAbstract {
* @param request raw log request
* @return dataset ID
*/
- public String findDataset(String request) {
+ private String findDataset(String request) {
String pattern1 = props.getProperty(MudrodConstants.VIEW_MARKER);
String pattern2;
if (request.contains("?")) {
@@ -112,7 +110,7 @@ public class SessionStatistic extends LogAbstract {
return null;
}
- public void processSessionInParallel() throws InterruptedException,
IOException {
+ private void processSessionInParallel() throws InterruptedException,
IOException {
List<String> sessions = this.getSessions();
JavaRDD<String> sessionRDD = spark.sc.parallelize(sessions, partition);
@@ -144,7 +142,7 @@ public class SessionStatistic extends LogAbstract {
LOG.info("Final Session count: {}", Integer.toString(sessionCount));
}
- public int processSession(ESDriver es, String sessionId) throws IOException,
InterruptedException, ExecutionException {
+ private int processSession(ESDriver es, String sessionId) throws
IOException, InterruptedException, ExecutionException {
String inputType = cleanupType;
String outputType = sessionStats;
diff --git
a/core/src/main/java/org/apache/sdap/mudrod/weblog/structure/log/ApacheAccessLog.java
b/core/src/main/java/org/apache/sdap/mudrod/weblog/structure/log/ApacheAccessLog.java
index 6ffd3be..8887dee 100644
---
a/core/src/main/java/org/apache/sdap/mudrod/weblog/structure/log/ApacheAccessLog.java
+++
b/core/src/main/java/org/apache/sdap/mudrod/weblog/structure/log/ApacheAccessLog.java
@@ -18,7 +18,6 @@ import com.google.gson.Gson;
import org.apache.sdap.mudrod.main.MudrodConstants;
import org.apache.sdap.mudrod.weblog.pre.CrawlerDetection;
-import java.io.IOException;
import java.io.Serializable;
import java.text.ParseException;
import java.text.SimpleDateFormat;
@@ -32,9 +31,10 @@ import java.util.regex.Pattern;
* http://httpd.apache.org/docs/2.2/logs.html for more details.
*/
public class ApacheAccessLog extends WebLog implements Serializable {
- String Response;
- String Referer;
- String Browser;
+
+ private String Response;
+ private String Referer;
+ private String Browser;
@Override
public double getBytes() {
@@ -110,7 +110,7 @@ public class ApacheAccessLog extends WebLog implements
Serializable {
lineJson = gson.toJson(accesslog);
return lineJson;
- }
+ }
}
public static boolean checknull(WebLog s) {
diff --git
a/core/src/main/java/org/apache/sdap/mudrod/weblog/structure/log/Coordinates.java
b/core/src/main/java/org/apache/sdap/mudrod/weblog/structure/log/Coordinates.java
index 43a7642..04d811a 100644
---
a/core/src/main/java/org/apache/sdap/mudrod/weblog/structure/log/Coordinates.java
+++
b/core/src/main/java/org/apache/sdap/mudrod/weblog/structure/log/Coordinates.java
@@ -18,4 +18,5 @@ public class Coordinates {
* public String lat; public String lon;
*/
public String latlon;
+
}
diff --git
a/core/src/main/java/org/apache/sdap/mudrod/weblog/structure/log/RequestUrl.java
b/core/src/main/java/org/apache/sdap/mudrod/weblog/structure/log/RequestUrl.java
index 0c365a4..fe76e76 100644
---
a/core/src/main/java/org/apache/sdap/mudrod/weblog/structure/log/RequestUrl.java
+++
b/core/src/main/java/org/apache/sdap/mudrod/weblog/structure/log/RequestUrl.java
@@ -87,7 +87,7 @@ public class RequestUrl {
* @param URL request url
* @return url params key value map
*/
- public static Map<String, String> uRLRequest(String URL) {
+ private static Map<String, String> uRLRequest(String URL) {
Map<String, String> mapRequest = new HashMap<String, String>();
String[] arrSplit = null;
@@ -213,8 +213,7 @@ public class RequestUrl {
keyword = keyword.replaceAll("[-+^:,*_\"]", " ").replace("\\", "
").replaceAll("\\s+", " ").trim();
} catch (UnsupportedEncodingException e) {
- LOG.error(mapRequest.get("search"));
- e.printStackTrace();
+ LOG.error(mapRequest.get("search"), e);
}
if (!"".equals(keyword)) {
info.add(keyword);
@@ -246,8 +245,7 @@ public class RequestUrl {
filterValues.put(ids[i], item);
}
} catch (Exception e) {
- LOG.error(values[i]);
- e.printStackTrace();
+ LOG.error("Error occurred while decoding value " + values[i], e);
}
}
}
diff --git
a/core/src/main/java/org/apache/sdap/mudrod/weblog/structure/session/ClickStream.java
b/core/src/main/java/org/apache/sdap/mudrod/weblog/structure/session/ClickStream.java
index 7e5069e..5d698ed 100644
---
a/core/src/main/java/org/apache/sdap/mudrod/weblog/structure/session/ClickStream.java
+++
b/core/src/main/java/org/apache/sdap/mudrod/weblog/structure/session/ClickStream.java
@@ -15,6 +15,8 @@ package org.apache.sdap.mudrod.weblog.structure.session;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.Serializable;
@@ -37,6 +39,9 @@ public class ClickStream implements Serializable {
// type: session type name
private String type;
+ private static final Logger LOG = LoggerFactory.getLogger(ClickStream.class);
+
+
/**
* Creates a new instance of ClickStream.
*
@@ -180,7 +185,7 @@ public class ClickStream implements Serializable {
data.setDownloadDataset(jsonData.getString("downloaddataset"));
} catch (JSONException e) {
- e.printStackTrace();
+ LOG.error("Error occurred while parsing the json log line : ", e);
}
return data;
diff --git
a/core/src/main/java/org/apache/sdap/mudrod/weblog/structure/session/RankingTrainData.java
b/core/src/main/java/org/apache/sdap/mudrod/weblog/structure/session/RankingTrainData.java
index bdf477a..8199617 100644
---
a/core/src/main/java/org/apache/sdap/mudrod/weblog/structure/session/RankingTrainData.java
+++
b/core/src/main/java/org/apache/sdap/mudrod/weblog/structure/session/RankingTrainData.java
@@ -1,3 +1,16 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.sdap.mudrod.weblog.structure.session;
import java.io.Serializable;
diff --git
a/core/src/main/java/org/apache/sdap/mudrod/weblog/structure/session/Session.java
b/core/src/main/java/org/apache/sdap/mudrod/weblog/structure/session/Session.java
index 2c917a6..a3efeab 100644
---
a/core/src/main/java/org/apache/sdap/mudrod/weblog/structure/session/Session.java
+++
b/core/src/main/java/org/apache/sdap/mudrod/weblog/structure/session/Session.java
@@ -166,7 +166,6 @@ public class Session /*extends MudrodAbstract*/ implements
Comparable<Session> {
sessionResults.add("RequestList", jsonRequest);
} catch (UnsupportedEncodingException e) {
LOG.error("Encoding error detected.", e);
-
}
return sessionResults;
@@ -283,7 +282,7 @@ public class Session /*extends MudrodAbstract*/ implements
Comparable<Session> {
try {
trainData = tree.getRankingTrainData(indexName);
} catch (UnsupportedEncodingException e) {
- LOG.error("Error whilst retreiving ranking training data: {}", e);
+ LOG.error("Error whilst retrieving ranking training data: {}", e);
}
return trainData;
diff --git
a/core/src/main/java/org/apache/sdap/mudrod/weblog/structure/session/SessionExtractor.java
b/core/src/main/java/org/apache/sdap/mudrod/weblog/structure/session/SessionExtractor.java
index f7eb602..8286e8c 100644
---
a/core/src/main/java/org/apache/sdap/mudrod/weblog/structure/session/SessionExtractor.java
+++
b/core/src/main/java/org/apache/sdap/mudrod/weblog/structure/session/SessionExtractor.java
@@ -73,7 +73,7 @@ public class SessionExtractor implements Serializable {
return getClickStreamListInParallel(props, spark, es);
}
- protected JavaRDD<ClickStream> getClickStreamListInParallel(Properties
props, SparkDriver spark, ESDriver es) {
+ private JavaRDD<ClickStream> getClickStreamListInParallel(Properties props,
SparkDriver spark, ESDriver es) {
List<String> logIndexList =
es.getIndexListWithPrefix(props.getProperty(MudrodConstants.LOG_INDEX));
@@ -205,7 +205,7 @@ public class SessionExtractor implements Serializable {
* a log index name
* @return list of session names
*/
- protected List<String> getSessions(Properties props, ESDriver es, String
logIndex) {
+ private List<String> getSessions(Properties props, ESDriver es, String
logIndex) {
String cleanupType = MudrodConstants.CLEANUP_TYPE;
String sessionStatType = MudrodConstants.SESSION_STATS_TYPE;
@@ -412,7 +412,7 @@ public class SessionExtractor implements Serializable {
* the Elasticsearch driver
* @return clickstream list {@link ClickStream}
*/
- protected List<RankingTrainData> extractRankingTrainData(Properties props,
ESDriver es) {
+ private List<RankingTrainData> extractRankingTrainData(Properties props,
ESDriver es) {
List<String> logIndexList =
es.getIndexListWithPrefix(props.getProperty(MudrodConstants.LOG_INDEX));
LOG.info(logIndexList.toString());
@@ -436,7 +436,7 @@ public class SessionExtractor implements Serializable {
return result;
}
- protected JavaRDD<RankingTrainData>
extractRankingTrainDataInParallel(Properties props, SparkDriver spark, ESDriver
es) {
+ public JavaRDD<RankingTrainData>
extractRankingTrainDataInParallel(Properties props, SparkDriver spark, ESDriver
es) {
List<String> logIndexList =
es.getIndexListWithPrefix(props.getProperty(MudrodConstants.LOG_INDEX));
diff --git
a/core/src/main/java/org/apache/sdap/mudrod/weblog/structure/session/SessionNode.java
b/core/src/main/java/org/apache/sdap/mudrod/weblog/structure/session/SessionNode.java
index 91a29e1..b893d6b 100644
---
a/core/src/main/java/org/apache/sdap/mudrod/weblog/structure/session/SessionNode.java
+++
b/core/src/main/java/org/apache/sdap/mudrod/weblog/structure/session/SessionNode.java
@@ -15,7 +15,12 @@ package org.apache.sdap.mudrod.weblog.structure.session;
import java.io.UnsupportedEncodingException;
import java.net.URLDecoder;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -158,7 +163,7 @@ public class SessionNode {
* @param request request url
* @param logType url type
*/
- public void setKey(Properties props, String request, String logType) {
+ private void setKey(Properties props, String request, String logType) {
this.key = "";
String datasetlist = props.getProperty(MudrodConstants.SEARCH_MARKER);
String dataset = props.getProperty(MudrodConstants.VIEW_MARKER);
@@ -283,7 +288,7 @@ public class SessionNode {
*
* @param request request url of this node
*/
- public void parseRequest(String request) {
+ private void parseRequest(String request) {
Pattern pattern = Pattern.compile("get (.*?) http/*");
Matcher matcher = pattern.matcher(request.trim().toLowerCase());
while (matcher.find()) {
@@ -330,7 +335,7 @@ public class SessionNode {
*
* @param request request url
*/
- public void parseDatasetId(String request) {
+ private void parseDatasetId(String request) {
try {
request = URLDecoder.decode(request, "UTF-8");
} catch (UnsupportedEncodingException e) {
diff --git
a/core/src/test/java/org/apache/sdap/mudrod/ontology/process/TestLocalOntology.java
b/core/src/test/java/org/apache/sdap/mudrod/ontology/process/TestLocalOntology.java
index 192a29b..d332f5d 100644
---
a/core/src/test/java/org/apache/sdap/mudrod/ontology/process/TestLocalOntology.java
+++
b/core/src/test/java/org/apache/sdap/mudrod/ontology/process/TestLocalOntology.java
@@ -13,11 +13,13 @@
*/
package org.apache.sdap.mudrod.ontology.process;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
import java.io.IOException;
-import org.apache.sdap.mudrod.ontology.process.LocalOntology;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
diff --git
a/service/src/main/java/org/apache/sdap/mudrod/services/MudrodContextListener.java
b/service/src/main/java/org/apache/sdap/mudrod/services/MudrodContextListener.java
index 7d89508..96693db 100644
---
a/service/src/main/java/org/apache/sdap/mudrod/services/MudrodContextListener.java
+++
b/service/src/main/java/org/apache/sdap/mudrod/services/MudrodContextListener.java
@@ -33,7 +33,7 @@ import java.util.Properties;
@WebListener
public class MudrodContextListener implements ServletContextListener {
- MudrodEngine me = null;
+ private MudrodEngine me = null;
/**
* Default constructor.
diff --git
a/service/src/main/java/org/apache/sdap/mudrod/services/autocomplete/AutoCompleteResource.java
b/service/src/main/java/org/apache/sdap/mudrod/services/autocomplete/AutoCompleteResource.java
index fb9a036..87f1b36 100644
---
a/service/src/main/java/org/apache/sdap/mudrod/services/autocomplete/AutoCompleteResource.java
+++
b/service/src/main/java/org/apache/sdap/mudrod/services/autocomplete/AutoCompleteResource.java
@@ -21,7 +21,11 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.servlet.ServletContext;
-import javax.ws.rs.*;
+import javax.ws.rs.Consumes;
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
diff --git
a/service/src/main/java/org/apache/sdap/mudrod/services/search/SearchDatasetDetailResource.java
b/service/src/main/java/org/apache/sdap/mudrod/services/search/SearchDatasetDetailResource.java
index d31d1f4..095bceb 100644
---
a/service/src/main/java/org/apache/sdap/mudrod/services/search/SearchDatasetDetailResource.java
+++
b/service/src/main/java/org/apache/sdap/mudrod/services/search/SearchDatasetDetailResource.java
@@ -19,7 +19,11 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.servlet.ServletContext;
-import javax.ws.rs.*;
+import javax.ws.rs.Consumes;
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
diff --git
a/service/src/main/java/org/apache/sdap/mudrod/services/search/SearchMetadataResource.java
b/service/src/main/java/org/apache/sdap/mudrod/services/search/SearchMetadataResource.java
index 9bdf455..b385d04 100644
---
a/service/src/main/java/org/apache/sdap/mudrod/services/search/SearchMetadataResource.java
+++
b/service/src/main/java/org/apache/sdap/mudrod/services/search/SearchMetadataResource.java
@@ -25,7 +25,11 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.servlet.ServletContext;
-import javax.ws.rs.*;
+import javax.ws.rs.Consumes;
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
diff --git
a/service/src/main/java/org/apache/sdap/mudrod/services/search/SearchVocabResource.java
b/service/src/main/java/org/apache/sdap/mudrod/services/search/SearchVocabResource.java
index 6e58761..c9b0d73 100644
---
a/service/src/main/java/org/apache/sdap/mudrod/services/search/SearchVocabResource.java
+++
b/service/src/main/java/org/apache/sdap/mudrod/services/search/SearchVocabResource.java
@@ -22,7 +22,11 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.servlet.ServletContext;
-import javax.ws.rs.*;
+import javax.ws.rs.Consumes;
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
diff --git
a/service/src/main/java/org/apache/sdap/mudrod/services/search/SessionDetailResource.java
b/service/src/main/java/org/apache/sdap/mudrod/services/search/SessionDetailResource.java
index 9990c7a..6bd4e61 100644
---
a/service/src/main/java/org/apache/sdap/mudrod/services/search/SessionDetailResource.java
+++
b/service/src/main/java/org/apache/sdap/mudrod/services/search/SessionDetailResource.java
@@ -23,7 +23,12 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.servlet.ServletContext;
-import javax.ws.rs.*;
+import javax.ws.rs.Consumes;
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.POST;
+import javax.ws.rs.Produces;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
