This is an automated email from the ASF dual-hosted git repository.
lewismc pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-mudrod.git
The following commit(s) were added to refs/heads/master by this push:
new 54b7780 SDAP-182 Localization and charset definitions are done. (#42)
54b7780 is described below
commit 54b778008350e85eb798bf2e2c127a36f0e21b4a
Author: Furkan KAMACI <[email protected]>
AuthorDate: Tue Aug 6 19:06:38 2019 +0300
SDAP-182 Localization and charset definitions are done. (#42)
* SDAP-182 Localization and charset definitions are done.
* Fix for imports.
* Locale.ROOT is replaced with Locale.ENGLISH.
---
.../org/apache/sdap/mudrod/driver/ESDriver.java | 5 ++--
.../mudrod/integration/LinkageIntegration.java | 7 +++++-
.../sdap/mudrod/metadata/pre/ApiHarvester.java | 13 ++++++-----
.../sdap/mudrod/ontology/pre/AggregateTriples.java | 24 ++++++++-----------
.../mudrod/ontology/process/LocalOntology.java | 5 ++--
.../mudrod/ontology/process/OntologyLinkCal.java | 9 +++++---
.../recommendation/pre/SessionCooccurence.java | 3 ++-
.../process/FeatureBasedSimilarity.java | 7 +++++-
.../structure/HybridRecommendation.java | 6 ++++-
.../sdap/mudrod/ssearch/ClickStreamImporter.java | 6 +++--
.../org/apache/sdap/mudrod/ssearch/Dispatcher.java | 5 ++--
.../org/apache/sdap/mudrod/ssearch/Ranker.java | 6 ++++-
.../org/apache/sdap/mudrod/ssearch/Searcher.java | 13 +++++++----
.../sdap/mudrod/ssearch/ranking/DataGenerator.java | 18 +++++++--------
.../mudrod/ssearch/ranking/SparkFormatter.java | 27 +++++++++++++---------
.../mudrod/ssearch/ranking/TrainingImporter.java | 6 +++--
.../org/apache/sdap/mudrod/tools/InstallCert.java | 4 ++--
.../org/apache/sdap/mudrod/utils/HttpRequest.java | 3 ++-
.../apache/sdap/mudrod/utils/LinkageTriple.java | 7 +++++-
.../org/apache/sdap/mudrod/utils/MatrixUtil.java | 12 +++++-----
.../sdap/mudrod/weblog/pre/CrawlerDetection.java | 5 ++--
.../sdap/mudrod/weblog/pre/HistoryGenerator.java | 24 +++++++++----------
.../sdap/mudrod/weblog/pre/SessionStatistic.java | 5 ++--
.../weblog/structure/log/ApacheAccessLog.java | 7 +++---
.../sdap/mudrod/weblog/structure/log/FtpLog.java | 7 +++---
.../mudrod/weblog/structure/log/RequestUrl.java | 5 ++--
.../weblog/structure/session/SessionNode.java | 18 ++++++++-------
27 files changed, 152 insertions(+), 105 deletions(-)
diff --git a/core/src/main/java/org/apache/sdap/mudrod/driver/ESDriver.java
b/core/src/main/java/org/apache/sdap/mudrod/driver/ESDriver.java
index b761d3b..0009d18 100644
--- a/core/src/main/java/org/apache/sdap/mudrod/driver/ESDriver.java
+++ b/core/src/main/java/org/apache/sdap/mudrod/driver/ESDriver.java
@@ -69,6 +69,7 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Properties;
@@ -160,7 +161,7 @@ public class ESDriver implements Serializable {
}
public String customAnalyzing(String indexName, String analyzer, String str)
throws InterruptedException, ExecutionException {
- String[] strList = str.toLowerCase().split(",");
+ String[] strList = str.toLowerCase(Locale.ENGLISH).split(",");
for (int i = 0; i < strList.length; i++) {
String tmp = "";
AnalyzeResponse r =
client.admin().indices().prepareAnalyze(strList[i]).setIndex(indexName).setAnalyzer(analyzer).execute().get();
@@ -451,7 +452,7 @@ public class ESDriver implements Serializable {
while (iterator.hasNext()) {
Suggest.Suggestion.Entry.Option next = iterator.next();
- String suggest = next.getText().string().toLowerCase();
+ String suggest = next.getText().string().toLowerCase(Locale.ENGLISH);
suggestList.add(suggest);
}
diff --git
a/core/src/main/java/org/apache/sdap/mudrod/integration/LinkageIntegration.java
b/core/src/main/java/org/apache/sdap/mudrod/integration/LinkageIntegration.java
index 21aa5ba..a20184e 100644
---
a/core/src/main/java/org/apache/sdap/mudrod/integration/LinkageIntegration.java
+++
b/core/src/main/java/org/apache/sdap/mudrod/integration/LinkageIntegration.java
@@ -29,12 +29,14 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.text.DecimalFormat;
+import java.text.NumberFormat;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.LinkedHashMap;
+import java.util.Locale;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Properties;
@@ -49,12 +51,15 @@ public class LinkageIntegration extends
DiscoveryStepAbstract {
private static final Logger LOG =
LoggerFactory.getLogger(LinkageIntegration.class);
private static final long serialVersionUID = 1L;
private transient List<LinkedTerm> termList = new ArrayList<>();
- private DecimalFormat df = new DecimalFormat("#.00");
+ private DecimalFormat df;
private static final String INDEX_NAME = MudrodConstants.ES_INDEX_NAME;
private static final String WEIGHT = "weight";
public LinkageIntegration(Properties props, ESDriver es, SparkDriver spark) {
super(props, es, spark);
+ NumberFormat nf = NumberFormat.getNumberInstance(Locale.ENGLISH);
+ df = (DecimalFormat) nf;
+ df.applyPattern("#.00");
}
/**
diff --git
a/core/src/main/java/org/apache/sdap/mudrod/metadata/pre/ApiHarvester.java
b/core/src/main/java/org/apache/sdap/mudrod/metadata/pre/ApiHarvester.java
index 48127ad..6dd7a53 100644
--- a/core/src/main/java/org/apache/sdap/mudrod/metadata/pre/ApiHarvester.java
+++ b/core/src/main/java/org/apache/sdap/mudrod/metadata/pre/ApiHarvester.java
@@ -28,14 +28,15 @@ import org.elasticsearch.action.index.IndexRequest;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
-import java.io.FileWriter;
+import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
+import java.io.OutputStreamWriter;
+import java.nio.charset.StandardCharsets;
import java.util.Properties;
/**
@@ -64,7 +65,7 @@ public class ApiHarvester extends DiscoveryStepAbstract {
//remove old metadata from ES
es.deleteType(props.getProperty(MudrodConstants.ES_INDEX_NAME),
props.getProperty(MudrodConstants.RAW_METADATA_TYPE));
//harvest new metadata using PO.DAAC web services
- if("1".equals(props.getProperty(MudrodConstants.METADATA_DOWNLOAD)))
+ if("1".equals(props.getProperty(MudrodConstants.METADATA_DOWNLOAD)))
harvestMetadatafromWeb();
es.createBulkProcessor();
addMetadataMapping();
@@ -157,9 +158,9 @@ public class ApiHarvester extends DiscoveryStepAbstract {
int docId = startIndex + i;
File itemfile = new
File(props.getProperty(MudrodConstants.RAW_METADATA_PATH) + "/" + docId +
".json");
- try (FileWriter fw = new FileWriter(itemfile.getAbsoluteFile());
BufferedWriter bw = new BufferedWriter(fw)) {
+ try (OutputStreamWriter osw = new OutputStreamWriter(new
FileOutputStream(itemfile.getAbsoluteFile()), StandardCharsets.UTF_8)) {
itemfile.createNewFile();
- bw.write(item.toString());
+ osw.write(item.toString());
} catch (IOException e) {
LOG.error("Error writing metadata to local file!", e);
}
@@ -175,7 +176,7 @@ public class ApiHarvester extends DiscoveryStepAbstract {
}
} while (docLength != 0);
-
+
LOG.info("Metadata downloading finished");
}
diff --git
a/core/src/main/java/org/apache/sdap/mudrod/ontology/pre/AggregateTriples.java
b/core/src/main/java/org/apache/sdap/mudrod/ontology/pre/AggregateTriples.java
index f92d423..91fb5db 100644
---
a/core/src/main/java/org/apache/sdap/mudrod/ontology/pre/AggregateTriples.java
+++
b/core/src/main/java/org/apache/sdap/mudrod/ontology/pre/AggregateTriples.java
@@ -27,10 +27,11 @@ import org.jdom2.input.SAXBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.BufferedWriter;
import java.io.File;
-import java.io.FileWriter;
+import java.io.FileOutputStream;
import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
@@ -62,8 +63,8 @@ public class AggregateTriples extends DiscoveryStepAbstract {
LOG.error("File creation failed: ", e2);
}
- try(FileWriter fw = new FileWriter(file.getAbsoluteFile())){
- bw = new BufferedWriter(fw);
+ try(OutputStreamWriter osw = new OutputStreamWriter(new
FileOutputStream(file.getAbsoluteFile()), StandardCharsets.UTF_8)) {
+ OSW = osw;
File[] files = new
File(this.props.getProperty(MudrodConstants.ONTOLOGY_INPUT_PATH)).listFiles();
for (File fileIn : files) {
String ext = FilenameUtils.getExtension(fileIn.getAbsolutePath());
@@ -79,12 +80,7 @@ public class AggregateTriples extends DiscoveryStepAbstract {
} catch (IOException e) {
e.printStackTrace();
}
-
- try {
- bw.close();
- } catch (IOException e) {
- e.printStackTrace();
- }
+
return null;
}
@@ -94,7 +90,7 @@ public class AggregateTriples extends DiscoveryStepAbstract {
private final static String rdf_namespace =
"http://www.w3.org/1999/02/22-rdf-syntax-ns#";;
private final static String rdfs_namespace =
"http://www.w3.org/2000/01/rdf-schema#";;
- BufferedWriter bw = null;
+ OutputStreamWriter OSW = null;
/**
* Load OWL file into memory
@@ -178,10 +174,10 @@ public class AggregateTriples extends
DiscoveryStepAbstract {
Element allValuesFromEle = findChild("allValuesFrom",
subclassElement);
if (allValuesFromEle != null) {
subclassName = allValuesFromEle.getAttributeValue("resource",
Namespace.getNamespace("rdf", rdf_namespace));
- bw.write(cutString(className) + ",SubClassOf," +
cutString(subclassName) + "\n");
+ OSW.write(cutString(className) + ",SubClassOf," +
cutString(subclassName) + "\n");
}
} else {
- bw.write(cutString(className) + ",SubClassOf," +
cutString(subclassName) + "\n");
+ OSW.write(cutString(className) + ",SubClassOf," +
cutString(subclassName) + "\n");
}
}
@@ -192,7 +188,7 @@ public class AggregateTriples extends DiscoveryStepAbstract
{
String equalClassElementName =
equalClassElement.getAttributeValue("resource", Namespace.getNamespace("rdf",
rdf_namespace));
if (equalClassElementName != null) {
- bw.write(cutString(className) + ",equivalentClass," +
cutString(equalClassElementName) + "\n");
+ OSW.write(cutString(className) + ",equivalentClass," +
cutString(equalClassElementName) + "\n");
}
}
diff --git
a/core/src/main/java/org/apache/sdap/mudrod/ontology/process/LocalOntology.java
b/core/src/main/java/org/apache/sdap/mudrod/ontology/process/LocalOntology.java
index 8381413..5bb9994 100644
---
a/core/src/main/java/org/apache/sdap/mudrod/ontology/process/LocalOntology.java
+++
b/core/src/main/java/org/apache/sdap/mudrod/ontology/process/LocalOntology.java
@@ -36,6 +36,7 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
/**
@@ -277,7 +278,7 @@ public class LocalOntology implements Ontology {
m = new HashMap<>();
}
m.put(resource, "1");
- searchTerms.put(label.toLowerCase(), m);
+ searchTerms.put(label.toLowerCase(Locale.ENGLISH), m);
}
/**
@@ -293,7 +294,7 @@ public class LocalOntology implements Ontology {
*/
public Map<OntResource, String> retrieve(String label) {
@SuppressWarnings("unchecked")
- Map<OntResource, String> m = (Map<OntResource, String>)
searchTerms.get(label.toLowerCase());
+ Map<OntResource, String> m = (Map<OntResource, String>)
searchTerms.get(label.toLowerCase(Locale.ENGLISH));
if (m == null) {
m = new HashMap<>();
}
diff --git
a/core/src/main/java/org/apache/sdap/mudrod/ontology/process/OntologyLinkCal.java
b/core/src/main/java/org/apache/sdap/mudrod/ontology/process/OntologyLinkCal.java
index 1cf1376..59b9aaa 100644
---
a/core/src/main/java/org/apache/sdap/mudrod/ontology/process/OntologyLinkCal.java
+++
b/core/src/main/java/org/apache/sdap/mudrod/ontology/process/OntologyLinkCal.java
@@ -22,8 +22,11 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import java.io.BufferedReader;
-import java.io.FileReader;
+import java.io.FileInputStream;
import java.io.IOException;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
+import java.util.Locale;
import java.util.Properties;
import java.util.concurrent.ExecutionException;
@@ -71,9 +74,9 @@ public class OntologyLinkCal extends DiscoveryStepAbstract {
double weight = 0;
try {
- br = new BufferedReader(new
FileReader(props.getProperty(MudrodConstants.ONTOLOGY_PATH)));
+ br = new BufferedReader(new InputStreamReader(new
FileInputStream(props.getProperty(MudrodConstants.ONTOLOGY_PATH)),
StandardCharsets.UTF_8));
while ((line = br.readLine()) != null) {
- String[] strList = line.toLowerCase().split(",");
+ String[] strList = line.toLowerCase(Locale.ENGLISH).split(",");
if (strList[1].equals("subclassof")) {
weight = 0.75;
} else {
diff --git
a/core/src/main/java/org/apache/sdap/mudrod/recommendation/pre/SessionCooccurence.java
b/core/src/main/java/org/apache/sdap/mudrod/recommendation/pre/SessionCooccurence.java
index 61492ca..40b61ab 100644
---
a/core/src/main/java/org/apache/sdap/mudrod/recommendation/pre/SessionCooccurence.java
+++
b/core/src/main/java/org/apache/sdap/mudrod/recommendation/pre/SessionCooccurence.java
@@ -33,6 +33,7 @@ import scala.Tuple2;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
import java.util.Properties;
@@ -143,7 +144,7 @@ public class SessionCooccurence extends
DiscoveryStepAbstract {
for (SearchHit hit : scrollResp.getHits().getHits()) {
Map<String, Object> metadata = hit.getSource();
String shortName = (String)
metadata.get(props.getProperty(MudrodConstants.METADATA_ID));
- shortnameMap.put(shortName.toLowerCase(), shortName);
+ shortnameMap.put(shortName.toLowerCase(Locale.ENGLISH), shortName);
}
scrollResp =
es.getClient().prepareSearchScroll(scrollResp.getScrollId()).setScroll(new
TimeValue(600000)).execute().actionGet();
diff --git
a/core/src/main/java/org/apache/sdap/mudrod/recommendation/process/FeatureBasedSimilarity.java
b/core/src/main/java/org/apache/sdap/mudrod/recommendation/process/FeatureBasedSimilarity.java
index 4b2bfeb..3e03db5 100644
---
a/core/src/main/java/org/apache/sdap/mudrod/recommendation/process/FeatureBasedSimilarity.java
+++
b/core/src/main/java/org/apache/sdap/mudrod/recommendation/process/FeatureBasedSimilarity.java
@@ -33,6 +33,8 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.io.Serializable;
import java.text.DecimalFormat;
+import java.text.NumberFormat;
+import java.util.Locale;
import java.util.Properties;
import java.util.ArrayList;
import java.util.List;
@@ -45,7 +47,7 @@ public class FeatureBasedSimilarity extends
DiscoveryStepAbstract implements Ser
private static final Logger LOG =
LoggerFactory.getLogger(FeatureBasedSimilarity.class);
- private DecimalFormat df = new DecimalFormat("#.000");
+ private DecimalFormat df;
// a map from variable to its type
MetadataFeature metadata = null;
public Map<String, Integer> variableTypes;
@@ -77,6 +79,9 @@ public class FeatureBasedSimilarity extends
DiscoveryStepAbstract implements Ser
metadata.inital();
variableTypes = metadata.featureTypes;
variableWeights = metadata.featureWeights;
+ NumberFormat nf = NumberFormat.getNumberInstance(Locale.ENGLISH);
+ df = (DecimalFormat) nf;
+ df.applyPattern("#.000");
}
@Override
diff --git
a/core/src/main/java/org/apache/sdap/mudrod/recommendation/structure/HybridRecommendation.java
b/core/src/main/java/org/apache/sdap/mudrod/recommendation/structure/HybridRecommendation.java
index 6d915e5..6722e82 100644
---
a/core/src/main/java/org/apache/sdap/mudrod/recommendation/structure/HybridRecommendation.java
+++
b/core/src/main/java/org/apache/sdap/mudrod/recommendation/structure/HybridRecommendation.java
@@ -30,6 +30,7 @@ import org.elasticsearch.search.sort.SortOrder;
import java.io.IOException;
import java.text.DecimalFormat;
+import java.text.NumberFormat;
import java.util.*;
/**
@@ -44,7 +45,7 @@ public class HybridRecommendation extends
DiscoveryStepAbstract {
// recommended metadata list
private transient List<LinkedTerm> termList = new ArrayList<>();
// format decimal
- private DecimalFormat df = new DecimalFormat("#.00");
+ private DecimalFormat df;
// index name
private static final String INDEX_NAME = MudrodConstants.ES_INDEX_NAME;
private static final String WEIGHT = "weight";
@@ -66,6 +67,9 @@ public class HybridRecommendation extends
DiscoveryStepAbstract {
public HybridRecommendation(Properties props, ESDriver es, SparkDriver
spark) {
super(props, es, spark);
+ NumberFormat nf = NumberFormat.getNumberInstance(Locale.ENGLISH);
+ df = (DecimalFormat) nf;
+ df.applyPattern("#.00");
}
@Override
diff --git
a/core/src/main/java/org/apache/sdap/mudrod/ssearch/ClickStreamImporter.java
b/core/src/main/java/org/apache/sdap/mudrod/ssearch/ClickStreamImporter.java
index 5694236..a07a67b 100644
--- a/core/src/main/java/org/apache/sdap/mudrod/ssearch/ClickStreamImporter.java
+++ b/core/src/main/java/org/apache/sdap/mudrod/ssearch/ClickStreamImporter.java
@@ -23,8 +23,10 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.BufferedReader;
-import java.io.FileReader;
+import java.io.FileInputStream;
import java.io.IOException;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
import java.util.Properties;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
@@ -84,7 +86,7 @@ public class ClickStreamImporter extends MudrodAbstract {
String cvsSplitBy = ",";
try {
- br = new BufferedReader(new
FileReader(props.getProperty(MudrodConstants.CLICKSTREAM_PATH)));
+ br = new BufferedReader(new InputStreamReader(new
FileInputStream(props.getProperty(MudrodConstants.CLICKSTREAM_PATH)),
StandardCharsets.UTF_8));
String line = br.readLine();
// first item needs to be skipped
String[] dataList = line.split(cvsSplitBy);
diff --git a/core/src/main/java/org/apache/sdap/mudrod/ssearch/Dispatcher.java
b/core/src/main/java/org/apache/sdap/mudrod/ssearch/Dispatcher.java
index 611c76b..a0342f4 100644
--- a/core/src/main/java/org/apache/sdap/mudrod/ssearch/Dispatcher.java
+++ b/core/src/main/java/org/apache/sdap/mudrod/ssearch/Dispatcher.java
@@ -25,6 +25,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.HashMap;
+import java.util.Locale;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Properties;
@@ -101,7 +102,7 @@ public class Dispatcher extends MudrodAbstract {
"DatasetSource-Sensor-ShortName-Full" };
BoolQueryBuilder qb = new BoolQueryBuilder();
for (Entry<String, Double> entry : selected_Map.entrySet()) {
- if (query_operator.toLowerCase().trim().equals("phrase")) {
+ if (query_operator.toLowerCase(Locale.ENGLISH).trim().equals("phrase")) {
qb.should(QueryBuilders.multiMatchQuery(entry.getKey(),
fieldsList).boost(entry.getValue().floatValue()).type(MultiMatchQueryBuilder.Type.PHRASE).tieBreaker((float)
0.5)); // when
// set
// to
@@ -114,7 +115,7 @@ public class Dispatcher extends MudrodAbstract {
// "most
// fields"
// query
- } else if (query_operator.toLowerCase().trim().equals("and")) {
+ } else if
(query_operator.toLowerCase(Locale.ENGLISH).trim().equals("and")) {
qb.should(QueryBuilders.multiMatchQuery(entry.getKey(),
fieldsList).boost(entry.getValue().floatValue()).operator(MatchQueryBuilder.DEFAULT_OPERATOR.AND).tieBreaker((float)
0.5));
} else {
qb.should(QueryBuilders.multiMatchQuery(entry.getKey(),
fieldsList).boost(entry.getValue().floatValue()).operator(MatchQueryBuilder.DEFAULT_OPERATOR.OR).tieBreaker((float)
0.5));
diff --git a/core/src/main/java/org/apache/sdap/mudrod/ssearch/Ranker.java
b/core/src/main/java/org/apache/sdap/mudrod/ssearch/Ranker.java
index 9215859..d974870 100644
--- a/core/src/main/java/org/apache/sdap/mudrod/ssearch/Ranker.java
+++ b/core/src/main/java/org/apache/sdap/mudrod/ssearch/Ranker.java
@@ -24,10 +24,12 @@ import org.apache.spark.mllib.regression.LabeledPoint;
import java.io.Serializable;
import java.text.DecimalFormat;
+import java.text.NumberFormat;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
+import java.util.Locale;
import java.util.Properties;
/**
@@ -116,7 +118,9 @@ public class Ranker extends MudrodAbstract implements
Serializable {
* @return processed double value
*/
private double getNDForm(double d) {
- DecimalFormat ndForm = new DecimalFormat("#.###");
+ NumberFormat nf = NumberFormat.getNumberInstance(Locale.ENGLISH);
+ DecimalFormat ndForm = (DecimalFormat) nf;
+ ndForm.applyPattern("#.###");
return Double.valueOf(ndForm.format(d));
}
diff --git a/core/src/main/java/org/apache/sdap/mudrod/ssearch/Searcher.java
b/core/src/main/java/org/apache/sdap/mudrod/ssearch/Searcher.java
index adec951..9f80b58 100644
--- a/core/src/main/java/org/apache/sdap/mudrod/ssearch/Searcher.java
+++ b/core/src/main/java/org/apache/sdap/mudrod/ssearch/Searcher.java
@@ -16,7 +16,6 @@ package org.apache.sdap.mudrod.ssearch;
import com.google.gson.Gson;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
-
import org.apache.sdap.mudrod.discoveryengine.MudrodAbstract;
import org.apache.sdap.mudrod.driver.ESDriver;
import org.apache.sdap.mudrod.driver.SparkDriver;
@@ -31,10 +30,12 @@ import org.elasticsearch.search.sort.SortOrder;
import java.io.Serializable;
import java.text.DecimalFormat;
+import java.text.NumberFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
import java.util.Properties;
import java.util.regex.Pattern;
@@ -47,11 +48,15 @@ public class Searcher extends MudrodAbstract implements
Serializable {
*
*/
private static final long serialVersionUID = 1L;
- DecimalFormat ndForm = new DecimalFormat("#.##");
- static final Integer MAX_CHAR = 700;
+ private DecimalFormat ndForm;
+
+ private static final Integer MAX_CHAR = 700;
public Searcher(Properties props, ESDriver es, SparkDriver spark) {
super(props, es, spark);
+ NumberFormat nf = NumberFormat.getNumberInstance(Locale.ENGLISH);
+ ndForm = (DecimalFormat) nf;
+ ndForm.applyPattern("#.##");
}
/**
@@ -185,7 +190,7 @@ public class Searcher extends MudrodAbstract implements
Serializable {
ArrayList<String> longdate = (ArrayList<String>)
result.get("DatasetCitation-ReleaseDateLong");
Date date = new Date(Long.valueOf(longdate.get(0)));
- SimpleDateFormat df2 = new SimpleDateFormat("MM/dd/yyyy");
+ SimpleDateFormat df2 = new SimpleDateFormat("MM/dd/yyyy",
Locale.ENGLISH);
String dateText = df2.format(date);
// start date
diff --git
a/core/src/main/java/org/apache/sdap/mudrod/ssearch/ranking/DataGenerator.java
b/core/src/main/java/org/apache/sdap/mudrod/ssearch/ranking/DataGenerator.java
index 30abde5..f6153bc 100644
---
a/core/src/main/java/org/apache/sdap/mudrod/ssearch/ranking/DataGenerator.java
+++
b/core/src/main/java/org/apache/sdap/mudrod/ssearch/ranking/DataGenerator.java
@@ -18,11 +18,13 @@ import au.com.bytecode.opencsv.CSVWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-
import java.io.File;
-import java.io.FileReader;
-import java.io.FileWriter;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -98,7 +100,7 @@ public class DataGenerator {
if (directoryListing != null) {
for (File child : directoryListing) {
- try (CSVReader csvReader = new CSVReader(new FileReader(child))) {
+ try (CSVReader csvReader = new CSVReader(new InputStreamReader(new
FileInputStream(child), StandardCharsets.UTF_8))) {
List<String[]> list = csvReader.readAll();
// Store into 2D array by transforming array list to normal array
dataArr = new String[list.size()][];
@@ -113,7 +115,7 @@ public class DataGenerator {
File file = new File(sourceDir);
if (file != null) {
- try (CSVReader csvReader = new CSVReader(new FileReader(file))) {
+ try (CSVReader csvReader = new CSVReader(new InputStreamReader(new
FileInputStream(file), StandardCharsets.UTF_8))) {
List<String[]> list = csvReader.readAll();
// Store into 2D array by transforming array list to normal array
@@ -289,9 +291,8 @@ public class DataGenerator {
String outputFile = myResultDir;
boolean alreadyExists = new File(outputFile).exists();
- try {
- CSVWriter csvOutput = new CSVWriter(new FileWriter(outputFile), ','); //
Create new instance of CSVWriter to write to file output
-
+ // Create new instance of CSVWriter to write to file output
+ try(CSVWriter csvOutput = new CSVWriter (new OutputStreamWriter(new
FileOutputStream(outputFile), StandardCharsets.UTF_8), ',')) {
if (!alreadyExists) {
csvOutput.writeNext(myHeader); // Write the text headers first before
data
@@ -302,7 +303,6 @@ public class DataGenerator {
}
}
- csvOutput.close(); // Close csvWriter
} catch (IOException e) {
LOG.error("Writing to CSV file failed : ", e);
}
diff --git
a/core/src/main/java/org/apache/sdap/mudrod/ssearch/ranking/SparkFormatter.java
b/core/src/main/java/org/apache/sdap/mudrod/ssearch/ranking/SparkFormatter.java
index 0db16f4..5cbef59 100644
---
a/core/src/main/java/org/apache/sdap/mudrod/ssearch/ranking/SparkFormatter.java
+++
b/core/src/main/java/org/apache/sdap/mudrod/ssearch/ranking/SparkFormatter.java
@@ -17,19 +17,25 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.BufferedReader;
-import java.io.BufferedWriter;
import java.io.File;
-import java.io.FileReader;
-import java.io.FileWriter;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.nio.charset.StandardCharsets;
import java.text.DecimalFormat;
+import java.text.NumberFormat;
+import java.util.Locale;
public class SparkFormatter {
-
- private DecimalFormat NDForm = new DecimalFormat("#.###");
- private static final Logger LOG =
LoggerFactory.getLogger(SparkFormatter.class);
+ private DecimalFormat ndForm;
+ private static final Logger LOG =
LoggerFactory.getLogger(SparkFormatter.class);
public SparkFormatter() {
+ NumberFormat nf = NumberFormat.getNumberInstance(Locale.ENGLISH);
+ ndForm = (DecimalFormat) nf;
+ ndForm.applyPattern("#.##");
}
public void toSparkSVMformat(String inputCSVFileName, String
outputTXTFileName) {
@@ -40,9 +46,8 @@ public class SparkFormatter {
try {
file.createNewFile();
- try (FileWriter fw = new FileWriter(outputTXTFileName);
- BufferedWriter bw = new BufferedWriter(fw);
- BufferedReader br = new BufferedReader(new
FileReader(inputCSVFileName));) {
+ try (OutputStreamWriter osw = new OutputStreamWriter(new
FileOutputStream(outputTXTFileName), StandardCharsets.UTF_8);
+ BufferedReader br = new BufferedReader(new InputStreamReader(new
FileInputStream(inputCSVFileName), StandardCharsets.UTF_8))) {
String line = null;
line = br.readLine(); //header
@@ -58,9 +63,9 @@ public class SparkFormatter {
for (int i = 0; i < list.length - 1; i++) {
int index = i + 1;
- output += index + ":" +
NDForm.format(Double.parseDouble(list[i].replace("\"", ""))) + " ";
+ output += index + ":" +
ndForm.format(Double.parseDouble(list[i].replace("\"", ""))) + " ";
}
- bw.write(output + "\n");
+ osw.write(output + "\n");
}
}
} catch (IOException e) {
diff --git
a/core/src/main/java/org/apache/sdap/mudrod/ssearch/ranking/TrainingImporter.java
b/core/src/main/java/org/apache/sdap/mudrod/ssearch/ranking/TrainingImporter.java
index ebe4628..f98ed0a 100644
---
a/core/src/main/java/org/apache/sdap/mudrod/ssearch/ranking/TrainingImporter.java
+++
b/core/src/main/java/org/apache/sdap/mudrod/ssearch/ranking/TrainingImporter.java
@@ -23,8 +23,10 @@ import org.elasticsearch.index.query.QueryBuilders;
import java.io.BufferedReader;
import java.io.File;
-import java.io.FileReader;
+import java.io.FileInputStream;
import java.io.IOException;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
import java.util.Properties;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
@@ -71,7 +73,7 @@ public class TrainingImporter extends MudrodAbstract {
File[] files = new File(dataFolder).listFiles();
for (File file : files) {
- try (BufferedReader br = new BufferedReader(new
FileReader(file.getAbsolutePath()))) {
+ try (BufferedReader br = new BufferedReader(new InputStreamReader(new
FileInputStream(file.getAbsolutePath()), StandardCharsets.UTF_8))) {
br.readLine();
String line = br.readLine();
while (line != null) {
diff --git a/core/src/main/java/org/apache/sdap/mudrod/tools/InstallCert.java
b/core/src/main/java/org/apache/sdap/mudrod/tools/InstallCert.java
index 370697c..f81dc9f 100644
--- a/core/src/main/java/org/apache/sdap/mudrod/tools/InstallCert.java
+++ b/core/src/main/java/org/apache/sdap/mudrod/tools/InstallCert.java
@@ -38,6 +38,7 @@ import java.io.FileOutputStream;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
import java.security.KeyStore;
import java.security.MessageDigest;
import java.security.cert.CertificateException;
@@ -116,8 +117,7 @@ public class InstallCert {
return;
}
- BufferedReader reader =
- new BufferedReader(new InputStreamReader(System.in));
+ BufferedReader reader = new BufferedReader(new
InputStreamReader(System.in, StandardCharsets.UTF_8));
System.out.println();
System.out.println("Server sent " + chain.length + " certificate(s):");
diff --git a/core/src/main/java/org/apache/sdap/mudrod/utils/HttpRequest.java
b/core/src/main/java/org/apache/sdap/mudrod/utils/HttpRequest.java
index ed596f6..557e4ee 100644
--- a/core/src/main/java/org/apache/sdap/mudrod/utils/HttpRequest.java
+++ b/core/src/main/java/org/apache/sdap/mudrod/utils/HttpRequest.java
@@ -21,6 +21,7 @@ import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
+import java.nio.charset.StandardCharsets;
/**
* ClassName: HttpRequest
@@ -49,7 +50,7 @@ public class HttpRequest {
LOG.info(line);
} else {
InputStream content = connection.getInputStream();
- BufferedReader in = new BufferedReader(new InputStreamReader(content));
+ BufferedReader in = new BufferedReader(new InputStreamReader(content,
StandardCharsets.UTF_8));
line = in.readLine();
}
} catch (Exception e) {
diff --git a/core/src/main/java/org/apache/sdap/mudrod/utils/LinkageTriple.java
b/core/src/main/java/org/apache/sdap/mudrod/utils/LinkageTriple.java
index 9f50c98..1714251 100644
--- a/core/src/main/java/org/apache/sdap/mudrod/utils/LinkageTriple.java
+++ b/core/src/main/java/org/apache/sdap/mudrod/utils/LinkageTriple.java
@@ -31,7 +31,9 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.io.Serializable;
import java.text.DecimalFormat;
+import java.text.NumberFormat;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
@@ -56,11 +58,14 @@ public class LinkageTriple implements Serializable {
// keyB: TermB
public String keyB;
// df: Format number
- private static DecimalFormat df = new DecimalFormat("#.00");
+ private static DecimalFormat df;
private static final Logger LOG =
LoggerFactory.getLogger(LinkageTriple.class);
public LinkageTriple() {
+ NumberFormat nf = NumberFormat.getNumberInstance(Locale.ENGLISH);
+ df = (DecimalFormat) nf;
+ df.applyPattern("#.00");
}
/**
diff --git a/core/src/main/java/org/apache/sdap/mudrod/utils/MatrixUtil.java
b/core/src/main/java/org/apache/sdap/mudrod/utils/MatrixUtil.java
index e058abe..b89e31b 100644
--- a/core/src/main/java/org/apache/sdap/mudrod/utils/MatrixUtil.java
+++ b/core/src/main/java/org/apache/sdap/mudrod/utils/MatrixUtil.java
@@ -37,10 +37,11 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import scala.Tuple2;
-import java.io.BufferedWriter;
import java.io.File;
-import java.io.FileWriter;
+import java.io.FileOutputStream;
import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Iterator;
@@ -472,14 +473,13 @@ public class MatrixUtil {
} catch (IOException e1) {
LOG.error("Failed to create file : ", e1);
}
- try (FileWriter fw = new FileWriter(file.getAbsoluteFile());
- BufferedWriter bw = new BufferedWriter(fw);){
+ try (OutputStreamWriter osw = new OutputStreamWriter(new
FileOutputStream(file.getAbsoluteFile()), StandardCharsets.UTF_8)){
String coltitle = " Num" + ",";
for (int j = 0; j < colnum; j++) {
coltitle += "\"" + colKeys.get(j) + "\",";
}
coltitle = coltitle.substring(0, coltitle.length() - 1);
- bw.write(coltitle + "\n");
+ osw.write(coltitle + "\n");
for (int i = 0; i < rownum; i++) {
double[] rowvlaue = rows.get(i).toArray();
@@ -488,7 +488,7 @@ public class MatrixUtil {
row += rowvlaue[j] + ",";
}
row = row.substring(0, row.length() - 1);
- bw.write(row + "\n");
+ osw.write(row + "\n");
}
} catch (IOException e) {
LOG.error("Failed to write to the file : ", e);
diff --git
a/core/src/main/java/org/apache/sdap/mudrod/weblog/pre/CrawlerDetection.java
b/core/src/main/java/org/apache/sdap/mudrod/weblog/pre/CrawlerDetection.java
index 38e7d29..1c4c289 100644
--- a/core/src/main/java/org/apache/sdap/mudrod/weblog/pre/CrawlerDetection.java
+++ b/core/src/main/java/org/apache/sdap/mudrod/weblog/pre/CrawlerDetection.java
@@ -42,6 +42,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
import java.util.Properties;
import java.util.regex.Matcher;
@@ -101,7 +102,7 @@ public class CrawlerDetection extends LogAbstract {
public boolean checkKnownCrawler(String agent) {
String[] crawlers =
props.getProperty(MudrodConstants.BLACK_LIST_AGENT).split(",");
for (int i = 0; i < crawlers.length; i++) {
- if (agent.toLowerCase().contains(crawlers[i].trim()))
+ if (agent.toLowerCase(Locale.ENGLISH).contains(crawlers[i].trim()))
return true;
}
return false;
@@ -169,7 +170,7 @@ public class CrawlerDetection extends LogAbstract {
String logtype = (String) result.get("LogType");
if (logtype.equals(MudrodConstants.HTTP_LOG)) {
String request = (String) result.get("Request");
- matcher = pattern.matcher(request.trim().toLowerCase());
+ matcher =
pattern.matcher(request.trim().toLowerCase(Locale.ENGLISH));
boolean find = false;
while (matcher.find()) {
request = matcher.group(1);
diff --git
a/core/src/main/java/org/apache/sdap/mudrod/weblog/pre/HistoryGenerator.java
b/core/src/main/java/org/apache/sdap/mudrod/weblog/pre/HistoryGenerator.java
index 08c8c6c..9c29248 100644
--- a/core/src/main/java/org/apache/sdap/mudrod/weblog/pre/HistoryGenerator.java
+++ b/core/src/main/java/org/apache/sdap/mudrod/weblog/pre/HistoryGenerator.java
@@ -24,11 +24,11 @@ import
org.elasticsearch.search.aggregations.bucket.terms.Terms;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.BufferedWriter;
import java.io.File;
-import java.io.FileWriter;
+import java.io.FileOutputStream;
import java.io.IOException;
-
+import java.io.OutputStreamWriter;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -72,9 +72,8 @@ public class HistoryGenerator extends LogAbstract {
file.createNewFile();
- FileWriter fw = new FileWriter(file.getAbsoluteFile());
- try (BufferedWriter bw = new BufferedWriter(fw)) {
- bw.write("Num" + ",");
+ try (OutputStreamWriter osw = new OutputStreamWriter(new
FileOutputStream(file.getAbsoluteFile()), StandardCharsets.UTF_8)) {
+ osw.write("Num" + ",");
// step 1: write first row of csv
List<String> logIndexList = es.getIndexListWithPrefix(
@@ -87,7 +86,7 @@ public class HistoryGenerator extends LogAbstract {
LOG.info("{}: {}", this.sessionStats, docCount);
if (docCount == 0) {
- bw.close();
+ osw.close();
file.delete();
return;
}
@@ -111,7 +110,7 @@ public class HistoryGenerator extends LogAbstract {
ipList.add(entry.getKey().toString());
}
}
- bw.write(String.join(",", ipList) + "\n");
+ osw.write(String.join(",", ipList) + "\n");
// step 2: step the rest rows of csv
SearchRequestBuilder sr2Builder = es.getClient()
@@ -136,23 +135,22 @@ public class HistoryGenerator extends LogAbstract {
int distinctUser = ipAgg.getBuckets().size();
if (distinctUser >=
Integer.parseInt(props.getProperty(MudrodConstants.QUERY_MIN))) {
- bw.write(keyword.getKey() + ",");
+ osw.write(keyword.getKey() + ",");
for (Terms.Bucket IP : ipAgg.getBuckets()) {
ipMap.put(IP.getKey().toString(), 1);
}
for (String anIpList : ipList) {
if (ipMap.containsKey(anIpList)) {
- bw.write(ipMap.get(anIpList) + ",");
+ osw.write(ipMap.get(anIpList) + ",");
} else {
- bw.write("0,");
+ osw.write("0,");
}
}
- bw.write("\n");
+ osw.write("\n");
}
}
}
- fw.close();
} catch (IOException e) {
LOG.error("Failed to generate Binary Matrix : ", e);
}
diff --git
a/core/src/main/java/org/apache/sdap/mudrod/weblog/pre/SessionStatistic.java
b/core/src/main/java/org/apache/sdap/mudrod/weblog/pre/SessionStatistic.java
index 2846979..7b14953 100644
--- a/core/src/main/java/org/apache/sdap/mudrod/weblog/pre/SessionStatistic.java
+++ b/core/src/main/java/org/apache/sdap/mudrod/weblog/pre/SessionStatistic.java
@@ -41,6 +41,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.ExecutionException;
@@ -205,7 +206,7 @@ public class SessionStatistic extends LogAbstract {
String request = (String) result.get("Request");
String logType = (String) result.get("LogType");
iP = (String) result.get("IP");
- Matcher matcher = pattern.matcher(request.trim().toLowerCase());
+ Matcher matcher =
pattern.matcher(request.trim().toLowerCase(Locale.ENGLISH));
while (matcher.find()) {
request = matcher.group(1);
}
@@ -247,7 +248,7 @@ public class SessionStatistic extends LogAbstract {
if (MudrodConstants.FTP_LOG.equals(logType)) {
ftpRequestCount++;
String download = "";
- String requestLowercase = request.toLowerCase();
+ String requestLowercase = request.toLowerCase(Locale.ENGLISH);
if (!requestLowercase.endsWith(".jpg") &&
!requestLowercase.endsWith(".pdf") &&
!requestLowercase.endsWith(".txt") &&
diff --git
a/core/src/main/java/org/apache/sdap/mudrod/weblog/structure/log/ApacheAccessLog.java
b/core/src/main/java/org/apache/sdap/mudrod/weblog/structure/log/ApacheAccessLog.java
index 8887dee..6dcfdab 100644
---
a/core/src/main/java/org/apache/sdap/mudrod/weblog/structure/log/ApacheAccessLog.java
+++
b/core/src/main/java/org/apache/sdap/mudrod/weblog/structure/log/ApacheAccessLog.java
@@ -22,6 +22,7 @@ import java.io.Serializable;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
+import java.util.Locale;
import java.util.Properties;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -72,7 +73,7 @@ public class ApacheAccessLog extends WebLog implements
Serializable {
String time = matcher.group(4);
time = SwithtoNum(time);
- SimpleDateFormat formatter = new SimpleDateFormat("dd/MM/yyyy:HH:mm:ss");
+ SimpleDateFormat formatter = new SimpleDateFormat("dd/MM/yyyy:HH:mm:ss",
Locale.ENGLISH);
Date date = formatter.parse(time);
String bytes = matcher.group(7);
@@ -81,7 +82,7 @@ public class ApacheAccessLog extends WebLog implements
Serializable {
bytes = "0";
}
- String request = matcher.group(5).toLowerCase();
+ String request = matcher.group(5).toLowerCase(Locale.ENGLISH);
String agent = matcher.group(9);
CrawlerDetection crawlerDe = new CrawlerDetection(props);
if (crawlerDe.checkKnownCrawler(agent)) {
@@ -103,7 +104,7 @@ public class ApacheAccessLog extends WebLog implements
Serializable {
accesslog.Bytes = Double.parseDouble(bytes);
accesslog.Referer = matcher.group(8);
accesslog.Browser = matcher.group(9);
- SimpleDateFormat df = new
SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.sss'Z'");
+ SimpleDateFormat df = new
SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.sss'Z'", Locale.ENGLISH);
accesslog.Time = df.format(date);
Gson gson = new Gson();
diff --git
a/core/src/main/java/org/apache/sdap/mudrod/weblog/structure/log/FtpLog.java
b/core/src/main/java/org/apache/sdap/mudrod/weblog/structure/log/FtpLog.java
index 91fa228..0a78675 100644
--- a/core/src/main/java/org/apache/sdap/mudrod/weblog/structure/log/FtpLog.java
+++ b/core/src/main/java/org/apache/sdap/mudrod/weblog/structure/log/FtpLog.java
@@ -23,6 +23,7 @@ import org.slf4j.LoggerFactory;
import java.io.Serializable;
import java.text.SimpleDateFormat;
import java.util.Date;
+import java.util.Locale;
import java.util.Properties;
/**
@@ -44,12 +45,12 @@ public class FtpLog extends WebLog implements Serializable {
String time = log.split(" +")[1] + ":" + log.split(" +")[2] + ":" +
log.split(" +")[3] + ":" + log.split(" +")[4];
time = SwithtoNum(time);
- SimpleDateFormat formatter = new SimpleDateFormat("MM:dd:HH:mm:ss:yyyy");
+ SimpleDateFormat formatter = new SimpleDateFormat("MM:dd:HH:mm:ss:yyyy",
Locale.ENGLISH);
Date date = formatter.parse(time);
String bytes = log.split(" +")[7];
- String request = log.split(" +")[8].toLowerCase();
+ String request = log.split(" +")[8].toLowerCase(Locale.ENGLISH);
if (!request.contains("/misc/") && !request.contains("readme")) {
FtpLog ftplog = new FtpLog();
@@ -58,7 +59,7 @@ public class FtpLog extends WebLog implements Serializable {
ftplog.Request = request;
ftplog.Bytes = Double.parseDouble(bytes);
- SimpleDateFormat df = new
SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.sss'Z'");
+ SimpleDateFormat df = new
SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.sss'Z'", Locale.ENGLISH);
ftplog.Time = df.format(date);
return new Gson().toJson(ftplog);
diff --git
a/core/src/main/java/org/apache/sdap/mudrod/weblog/structure/log/RequestUrl.java
b/core/src/main/java/org/apache/sdap/mudrod/weblog/structure/log/RequestUrl.java
index fe76e76..90bc1f6 100644
---
a/core/src/main/java/org/apache/sdap/mudrod/weblog/structure/log/RequestUrl.java
+++
b/core/src/main/java/org/apache/sdap/mudrod/weblog/structure/log/RequestUrl.java
@@ -21,6 +21,7 @@ import java.net.URLDecoder;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
/**
@@ -47,7 +48,7 @@ public class RequestUrl {
String strPage = null;
String[] arrSplit = null;
- String newURL = strURL.trim().toLowerCase();
+ String newURL = strURL.trim().toLowerCase(Locale.ENGLISH);
arrSplit = newURL.split("[?]");
if (newURL.length() > 0 && arrSplit.length > 1 && arrSplit[0] != null) {
@@ -67,7 +68,7 @@ public class RequestUrl {
String strAllParam = null;
String[] arrSplit = null;
- strURL = strURL.trim().toLowerCase(); // keep this in mind
+ strURL = strURL.trim().toLowerCase(Locale.ENGLISH); // keep this in mind
arrSplit = strURL.split("[?]");
if (strURL.length() > 1) {
diff --git
a/core/src/main/java/org/apache/sdap/mudrod/weblog/structure/session/SessionNode.java
b/core/src/main/java/org/apache/sdap/mudrod/weblog/structure/session/SessionNode.java
index b893d6b..24b4ad5 100644
---
a/core/src/main/java/org/apache/sdap/mudrod/weblog/structure/session/SessionNode.java
+++
b/core/src/main/java/org/apache/sdap/mudrod/weblog/structure/session/SessionNode.java
@@ -15,10 +15,12 @@ package org.apache.sdap.mudrod.weblog.structure.session;
import java.io.UnsupportedEncodingException;
import java.net.URLDecoder;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
import java.util.Properties;
import java.util.regex.Matcher;
@@ -28,7 +30,7 @@ import org.apache.sdap.mudrod.main.MudrodConstants;
/**
* ClassName: SessionNode Function: Functions related to a node in a session
- * tree sturcture.
+ * tree structure.
*/
public class SessionNode {
// id: Node ID
@@ -91,7 +93,7 @@ public class SessionNode {
this.referer = "";
return;
}
- this.referer= referer.toLowerCase().replace(basicUrl, "");
+ this.referer= referer.toLowerCase(Locale.ENGLISH).replace(basicUrl, "");
}
/**
@@ -290,7 +292,7 @@ public class SessionNode {
*/
private void parseRequest(String request) {
Pattern pattern = Pattern.compile("get (.*?) http/*");
- Matcher matcher = pattern.matcher(request.trim().toLowerCase());
+ Matcher matcher =
pattern.matcher(request.trim().toLowerCase(Locale.ENGLISH));
while (matcher.find()) {
request = matcher.group(1);
}
@@ -298,7 +300,7 @@ public class SessionNode {
this.parseDatasetId(request);
}
- this.request = request.toLowerCase();
+ this.request = request.toLowerCase(Locale.ENGLISH);
}
/**
@@ -306,16 +308,16 @@ public class SessionNode {
*
* @param params filter key value pairs of this node
*/
- private void parseFilterParams(Map<String, String> params) {
- this.filter = new HashMap<String, String>();
+ private void parseFilterParams(Map<String, String> params) throws
UnsupportedEncodingException {
+ this.filter = new HashMap<>();
if (params.containsKey("ids")) {
String idsStr = params.get("ids");
if (!idsStr.equals("")) {
- idsStr = URLDecoder.decode(idsStr);
+ idsStr = URLDecoder.decode(idsStr,
StandardCharsets.UTF_8.displayName(Locale.ENGLISH));
String[] ids = idsStr.split(":");
String valueStr = params.get("values");
if (valueStr != null) {
- valueStr = URLDecoder.decode(valueStr);
+ valueStr = URLDecoder.decode(valueStr,
StandardCharsets.UTF_8.displayName(Locale.ENGLISH));
String[] values = valueStr.split(":");
int size = ids.length;
for (int i = 0; i < size; i++) {
