This is an automated email from the ASF dual-hosted git repository.
lewismc pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-mudrod.git
The following commit(s) were added to refs/heads/master by this push:
new 67ae45a Added test for log pattern. Currently fails (#32)
67ae45a is described below
commit 67ae45a8964c1f7cacc3a531dfc7d26566ba48db
Author: fgreg <[email protected]>
AuthorDate: Wed Aug 22 10:47:33 2018 -0700
Added test for log pattern. Currently fails (#32)
---
.../weblog/structure/TestApacheAccessLog.java | 55 ++++++++++++++++
core/src/test/resources/config.properties | 74 ++++++++++++++++++++++
2 files changed, 129 insertions(+)
diff --git
a/core/src/test/java/org/apache/sdap/mudrod/weblog/structure/TestApacheAccessLog.java
b/core/src/test/java/org/apache/sdap/mudrod/weblog/structure/TestApacheAccessLog.java
new file mode 100644
index 0000000..e1453a5
--- /dev/null
+++
b/core/src/test/java/org/apache/sdap/mudrod/weblog/structure/TestApacheAccessLog.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sdap.mudrod.weblog.structure;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URL;
+import java.text.ParseException;
+import java.util.Properties;
+
+import static org.junit.Assert.assertNotEquals;
+
+
+public class TestApacheAccessLog {
+
+ private static Properties testProperties = new Properties();
+
+ @BeforeClass
+ public static void loadProperties() throws IOException {
+
+ URL configURL =
ClassLoader.getSystemClassLoader().getResource("config.properties");
+
+ assert configURL != null : "Could not load config.properties";
+ try (InputStream instream = new FileInputStream(configURL.getFile())) {
+ testProperties.load(instream);
+ }
+ }
+
+ @Test
+ public void testLogMatch() throws IOException, ParseException {
+
+
+ String testLogLine = "198.118.243.84 - - [31/Dec/2017:23:59:20 +0000]
\"GET
/events?page=12&%25252525252525252525252525252525252525253Bsort=asc&order=field_location&sort=desc
HTTP/1.1\" 200 86173";
+
+ String result = ApacheAccessLog.parseFromLogLine(testLogLine,
testProperties);
+
+ assertNotEquals("Log line does not match", "{}", result);
+ }
+}
diff --git a/core/src/test/resources/config.properties
b/core/src/test/resources/config.properties
new file mode 100644
index 0000000..4c8991e
--- /dev/null
+++ b/core/src/test/resources/config.properties
@@ -0,0 +1,74 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Database configuration
+mudrod.cluster.name=MudrodES
+mudrod.es.transport.tcp.port = 9300
+mudrod.es.unicast.hosts = 127.0.0.1
+mudrod.es.http.port = 9200
+mudrod.es.index = mudrod
+
+# Spark related
+# Log processing type. Possible values include 'sequential' or 'parallel'
+mudrod.processing.type = parallel
+mudrod.spark.app.name = MudrodSparkApp
+mudrod.spark.master = local[4]
+mudrod.spark.optimize = repartition
+
+# Web log processing configuration
+# index name has to be all lowercase
+mudrod.log.index = log
+mudrod.ftp.prefix = FTP.
+mudrod.http.prefix = WWW.
+mudrod.base.url = http://podaac.jpl.nasa.gov
+mudrod.black.request.list = .js, .css, .jpg, .png, .ico, image_captcha,
autocomplete, .gif, /alldata/, /api/, get / http/1.1, .jpeg, /ws/
+mudrod.black.agent.list = crawler, googlebot, bingbot, slurp, yacybot,
rogerbot, yandexbot, -, apache-httpclient, java, curl
+mudrod.search.freq = 100
+mudrod.view.freq = 200
+mudrod.download.freq = 100
+mudrod.request.rate = 30
+mudrod.session.port = 8080
+mudrod.session.url = /mudrod-service/session.html
+mudrod.request.time.gap = 600
+mudrod.view.url.marker = /dataset/
+mudrod.search.url.marker = /datasetlist?
+# In order to better parse a URL (getting searching keyword, etc.), please
consider custimize
+# org.apache.sdap.mudrod.weblog.structure.RequestUrl - GetSearchInfo,
getFilterInfo
+
+# User search history
+mudrod.query.min = 0
+mudrod.user.history.weight = 2
+
+# clickstream
+mudrod.download.weight = 3
+mudrod.clickstream.svd.d = 50
+mudrod.clickstream.weight = 2
+
+# metadata
+mudrod.metadata.download = 0
+mudrod.metadata.download.url =
https://podaac.jpl.nasa.gov/api/dataset?startIndex=$startIndex&entries=10&sortField=Dataset-AllTimePopularity&sortOrder=asc&id=&value=&search=
+mudrod.metadata.svd.d = 50
+mudrod.metadata.url = null
+mudrod.metadata.weight = 1
+mudrod.metadata.type = RawMetadata
+
+# ranking, ${svmSgdModel.value} is resolved at build time. See the property in
core/pom.xml for the value
+mudrod.ranking.machine.learning = 1
+mudrod.ranking.model = ${svmSgdModel.value}.zip
+
+# recommendation
+mudrod.metadata.id = Dataset-ShortName
+mudrod.metadata.semantic.fields =
DatasetParameter-Term,DatasetParameter-Variable,Dataset-ExtractTerm
+
+# ontology service implementation. Possible values include EsipPortal -
EsipPortalOntology EsipCOR - EsipCOROntology Local -
org.apache.sdap.mudrod.ontology.process.Local
+mudrod.ontology.implementation = Local
+mudrod.ontology.weight = 2
