This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 2f04803e2c860922a5f33469403f667bce004cc3
Author: Zhong <nju_y...@apache.org>
AuthorDate: Thu Oct 18 21:48:39 2018 +0800
KYLIN-2898 add unit test
---
.../RemoteLocalFailOverCacheManagerTest.java | 62 ++++++++
.../kylin/cache/memcached/MemcachedCacheTest.java | 84 +++++++++++
.../memcached/MemcachedChunkingCacheTest.java | 159 +++++++++++++++++++++
cache/src/test/resources/cacheContext.xml | 47 ++++++
cache/src/test/resources/ehcache-test.xml | 21 +++
5 files changed, 373 insertions(+)
diff --git
a/cache/src/test/java/org/apache/kylin/cache/cachemanager/RemoteLocalFailOverCacheManagerTest.java
b/cache/src/test/java/org/apache/kylin/cache/cachemanager/RemoteLocalFailOverCacheManagerTest.java
new file mode 100644
index 0000000..243e386
--- /dev/null
+++
b/cache/src/test/java/org/apache/kylin/cache/cachemanager/RemoteLocalFailOverCacheManagerTest.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.cache.cachemanager;
+
+import static org.apache.kylin.cache.cachemanager.CacheConstants.QUERY_CACHE;
+
+import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.cache.ehcache.EhCacheCache;
+import org.springframework.test.context.ActiveProfiles;
+import org.springframework.test.context.ContextConfiguration;
+import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
+
+@RunWith(SpringJUnit4ClassRunner.class)
+@ContextConfiguration(locations = { "classpath:cacheContext.xml" })
+@ActiveProfiles("testing-memcached")
+public class RemoteLocalFailOverCacheManagerTest {
+
+ @Autowired
+ @Qualifier("cacheManager")
+ RemoteLocalFailOverCacheManager cacheManager;
+
+ @BeforeClass
+ public static void setupResource() throws Exception {
+ LocalFileMetadataTestCase.staticCreateTestMetadata();
+ }
+
+ @AfterClass
+ public static void tearDownResource() {
+ }
+
+ @Test
+ public void testCacheManager() {
+ cacheManager.disableRemoteCacheManager();
+ Assert.assertTrue("Memcached failover to ehcache",
cacheManager.getCache(QUERY_CACHE) instanceof EhCacheCache);
+ cacheManager.enableRemoteCacheManager();
+ Assert.assertTrue("Memcached enabled",
+ cacheManager.getCache(QUERY_CACHE) instanceof
MemcachedCacheManager.MemCachedCacheAdaptor);
+ }
+}
\ No newline at end of file
diff --git
a/cache/src/test/java/org/apache/kylin/cache/memcached/MemcachedCacheTest.java
b/cache/src/test/java/org/apache/kylin/cache/memcached/MemcachedCacheTest.java
new file mode 100644
index 0000000..40571a7
--- /dev/null
+++
b/cache/src/test/java/org/apache/kylin/cache/memcached/MemcachedCacheTest.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.cache.memcached;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.kylin.cache.cachemanager.CacheConstants;
+import
org.apache.kylin.cache.cachemanager.MemcachedCacheManager.MemCachedCacheAdaptor;
+import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.google.common.collect.Maps;
+
+import net.spy.memcached.MemcachedClient;
+import net.spy.memcached.internal.GetFuture;
+
+public class MemcachedCacheTest extends LocalFileMetadataTestCase {
+
+ private Map<String, String> keyValueMap;
+ private MemCachedCacheAdaptor memCachedAdaptor;
+
+ @Before
+ public void setUp() throws Exception {
+ this.createTestMetadata();
+
+ keyValueMap = Maps.newHashMap();
+ keyValueMap.put("sql1", "value1");
+ keyValueMap.put("sql11", "value11");
+
+ MemcachedCacheConfig cacheConfig = new MemcachedCacheConfig();
+ MemcachedClient memcachedClient = mock(MemcachedClient.class);
+ MemcachedCache memcachedCache = new MemcachedCache(memcachedClient,
cacheConfig, CacheConstants.QUERY_CACHE,
+ 7 * 24 * 3600);
+ memCachedAdaptor = new MemCachedCacheAdaptor(memcachedCache);
+
+ //Mock put to cache
+ for (String key : keyValueMap.keySet()) {
+ String keyS = memcachedCache.serializeKey(key);
+ String hashedKey = memcachedCache.computeKeyHash(keyS);
+
+ String value = keyValueMap.get(key);
+ byte[] valueE = memcachedCache.encodeValue(keyS, value);
+
+ GetFuture<Object> future = mock(GetFuture.class);
+ when(future.get(cacheConfig.getTimeout(),
TimeUnit.MILLISECONDS)).thenReturn(valueE);
+ when(memcachedClient.asyncGet(hashedKey)).thenReturn(future);
+ }
+ }
+
+ @After
+ public void after() throws Exception {
+ this.cleanupTestMetadata();
+ }
+
+ @Test
+ public void testGet() {
+ for (String key : keyValueMap.keySet()) {
+ Assert.assertEquals("The value should not change",
keyValueMap.get(key), memCachedAdaptor.get(key).get());
+ }
+ }
+}
\ No newline at end of file
diff --git
a/cache/src/test/java/org/apache/kylin/cache/memcached/MemcachedChunkingCacheTest.java
b/cache/src/test/java/org/apache/kylin/cache/memcached/MemcachedChunkingCacheTest.java
new file mode 100644
index 0000000..295b20c
--- /dev/null
+++
b/cache/src/test/java/org/apache/kylin/cache/memcached/MemcachedChunkingCacheTest.java
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.cache.memcached;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.util.Arrays;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.kylin.cache.cachemanager.CacheConstants;
+import
org.apache.kylin.cache.cachemanager.MemcachedCacheManager.MemCachedCacheAdaptor;
+import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.apache.kylin.common.util.Pair;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.google.common.base.Charsets;
+import com.google.common.base.Strings;
+import com.google.common.collect.Maps;
+
+import net.spy.memcached.MemcachedClient;
+import net.spy.memcached.internal.BulkFuture;
+import net.spy.memcached.internal.GetFuture;
+
+public class MemcachedChunkingCacheTest extends LocalFileMetadataTestCase {
+
+ private Map<String, String> smallValueMap;
+ private Map<String, String> largeValueMap;
+ private MemCachedCacheAdaptor memCachedAdaptor;
+
+ @Before
+ public void setUp() throws Exception {
+ this.createTestMetadata();
+ final int maxObjectSize = 300;
+
+ smallValueMap = Maps.newHashMap();
+ smallValueMap.put("sql1", "value1");
+
+ largeValueMap = Maps.newHashMap();
+ largeValueMap.put("sql2", Strings.repeat("value2", maxObjectSize));
+
+ MemcachedCacheConfig cacheConfig = new MemcachedCacheConfig();
+ cacheConfig.setMaxObjectSize(maxObjectSize);
+ MemcachedClient memcachedClient = mock(MemcachedClient.class);
+ MemcachedCache memcachedCache = new MemcachedCache(memcachedClient,
cacheConfig, CacheConstants.QUERY_CACHE,
+ 7 * 24 * 3600);
+ MemcachedChunkingCache memcachedChunkingCache = new
MemcachedChunkingCache(memcachedCache);
+ memCachedAdaptor = new MemCachedCacheAdaptor(memcachedChunkingCache);
+
+ //Mock put to cache
+ for (String key : smallValueMap.keySet()) {
+ String keyS = memcachedCache.serializeKey(key);
+ String hashedKey = memcachedCache.computeKeyHash(keyS);
+
+ String value = smallValueMap.get(key);
+ byte[] valueB = memcachedCache.serializeValue(value);
+ KeyHookLookup.KeyHook keyHook = new KeyHookLookup.KeyHook(null,
valueB);
+ byte[] valueE = memcachedCache.encodeValue(keyS, keyHook);
+
+ GetFuture<Object> future = mock(GetFuture.class);
+ when(memcachedClient.asyncGet(hashedKey)).thenReturn(future);
+
+ when(future.get(cacheConfig.getTimeout(),
TimeUnit.MILLISECONDS)).thenReturn(valueE);
+ }
+
+ //Mock put large value to cache
+ for (String key : largeValueMap.keySet()) {
+ String keyS = memcachedCache.serializeKey(key);
+ String hashedKey = memcachedCache.computeKeyHash(keyS);
+
+ String value = largeValueMap.get(key);
+ byte[] valueB = memcachedCache.serializeValue(value);
+ int nSplit = MemcachedChunkingCache.getValueSplit(cacheConfig,
keyS, valueB.length);
+ Pair<KeyHookLookup.KeyHook, byte[][]> keyValuePair =
MemcachedChunkingCache.getKeyValuePair(nSplit, keyS,
+ valueB);
+ KeyHookLookup.KeyHook keyHook = keyValuePair.getFirst();
+ byte[][] splitValueB = keyValuePair.getSecond();
+
+ //For key
+ byte[] valueE = memcachedCache.encodeValue(keyS, keyHook);
+ GetFuture<Object> future = mock(GetFuture.class);
+ when(memcachedClient.asyncGet(hashedKey)).thenReturn(future);
+ when(future.get(cacheConfig.getTimeout(),
TimeUnit.MILLISECONDS)).thenReturn(valueE);
+
+ //For splits
+ Map<String, String> keyLookup = memcachedChunkingCache
+ .computeKeyHash(Arrays.asList(keyHook.getChunkskey()));
+ Map<String, Object> bulkResult = Maps.newHashMap();
+ for (int i = 0; i < nSplit; i++) {
+ String splitKeyS = keyHook.getChunkskey()[i];
+ bulkResult.put(memcachedCache.computeKeyHash(splitKeyS),
+
memcachedCache.encodeValue(splitKeyS.getBytes(Charsets.UTF_8), splitValueB[i]));
+ }
+
+ BulkFuture<Map<String, Object>> bulkFuture =
mock(BulkFuture.class);
+
when(memcachedClient.asyncGetBulk(keyLookup.keySet())).thenReturn(bulkFuture);
+ when(bulkFuture.get(cacheConfig.getTimeout(),
TimeUnit.MILLISECONDS)).thenReturn(bulkResult);
+ }
+ }
+
+ @After
+ public void after() throws Exception {
+ this.cleanupTestMetadata();
+ }
+
+ @Test
+ public void testGet() {
+ for (String key : smallValueMap.keySet()) {
+ Assert.assertEquals("The value should not change",
smallValueMap.get(key), memCachedAdaptor.get(key).get());
+ }
+ for (String key : largeValueMap.keySet()) {
+ Assert.assertEquals("The value should not change",
largeValueMap.get(key), memCachedAdaptor.get(key).get());
+ }
+ }
+
+ @Test
+ public void testSplitBytes() {
+ byte[] data = new byte[8];
+ for (int i = 0; i < data.length; i++) {
+ data[i] = (byte) i;
+ }
+
+ int nSplit;
+ byte[][] dataSplits;
+
+ nSplit = 2;
+ dataSplits = MemcachedChunkingCache.splitBytes(data, nSplit);
+ Assert.assertEquals(nSplit, dataSplits.length);
+ Assert.assertArrayEquals(dataSplits[0], new byte[] { 0, 1, 2, 3 });
+ Assert.assertArrayEquals(dataSplits[1], new byte[] { 4, 5, 6, 7 });
+
+ nSplit = 3;
+ dataSplits = MemcachedChunkingCache.splitBytes(data, nSplit);
+ Assert.assertEquals(nSplit, dataSplits.length);
+ Assert.assertArrayEquals(dataSplits[0], new byte[] { 0, 1, 2 });
+ Assert.assertArrayEquals(dataSplits[1], new byte[] { 3, 4, 5 });
+ Assert.assertArrayEquals(dataSplits[2], new byte[] { 6, 7 });
+ }
+}
\ No newline at end of file
diff --git a/cache/src/test/resources/cacheContext.xml
b/cache/src/test/resources/cacheContext.xml
new file mode 100644
index 0000000..a2fb9e9
--- /dev/null
+++ b/cache/src/test/resources/cacheContext.xml
@@ -0,0 +1,47 @@
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+<beans xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
+ xmlns:context="http://www.springframework.org/schema/context";
+ xmlns:cache="http://www.springframework.org/schema/cache";
+ xmlns:p="http://www.springframework.org/schema/p";
+ xmlns="http://www.springframework.org/schema/beans";
+ xsi:schemaLocation="http://www.springframework.org/schema/beans
+ http://www.springframework.org/schema/beans/spring-beans-4.3.xsd
+ http://www.springframework.org/schema/context
+ http://www.springframework.org/schema/context/spring-context-4.3.xsd
+ http://www.springframework.org/schema/cache
+ http://www.springframework.org/schema/cache/spring-cache.xsd";>
+
+ <description>Kylin Rest Service</description>
+ <context:annotation-config/>
+
+ <!-- Cache Config -->
+ <cache:annotation-driven/>
+
+ <beans profile="testing-memcached">
+ <bean id="ehcache"
+
class="org.springframework.cache.ehcache.EhCacheManagerFactoryBean"
+ p:configLocation="classpath:ehcache-test.xml" p:shared="true"/>
+ <bean id="localCacheManager"
class="org.apache.kylin.cache.cachemanager.InstrumentedEhCacheCacheManager"
+ p:cacheManager-ref="ehcache"/>
+
+ <bean id="remoteCacheManager"
class="org.apache.kylin.cache.cachemanager.MemcachedCacheManager"/>
+ <bean id="memcachedCacheConfig"
class="org.apache.kylin.cache.memcached.MemcachedCacheConfig">
+ <property name="timeout" value="500"/>
+ <property name="hosts" value="localhost:11211"/>
+ </bean>
+
+ <bean id="cacheManager"
class="org.apache.kylin.cache.cachemanager.RemoteLocalFailOverCacheManager"/>
+ </beans>
+
+</beans>
\ No newline at end of file
diff --git a/cache/src/test/resources/ehcache-test.xml
b/cache/src/test/resources/ehcache-test.xml
new file mode 100644
index 0000000..90299ec
--- /dev/null
+++ b/cache/src/test/resources/ehcache-test.xml
@@ -0,0 +1,21 @@
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+<ehcache maxBytesLocalHeap="256M">>
+ <cache name="StorageCache"
+ eternal="false"
+ timeToIdleSeconds="86400"
+ memoryStoreEvictionPolicy="LRU"
+ >
+ <persistence strategy="none"/>
+ </cache>
+</ehcache>
\ No newline at end of file
