KYLIN-2583 introduce DBCP as DataSource with cache logic
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5e6ed11e Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5e6ed11e Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5e6ed11e Branch: refs/heads/master Commit: 5e6ed11efd9a6a607fd1c8abf01f2c0d829c5e62 Parents: ca106b8 Author: Yifan Zhang <event.dim...@gmail.com> Authored: Fri May 5 09:46:28 2017 +0800 Committer: hongbin ma <m...@kyligence.io> Committed: Fri May 5 10:31:54 2017 +0800 ---------------------------------------------------------------------- .../org/apache/kylin/query/KylinTestBase.java | 4 +- .../java/org/apache/kylin/query/QueryCli.java | 9 +- .../org/apache/kylin/query/QueryDataSource.java | 162 +++++++++++++++++++ .../apache/kylin/query/QueryDataSourceTest.java | 86 ++++++++++ .../apache/kylin/rest/service/CacheService.java | 39 +---- 5 files changed, 256 insertions(+), 44 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/5e6ed11e/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java ---------------------------------------------------------------------- diff --git a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java index 2174094..fad7a94 100644 --- a/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java +++ b/kylin-it/src/test/java/org/apache/kylin/query/KylinTestBase.java @@ -49,7 +49,6 @@ import org.apache.kylin.common.util.Pair; import org.apache.kylin.metadata.project.ProjectInstance; import org.apache.kylin.query.relnode.OLAPContext; import org.apache.kylin.query.routing.rules.RemoveBlackoutRealizationsRule; -import org.apache.kylin.query.schema.OLAPSchemaFactory; import org.dbunit.DatabaseUnitException; import org.dbunit.database.DatabaseConfig; import org.dbunit.database.DatabaseConnection; @@ -659,8 +658,7 @@ public class KylinTestBase { config = KylinConfig.getInstanceFromEnv(); //setup cube conn - File olapTmp = OLAPSchemaFactory.createTempOLAPJson(ProjectInstance.DEFAULT_PROJECT_NAME, config); - cubeConnection = DriverManager.getConnection("jdbc:calcite:model=" + olapTmp.getAbsolutePath()); + cubeConnection = QueryDataSource.create(ProjectInstance.DEFAULT_PROJECT_NAME, config).getConnection(); //setup h2 h2Connection = DriverManager.getConnection("jdbc:h2:mem:db" + (h2InstanceCount++) + ";CACHE_SIZE=32072", "sa", ""); http://git-wip-us.apache.org/repos/asf/kylin/blob/5e6ed11e/query/src/main/java/org/apache/kylin/query/QueryCli.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/QueryCli.java b/query/src/main/java/org/apache/kylin/query/QueryCli.java index d3f1521..5ced8e3 100644 --- a/query/src/main/java/org/apache/kylin/query/QueryCli.java +++ b/query/src/main/java/org/apache/kylin/query/QueryCli.java @@ -18,14 +18,11 @@ package org.apache.kylin.query; -import java.io.File; import java.sql.Connection; -import java.sql.DriverManager; import java.sql.ResultSet; import java.sql.ResultSetMetaData; import java.sql.Statement; -import org.apache.calcite.jdbc.Driver; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.CommandLineParser; import org.apache.commons.cli.GnuParser; @@ -34,7 +31,6 @@ import org.apache.commons.cli.OptionBuilder; import org.apache.commons.cli.Options; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.util.DBUtils; -import org.apache.kylin.query.schema.OLAPSchemaFactory; public class QueryCli { @@ -55,14 +51,11 @@ public class QueryCli { KylinConfig config = KylinConfig.createInstanceFromUri(commandLine.getOptionValue(OPTION_METADATA.getOpt())); String sql = commandLine.getOptionValue(OPTION_SQL.getOpt()); - Class.forName(Driver.class.getName()); - File olapTmp = OLAPSchemaFactory.createTempOLAPJson(null, config); - Connection conn = null; Statement stmt = null; ResultSet rs = null; try { - conn = DriverManager.getConnection("jdbc:calcite:model=" + olapTmp.getAbsolutePath()); + conn = QueryDataSource.create(null, config).getConnection(); stmt = conn.createStatement(); rs = stmt.executeQuery(sql); http://git-wip-us.apache.org/repos/asf/kylin/blob/5e6ed11e/query/src/main/java/org/apache/kylin/query/QueryDataSource.java ---------------------------------------------------------------------- diff --git a/query/src/main/java/org/apache/kylin/query/QueryDataSource.java b/query/src/main/java/org/apache/kylin/query/QueryDataSource.java new file mode 100644 index 0000000..32feea6 --- /dev/null +++ b/query/src/main/java/org/apache/kylin/query/QueryDataSource.java @@ -0,0 +1,162 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.query; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.Charset; +import java.util.List; +import java.util.Properties; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +import javax.sql.DataSource; + +import org.apache.calcite.jdbc.Driver; +import org.apache.commons.dbcp.BasicDataSource; +import org.apache.commons.dbcp.BasicDataSourceFactory; +import org.apache.commons.io.FileUtils; +import org.apache.kylin.common.KylinConfig; +import org.apache.kylin.query.schema.OLAPSchemaFactory; +import org.apache.log4j.Logger; + +import com.google.common.collect.Lists; + +public class QueryDataSource { + + private static final Logger logger = Logger.getLogger(QueryDataSource.class); + + private ConcurrentMap<String, DataSource> olapDataSources = new ConcurrentHashMap<String, DataSource>(); + private List<File> usedFiles = Lists.newLinkedList(); + + /** + * Get available data source + * + * @param project + * @param config + * @return + * @throws Exception + */ + public DataSource get(String project, KylinConfig config) { + return get(project, config, new Properties()); + } + + /** + * Get available data source with advanced DBCP configuration + * + * @see http://commons.apache.org/proper/commons-dbcp/configuration.html + * + * @param project + * @param config + * @return + */ + public DataSource get(String project, KylinConfig config, Properties props) { + if (project == null) { + throw new IllegalArgumentException("project should not be null"); + } + + DataSource ds = olapDataSources.get(project); + if (ds != null) { + return ds; + } + + WrappedDataSource wrappedDS = getWrapped(project, config, props); + ds = wrappedDS.getDataSource(); + olapDataSources.putIfAbsent(project, ds); + usedFiles.add(wrappedDS.getOlapFile()); + return ds; + } + + public DataSource removeCache(String project) { + return olapDataSources.remove(project); + } + + public void clearCache() { + olapDataSources.clear(); + for (File usedFile : usedFiles) { + FileUtils.deleteQuietly(usedFile); + } + } + + /** + * Get data source without cache + * + * @param project + * @param config + * @return + */ + public static DataSource create(String project, KylinConfig config) { + return create(project, config, new Properties()); + } + + /** + * Get data source without cache, advanced configuration + * + * @param project + * @param config + * @param props + * @return + */ + public static DataSource create(String project, KylinConfig config, Properties props) { + return getWrapped(project, config, props).getDataSource(); + } + + private static WrappedDataSource getWrapped(String project, KylinConfig config, Properties props) { + File olapTmp = OLAPSchemaFactory.createTempOLAPJson(project, config); + if (logger.isDebugEnabled()) { + try { + String text = FileUtils.readFileToString(olapTmp, Charset.defaultCharset()); + logger.debug("The new temp olap json is :" + text); + } catch (IOException e) { + // logging failure is not critical + } + } + + BasicDataSource ds = null; + try { + ds = (BasicDataSource) BasicDataSourceFactory.createDataSource(props); + } catch (Exception e) { + // Basic mode + ds = new BasicDataSource(); + } + ds.setUrl("jdbc:calcite:model=" + olapTmp.getAbsolutePath()); + ds.setDriverClassName(Driver.class.getName()); + + WrappedDataSource wrappedDS = new WrappedDataSource(ds, olapTmp); + return wrappedDS; + } + + private static class WrappedDataSource { + private DataSource ds; + private File tempOlap; + + private WrappedDataSource(DataSource dataSource, File olapModel) { + this.ds = dataSource; + this.tempOlap = olapModel; + } + + public DataSource getDataSource() { + return ds; + } + + public File getOlapFile() { + return tempOlap; + } + } +} http://git-wip-us.apache.org/repos/asf/kylin/blob/5e6ed11e/query/src/test/java/org/apache/kylin/query/QueryDataSourceTest.java ---------------------------------------------------------------------- diff --git a/query/src/test/java/org/apache/kylin/query/QueryDataSourceTest.java b/query/src/test/java/org/apache/kylin/query/QueryDataSourceTest.java new file mode 100644 index 0000000..90d3518 --- /dev/null +++ b/query/src/test/java/org/apache/kylin/query/QueryDataSourceTest.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.query; + +import java.util.Properties; + +import javax.sql.DataSource; + +import org.apache.calcite.jdbc.Driver; +import org.apache.commons.dbcp.BasicDataSource; +import org.apache.kylin.common.KylinConfig; +import org.apache.kylin.common.util.LocalFileMetadataTestCase; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +public class QueryDataSourceTest extends LocalFileMetadataTestCase { + + @Before + public void setUp() throws Exception { + this.createTestMetadata(); + } + + @After + public void after() throws Exception { + this.cleanupTestMetadata(); + } + + @Test + public void testCreateDataSource() { + KylinConfig config = KylinConfig.getInstanceFromEnv(); + DataSource ds = QueryDataSource.create("default", config); + Assert.assertNotNull(ds); + } + + @Test + public void testCreateDataSourceWithProps() { + KylinConfig config = KylinConfig.getInstanceFromEnv(); + Properties props = new Properties(); + props.setProperty("username", "ADMIN"); + props.setProperty("password", "KYLIN"); + BasicDataSource ds = (BasicDataSource) QueryDataSource.create("default", config, props); + + Assert.assertNotNull(ds); + Assert.assertTrue(ds instanceof BasicDataSource); + Assert.assertTrue(ds.getUrl().startsWith("jdbc:calcite:model=")); + Assert.assertEquals(ds.getDriverClassName(), Driver.class.getName()); + Assert.assertEquals(ds.getUsername(), "ADMIN"); + Assert.assertEquals(ds.getPassword(), "KYLIN"); + } + + @Test + public void testGetCachedDataSource() { + QueryDataSource dsCache = new QueryDataSource(); + KylinConfig config = KylinConfig.getInstanceFromEnv(); + DataSource ds1 = dsCache.get("default", config); + DataSource ds2 = dsCache.get("default", config); + dsCache.removeCache("default"); + DataSource ds3 = dsCache.get("default", config); + + Assert.assertNotNull(ds1); + Assert.assertNotNull(ds2); + Assert.assertNotNull(ds3); + Assert.assertEquals(ds1, ds2); + Assert.assertNotEquals(ds1, ds3); + + dsCache.clearCache(); + } +} http://git-wip-us.apache.org/repos/asf/kylin/blob/5e6ed11e/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java index d3d6237..fc82bfe 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java +++ b/server-base/src/main/java/org/apache/kylin/rest/service/CacheService.java @@ -18,25 +18,18 @@ package org.apache.kylin.rest.service; -import java.io.File; import java.io.IOException; -import java.nio.charset.Charset; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; import javax.sql.DataSource; -import org.apache.calcite.jdbc.Driver; -import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; import org.apache.kylin.metadata.cachesync.Broadcaster; import org.apache.kylin.metadata.cachesync.Broadcaster.Event; import org.apache.kylin.metadata.project.ProjectInstance; -import org.apache.kylin.query.schema.OLAPSchemaFactory; +import org.apache.kylin.query.QueryDataSource; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.jdbc.datasource.DriverManagerDataSource; import org.springframework.stereotype.Component; import net.sf.ehcache.CacheManager; @@ -47,8 +40,8 @@ import net.sf.ehcache.CacheManager; public class CacheService extends BasicService { private static final Logger logger = LoggerFactory.getLogger(CacheService.class); - - private ConcurrentMap<String, DataSource> olapDataSources = new ConcurrentHashMap<String, DataSource>(); + + private static QueryDataSource queryDataSource = new QueryDataSource(); @Autowired private CubeService cubeService; @@ -137,40 +130,20 @@ public class CacheService extends BasicService { throw new IllegalArgumentException("removeOLAPDataSource: project name not given"); project = ProjectInstance.getNormalizedProjectName(project); - olapDataSources.remove(project); + queryDataSource.removeCache(project); } public void removeAllOLAPDataSources() { // brutal, yet simplest way logger.info("removeAllOLAPDataSources is called."); - olapDataSources.clear(); + queryDataSource.clearCache(); } public DataSource getOLAPDataSource(String project) { project = ProjectInstance.getNormalizedProjectName(project); - DataSource ret = olapDataSources.get(project); - if (ret == null) { - logger.debug("Creating a new data source, OLAP data source pointing to " + getConfig()); - File modelJson = OLAPSchemaFactory.createTempOLAPJson(project, getConfig()); - - try { - String text = FileUtils.readFileToString(modelJson, Charset.defaultCharset()); - logger.debug("The new temp olap json is :" + text); - } catch (IOException e) { - e.printStackTrace(); // logging failure is not critical - } - - DriverManagerDataSource ds = new DriverManagerDataSource(); - ds.setDriverClassName(Driver.class.getName()); - ds.setUrl("jdbc:calcite:model=" + modelJson.getAbsolutePath()); - - ret = olapDataSources.putIfAbsent(project, ds); - if (ret == null) { - ret = ds; - } - } + DataSource ret = queryDataSource.get(project, getConfig()); return ret; }
