integrating data manager with data catalog
Project: http://git-wip-us.apache.org/repos/asf/airavata/repo Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/15d8293c Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/15d8293c Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/15d8293c Branch: refs/heads/master Commit: 15d8293ce3d72943d0b855798610e56cfbb87f71 Parents: 387a8f6 Author: scnakandala <[email protected]> Authored: Wed Dec 2 23:10:35 2015 -0500 Committer: scnakandala <[email protected]> Committed: Tue Dec 15 21:11:40 2015 -0500 ---------------------------------------------------------------------- .../api/server/handler/utils/DataCatInit.java | 315 ++++++++++++++++++ modules/data-manager/pom.xml | 10 + .../airavata/data/manager/DataManager.java | 26 +- .../data/manager/DataManagerException.java | 35 ++ .../airavata/data/manager/DataManagerImpl.java | 61 +++- .../data/manager/DataManagerImptTest.java | 141 ++++++++ .../airavata/data/manager/utils/AppCatInit.java | 320 +++++++++++++++++++ .../data/manager/utils/DataCatInit.java | 315 ++++++++++++++++++ 8 files changed, 1206 insertions(+), 17 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/airavata/blob/15d8293c/airavata-api/airavata-api-server/src/test/java/org/apache/airavata/api/server/handler/utils/DataCatInit.java ---------------------------------------------------------------------- diff --git a/airavata-api/airavata-api-server/src/test/java/org/apache/airavata/api/server/handler/utils/DataCatInit.java b/airavata-api/airavata-api-server/src/test/java/org/apache/airavata/api/server/handler/utils/DataCatInit.java new file mode 100644 index 0000000..2b55692 --- /dev/null +++ b/airavata-api/airavata-api-server/src/test/java/org/apache/airavata/api/server/handler/utils/DataCatInit.java @@ -0,0 +1,315 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +package org.apache.airavata.api.server.handler.utils; + +import org.apache.airavata.common.exception.ApplicationSettingsException; +import org.apache.airavata.common.utils.ServerSettings; +import org.apache.airavata.registry.core.data.catalog.utils.DataCatalogConstants; +import org.apache.derby.drda.NetworkServerControl; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.InetAddress; +import java.net.URI; +import java.sql.*; +import java.util.StringTokenizer; + +public class DataCatInit { + private static final Logger logger = LoggerFactory.getLogger(DataCatInit.class); + public static final String DERBY_SERVER_MODE_SYS_PROPERTY = "derby.drda.startNetworkServer"; + public String scriptName = "datacatalog-derby.sql"; + private NetworkServerControl server; + private static final String delimiter = ";"; + private String jdbcUrl = null; + private String jdbcDriver = null; + private String jdbcUser = null; + private String jdbcPassword = null; + + public DataCatInit(String scriptName) { + this.scriptName = scriptName; + } + + public static boolean checkStringBufferEndsWith(StringBuffer buffer, String suffix) { + if (suffix.length() > buffer.length()) { + return false; + } + // this loop is done on purpose to avoid memory allocation performance + // problems on various JDKs + // StringBuffer.lastIndexOf() was introduced in jdk 1.4 and + // implementation is ok though does allocation/copying + // StringBuffer.toString().endsWith() does massive memory + // allocation/copying on JDK 1.5 + // See http://issues.apache.org/bugzilla/show_bug.cgi?id=37169 + int endIndex = suffix.length() - 1; + int bufferIndex = buffer.length() - 1; + while (endIndex >= 0) { + if (buffer.charAt(bufferIndex) != suffix.charAt(endIndex)) { + return false; + } + bufferIndex--; + endIndex--; + } + return true; + } + + private static boolean isServerStarted(NetworkServerControl server, int ntries) + { + for (int i = 1; i <= ntries; i ++) + { + try { + Thread.sleep(500); + server.ping(); + return true; + } + catch (Exception e) { + if (i == ntries) + return false; + } + } + return false; + } + + public void initializeDB() { + try{ + jdbcDriver = ServerSettings.getSetting("datacatalog.jdbc.driver"); + jdbcUrl = ServerSettings.getSetting("datacatalog.jdbc.url"); + jdbcUser = ServerSettings.getSetting("datacatalog.jdbc.user"); + jdbcPassword = ServerSettings.getSetting("datacatalog.jdbc.password"); + jdbcUrl = jdbcUrl + "?" + "user=" + jdbcUser + "&" + "password=" + jdbcPassword; + } catch (ApplicationSettingsException e) { + logger.error("Unable to read properties", e); + } + + startDerbyInServerMode(); + if(!isServerStarted(server, 20)){ + throw new RuntimeException("Derby server could not started within five seconds..."); + } + Connection conn = null; + try { + Class.forName(jdbcDriver).newInstance(); + conn = DriverManager.getConnection(jdbcUrl, jdbcUser, jdbcPassword); + if (!isDatabaseStructureCreated(DataCatalogConstants.CONFIGURATION, conn)) { + executeSQLScript(conn); + logger.info("New Database created for Data Catalog !!!"); + } else { + logger.debug("Database already created for Data Catalog!"); + } + } catch (Exception e) { + logger.error(e.getMessage(), e); + throw new RuntimeException("Database failure", e); + } finally { + try { + if (conn != null){ + if (!conn.getAutoCommit()) { + conn.commit(); + } + conn.close(); + } + } catch (SQLException e) { + logger.error(e.getMessage(), e); + } + } + } + + public static boolean isDatabaseStructureCreated(String tableName, Connection conn) { + try { + System.out.println("Running a query to test the database tables existence."); + // check whether the tables are already created with a query + Statement statement = null; + try { + statement = conn.createStatement(); + ResultSet rs = statement.executeQuery("select * from " + tableName); + if (rs != null) { + rs.close(); + } + } finally { + try { + if (statement != null) { + statement.close(); + } + } catch (SQLException e) { + return false; + } + } + } catch (SQLException e) { + return false; + } + + return true; + } + + private void executeSQLScript(Connection conn) throws Exception { + StringBuffer sql = new StringBuffer(); + BufferedReader reader = null; + try{ + + InputStream inputStream = this.getClass().getClassLoader().getResourceAsStream(scriptName); + reader = new BufferedReader(new InputStreamReader(inputStream)); + String line; + while ((line = reader.readLine()) != null) { + line = line.trim(); + if (line.startsWith("//")) { + continue; + } + if (line.startsWith("--")) { + continue; + } + StringTokenizer st = new StringTokenizer(line); + if (st.hasMoreTokens()) { + String token = st.nextToken(); + if ("REM".equalsIgnoreCase(token)) { + continue; + } + } + sql.append(" ").append(line); + + // SQL defines "--" as a comment to EOL + // and in Oracle it may contain a hint + // so we cannot just remove it, instead we must end it + if (line.indexOf("--") >= 0) { + sql.append("\n"); + } + if ((checkStringBufferEndsWith(sql, delimiter))) { + executeSQL(sql.substring(0, sql.length() - delimiter.length()), conn); + sql.replace(0, sql.length(), ""); + } + } + // Catch any statements not followed by ; + if (sql.length() > 0) { + executeSQL(sql.toString(), conn); + } + }catch (IOException e){ + logger.error("Error occurred while executing SQL script for creating Airavata Data Catalog database", e); + throw new Exception("Error occurred while executing SQL script for creating Airavata Data Catalog database", e); + }finally { + if (reader != null) { + reader.close(); + } + } + } + + private static void executeSQL(String sql, Connection conn) throws Exception { + // Check and ignore empty statements + if ("".equals(sql.trim())) { + return; + } + + Statement statement = null; + try { + logger.debug("SQL : " + sql); + + boolean ret; + int updateCount = 0, updateCountTotal = 0; + statement = conn.createStatement(); + ret = statement.execute(sql); + updateCount = statement.getUpdateCount(); + do { + if (!ret) { + if (updateCount != -1) { + updateCountTotal += updateCount; + } + } + ret = statement.getMoreResults(); + if (ret) { + updateCount = statement.getUpdateCount(); + } + } while (ret); + + logger.debug(sql + " : " + updateCountTotal + " rows affected"); + + SQLWarning warning = conn.getWarnings(); + while (warning != null) { + logger.warn(warning + " sql warning"); + warning = warning.getNextWarning(); + } + conn.clearWarnings(); + } catch (SQLException e) { + if (e.getSQLState().equals("X0Y32")) { + // eliminating the table already exception for the derby + // database + logger.info("Table Already Exists", e); + } else { + throw new Exception("Error occurred while executing : " + sql, e); + } + } finally { + if (statement != null) { + try { + statement.close(); + } catch (SQLException e) { + logger.error("Error occurred while closing result set.", e); + } + } + } + } + + private void startDerbyInServerMode() { + try { + System.setProperty(DERBY_SERVER_MODE_SYS_PROPERTY, "true"); + server = new NetworkServerControl(InetAddress.getByName("0.0.0.0"), + 20000, + jdbcUser, jdbcPassword); + java.io.PrintWriter consoleWriter = new java.io.PrintWriter(System.out, true); + server.start(consoleWriter); + } catch (IOException e) { + logger.error("Unable to start Apache derby in the server mode! Check whether " + + "specified port is available"); + } catch (Exception e) { + logger.error("Unable to start Apache derby in the server mode! Check whether " + + "specified port is available"); + } + + } + + public static int getPort(String jdbcURL){ + try{ + String cleanURI = jdbcURL.substring(5); + URI uri = URI.create(cleanURI); + return uri.getPort(); + } catch (Exception e) { + logger.error(e.getMessage(), e); + return -1; + } + } + + private void startDerbyInEmbeddedMode(){ + try { + Class.forName("org.apache.derby.jdbc.EmbeddedDriver"); + DriverManager.getConnection("jdbc:derby:memory:unit-testing-jpa;create=true").close(); + } catch (ClassNotFoundException e) { + logger.error(e.getMessage(), e); + } catch (SQLException e) { + logger.error(e.getMessage(), e); + } + } + + public void stopDerbyServer() { + try { + server.shutdown(); + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + } +} http://git-wip-us.apache.org/repos/asf/airavata/blob/15d8293c/modules/data-manager/pom.xml ---------------------------------------------------------------------- diff --git a/modules/data-manager/pom.xml b/modules/data-manager/pom.xml index ffd5082..cf898ea 100644 --- a/modules/data-manager/pom.xml +++ b/modules/data-manager/pom.xml @@ -21,6 +21,16 @@ <artifactId>airavata-data-models</artifactId> <version>${project.version}</version> </dependency> + <dependency> + <groupId>org.apache.airavata</groupId> + <artifactId>airavata-registry-core</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.airavata</groupId> + <artifactId>airavata-registry-cpi</artifactId> + <version>${project.version}</version> + </dependency> <dependency> <groupId>org.slf4j</groupId> http://git-wip-us.apache.org/repos/asf/airavata/blob/15d8293c/modules/data-manager/src/main/java/org/apache/airavata/data/manager/DataManager.java ---------------------------------------------------------------------- diff --git a/modules/data-manager/src/main/java/org/apache/airavata/data/manager/DataManager.java b/modules/data-manager/src/main/java/org/apache/airavata/data/manager/DataManager.java index 05afb90..7b046d5 100644 --- a/modules/data-manager/src/main/java/org/apache/airavata/data/manager/DataManager.java +++ b/modules/data-manager/src/main/java/org/apache/airavata/data/manager/DataManager.java @@ -30,28 +30,38 @@ public interface DataManager { * @param resource * @return */ - boolean publishResource(DataResourceModel resource); + String publishDataResource(DataResourceModel resource) throws DataManagerException; /** * To remove a resource entry from the replica catalog * @param resourceId * @return */ - boolean removeResource(String resourceId); + boolean removeDataResource(String resourceId) throws DataManagerException; + /** - * To copy an already existing resource to a specified location. After successful copying the new location will be - * added to the available replica locations of the resource - * @param resourceId - * @param destLocation + * To update an existing data resource model + * @param dataResourceModel * @return + * @throws DataManagerException */ - boolean copyResource(String resourceId, String destLocation); + boolean updateDataResource(DataResourceModel dataResourceModel) throws DataManagerException; /** * To retrieve a resource object providing the resourceId * @param resourceId * @return */ - DataResourceModel getResource(String resourceId); + DataResourceModel getDataResource(String resourceId) throws DataManagerException; + + /** + * To copy an already existing resource to a specified location. After successful copying the new location will be + * added to the available replica locations of the resource + * @param resourceId + * @param destLocation + * @return + */ + boolean copyDataResource(String resourceId, String destLocation) throws DataManagerException; + } http://git-wip-us.apache.org/repos/asf/airavata/blob/15d8293c/modules/data-manager/src/main/java/org/apache/airavata/data/manager/DataManagerException.java ---------------------------------------------------------------------- diff --git a/modules/data-manager/src/main/java/org/apache/airavata/data/manager/DataManagerException.java b/modules/data-manager/src/main/java/org/apache/airavata/data/manager/DataManagerException.java new file mode 100644 index 0000000..36a1a13 --- /dev/null +++ b/modules/data-manager/src/main/java/org/apache/airavata/data/manager/DataManagerException.java @@ -0,0 +1,35 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.airavata.data.manager; + +public class DataManagerException extends Exception{ + + public DataManagerException(Throwable e) { + super(e); + } + + public DataManagerException(String message) { + super(message, null); + } + + public DataManagerException(String message, Throwable e) { + super(message, e); + } +} http://git-wip-us.apache.org/repos/asf/airavata/blob/15d8293c/modules/data-manager/src/main/java/org/apache/airavata/data/manager/DataManagerImpl.java ---------------------------------------------------------------------- diff --git a/modules/data-manager/src/main/java/org/apache/airavata/data/manager/DataManagerImpl.java b/modules/data-manager/src/main/java/org/apache/airavata/data/manager/DataManagerImpl.java index fb92edf..a8b1ae6 100644 --- a/modules/data-manager/src/main/java/org/apache/airavata/data/manager/DataManagerImpl.java +++ b/modules/data-manager/src/main/java/org/apache/airavata/data/manager/DataManagerImpl.java @@ -20,30 +20,73 @@ */ package org.apache.airavata.data.manager; -import org.apache.airavata.model.data.resource.ResourceModel; +import org.apache.airavata.model.data.resource.DataResourceModel; +import org.apache.airavata.registry.core.experiment.catalog.impl.RegistryFactory; +import org.apache.airavata.registry.cpi.DataCatalog; +import org.apache.airavata.registry.cpi.DataCatalogException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class DataManagerImpl implements DataManager{ private final static Logger logger = LoggerFactory.getLogger(DataManagerImpl.class); + private DataCatalog dataCatalog; + + public DataManagerImpl() throws DataManagerException { + try { + this.dataCatalog = RegistryFactory.getDataCatalog(); + } catch (DataCatalogException e) { + logger.error(e.getMessage(), e); + throw new DataManagerException(e); + } + } + @Override - public boolean publishResource(ResourceModel resource) { - return false; + public String publishDataResource(DataResourceModel resourceModel) throws DataManagerException{ + try { + String resourceId = dataCatalog.publishResource(resourceModel); + return resourceId; + } catch (DataCatalogException e) { + logger.error(e.getMessage(), e); + throw new DataManagerException(e); + } } @Override - public boolean removeResource(String resourceId) { - return false; + public boolean removeDataResource(String resourceId) throws DataManagerException { + try { + boolean result = dataCatalog.removeResource(resourceId); + return result; + } catch (DataCatalogException e) { + logger.error(e.getMessage(), e); + throw new DataManagerException(e); + } } @Override - public boolean copyResource(String resourceId, String destLocation) { - return false; + public boolean updateDataResource(DataResourceModel dataResourceModel) throws DataManagerException { + try { + boolean result = dataCatalog.updateResource(dataResourceModel); + return result; + } catch (DataCatalogException e) { + logger.error(e.getMessage(), e); + throw new DataManagerException(e); + } } @Override - public ResourceModel getResource(String resourceId) { - return null; + public DataResourceModel getDataResource(String resourceId) throws DataManagerException { + try { + DataResourceModel dataResource = dataCatalog.getResource(resourceId); + return dataResource; + } catch (DataCatalogException e) { + logger.error(e.getMessage(), e); + throw new DataManagerException(e); + } + } + + @Override + public boolean copyDataResource(String resourceId, String destLocation) throws DataManagerException{ + return false; } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata/blob/15d8293c/modules/data-manager/src/test/java/org/apache/airavata/data/manager/DataManagerImptTest.java ---------------------------------------------------------------------- diff --git a/modules/data-manager/src/test/java/org/apache/airavata/data/manager/DataManagerImptTest.java b/modules/data-manager/src/test/java/org/apache/airavata/data/manager/DataManagerImptTest.java new file mode 100644 index 0000000..b6bc751 --- /dev/null +++ b/modules/data-manager/src/test/java/org/apache/airavata/data/manager/DataManagerImptTest.java @@ -0,0 +1,141 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.data.manager; + +import org.apache.airavata.data.manager.utils.AppCatInit; +import org.apache.airavata.data.manager.utils.DataCatInit; +import org.apache.airavata.model.data.resource.DataReplicaLocationModel; +import org.apache.airavata.model.data.resource.DataResourceModel; +import org.apache.airavata.registry.core.experiment.catalog.impl.RegistryFactory; +import org.apache.airavata.registry.cpi.DataCatalog; +import org.apache.airavata.registry.cpi.DataCatalogException; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; + +public class DataManagerImptTest { + private final static Logger logger = LoggerFactory.getLogger(DataManagerImptTest.class); + private static AppCatInit appCatInit; + private static DataCatInit dataCatInit; + private static DataManager dataManager; + private static DataResourceModel dataResourceModel; + + @BeforeClass + public static void setUp() { + try { + System.out.println("********** SET UP ************"); + appCatInit = new AppCatInit("appcatalog-derby.sql"); + appCatInit.initializeDB(); + dataCatInit = new DataCatInit("datacatalog-derby.sql"); + dataCatInit.initializeDB(); + dataManager = DataManagerFactory.getDataManager(); + dataResourceModel = new DataResourceModel(); + dataResourceModel.setResourceName("test-file.txt"); + List<DataReplicaLocationModel> replicaLocationModelList = new ArrayList<>(); + DataReplicaLocationModel dataReplicaLocationModel = new DataReplicaLocationModel(); + dataReplicaLocationModel.setReplicaName("1-st-replica"); + ArrayList<String> dataLocations = new ArrayList<>(); + dataLocations.add("scp://g75.iu.xsede.org:/var/www/portal/experimentData/test-file.txt"); + dataReplicaLocationModel.setDataLocations(dataLocations); + replicaLocationModelList.add(dataReplicaLocationModel); + dataResourceModel.setReplicaLocations(replicaLocationModelList); + } catch (DataCatalogException e) { + logger.error(e.getMessage(), e); + } + } + + @AfterClass + public static void tearDown() throws Exception { + System.out.println("********** TEAR DOWN ************"); + appCatInit.stopDerbyServer(); + dataCatInit.stopDerbyServer(); + } + + @Test + public void testPublishDataResource(){ + try { + String resourceId = dataManager.publishDataResource(dataResourceModel); + org.junit.Assert.assertNotNull(resourceId); + } catch (DataManagerException e) { + e.printStackTrace(); + } + } + + @Test + public void testRemoveDataResource(){ + try { + boolean result = dataManager.removeDataResource("234234234"); + Assert.assertFalse(result); + String resourceId = dataManager.publishDataResource(dataResourceModel); + Assert.assertNotNull(resourceId); + result = dataManager.removeDataResource(resourceId); + Assert.assertTrue(result); + result = dataManager.removeDataResource(resourceId); + Assert.assertFalse(result); + } catch (DataManagerException e) { + e.printStackTrace(); + } + } + + @Test + public void testGetDataResource(){ + try { + String resourceId = dataManager.publishDataResource(dataResourceModel); + Assert.assertNotNull(resourceId); + DataResourceModel persistedCopy = dataManager.getDataResource(resourceId); + Assert.assertNotNull(persistedCopy); + Assert.assertTrue(persistedCopy.getReplicaLocations().size()==1); + } catch (DataManagerException e) { + e.printStackTrace(); + } + } + + @Test + public void testUpdateDataResource(){ + try { + dataResourceModel.setResourceId(UUID.randomUUID().toString()); + boolean result = dataManager.updateDataResource(dataResourceModel); + Assert.assertFalse(result); + dataManager.publishDataResource(dataResourceModel); + DataReplicaLocationModel dataReplicaLocationModel = new DataReplicaLocationModel(); + dataReplicaLocationModel.setReplicaName("2-nd-replica"); + ArrayList<String> dataLocations = new ArrayList<>(); + dataLocations.add("scp://g175.iu.xsede.org:/var/www/portal/experimentData/test-file.txt"); + dataResourceModel.getReplicaLocations().add(dataReplicaLocationModel); + dataManager.updateDataResource(dataResourceModel); + dataResourceModel = dataManager.getDataResource(dataResourceModel.getResourceId()); + Assert.assertTrue(dataResourceModel.getReplicaLocations().size()==2); + dataResourceModel.getReplicaLocations().remove(1); + dataManager.updateDataResource(dataResourceModel); + dataResourceModel = dataManager.getDataResource(dataResourceModel.getResourceId()); + Assert.assertTrue(dataResourceModel.getReplicaLocations().size()==1); + } catch (DataManagerException e) { + e.printStackTrace(); + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata/blob/15d8293c/modules/data-manager/src/test/java/org/apache/airavata/data/manager/utils/AppCatInit.java ---------------------------------------------------------------------- diff --git a/modules/data-manager/src/test/java/org/apache/airavata/data/manager/utils/AppCatInit.java b/modules/data-manager/src/test/java/org/apache/airavata/data/manager/utils/AppCatInit.java new file mode 100644 index 0000000..2433b78 --- /dev/null +++ b/modules/data-manager/src/test/java/org/apache/airavata/data/manager/utils/AppCatInit.java @@ -0,0 +1,320 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +package org.apache.airavata.data.manager.utils; + +import org.apache.airavata.common.exception.ApplicationSettingsException; +import org.apache.airavata.common.utils.ServerSettings; +import org.apache.derby.drda.NetworkServerControl; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.InetAddress; +import java.net.URI; +import java.sql.*; +import java.util.StringTokenizer; + +public class AppCatInit { + private static final Logger logger = LoggerFactory.getLogger(AppCatInit.class); + public static final String DERBY_SERVER_MODE_SYS_PROPERTY = "derby.drda.startNetworkServer"; + public String scriptName = "appcatalog-derby.sql"; + private NetworkServerControl server; + private static final String delimiter = ";"; + public static final String COMPUTE_RESOURCE_TABLE = "COMPUTE_RESOURCE"; + private String jdbcUrl = null; + private String jdbcDriver = null; + private String jdbcUser = null; + private String jdbcPassword = null; + + public AppCatInit(String scriptName) { + this.scriptName = scriptName; + } + + public static boolean checkStringBufferEndsWith(StringBuffer buffer, String suffix) { + if (suffix.length() > buffer.length()) { + return false; + } + // this loop is done on purpose to avoid memory allocation performance + // problems on various JDKs + // StringBuffer.lastIndexOf() was introduced in jdk 1.4 and + // implementation is ok though does allocation/copying + // StringBuffer.toString().endsWith() does massive memory + // allocation/copying on JDK 1.5 + // See http://issues.apache.org/bugzilla/show_bug.cgi?id=37169 + int endIndex = suffix.length() - 1; + int bufferIndex = buffer.length() - 1; + while (endIndex >= 0) { + if (buffer.charAt(bufferIndex) != suffix.charAt(endIndex)) { + return false; + } + bufferIndex--; + endIndex--; + } + return true; + } + + private static boolean isServerStarted(NetworkServerControl server, int ntries) + { + for (int i = 1; i <= ntries; i ++) + { + try { + Thread.sleep(500); + server.ping(); + return true; + } + catch (Exception e) { + if (i == ntries) + return false; + } + } + return false; + } + + public void initializeDB() { + + try{ + jdbcDriver = ServerSettings.getSetting("appcatalog.jdbc.driver"); + jdbcUrl = ServerSettings.getSetting("appcatalog.jdbc.url"); + jdbcUser = ServerSettings.getSetting("appcatalog.jdbc.user"); + jdbcPassword = ServerSettings.getSetting("appcatalog.jdbc.password"); + jdbcUrl = jdbcUrl + "?" + "user=" + jdbcUser + "&" + "password=" + jdbcPassword; + } catch (ApplicationSettingsException e) { + logger.error("Unable to read properties", e); + } + + startDerbyInServerMode(); + if(!isServerStarted(server, 20)){ + throw new RuntimeException("Derby server cound not started within five seconds..."); + } +// startDerbyInEmbeddedMode(); + + Connection conn = null; + try { + Class.forName(jdbcDriver).newInstance(); + conn = DriverManager.getConnection(jdbcUrl, jdbcUser, jdbcPassword); + if (!isDatabaseStructureCreated(COMPUTE_RESOURCE_TABLE, conn)) { + executeSQLScript(conn); + logger.info("New Database created for App Catalog !!!"); + } else { + logger.debug("Database already created for App Catalog!"); + } + } catch (Exception e) { + logger.error(e.getMessage(), e); + throw new RuntimeException("Database failure", e); + } finally { + try { + if (conn != null){ + if (!conn.getAutoCommit()) { + conn.commit(); + } + conn.close(); + } + } catch (SQLException e) { + logger.error(e.getMessage(), e); + } + } + } + + public static boolean isDatabaseStructureCreated(String tableName, Connection conn) { + try { + System.out.println("Running a query to test the database tables existence."); + // check whether the tables are already created with a query + Statement statement = null; + try { + statement = conn.createStatement(); + ResultSet rs = statement.executeQuery("select * from " + tableName); + if (rs != null) { + rs.close(); + } + } finally { + try { + if (statement != null) { + statement.close(); + } + } catch (SQLException e) { + return false; + } + } + } catch (SQLException e) { + return false; + } + + return true; + } + + private void executeSQLScript(Connection conn) throws Exception { + StringBuffer sql = new StringBuffer(); + BufferedReader reader = null; + try{ + + InputStream inputStream = this.getClass().getClassLoader().getResourceAsStream(scriptName); + reader = new BufferedReader(new InputStreamReader(inputStream)); + String line; + while ((line = reader.readLine()) != null) { + line = line.trim(); + if (line.startsWith("//")) { + continue; + } + if (line.startsWith("--")) { + continue; + } + StringTokenizer st = new StringTokenizer(line); + if (st.hasMoreTokens()) { + String token = st.nextToken(); + if ("REM".equalsIgnoreCase(token)) { + continue; + } + } + sql.append(" ").append(line); + + // SQL defines "--" as a comment to EOL + // and in Oracle it may contain a hint + // so we cannot just remove it, instead we must end it + if (line.indexOf("--") >= 0) { + sql.append("\n"); + } + if ((checkStringBufferEndsWith(sql, delimiter))) { + executeSQL(sql.substring(0, sql.length() - delimiter.length()), conn); + sql.replace(0, sql.length(), ""); + } + } + // Catch any statements not followed by ; + if (sql.length() > 0) { + executeSQL(sql.toString(), conn); + } + }catch (IOException e){ + logger.error("Error occurred while executing SQL script for creating Airavata database", e); + throw new Exception("Error occurred while executing SQL script for creating Airavata database", e); + }finally { + if (reader != null) { + reader.close(); + } + + } + + } + + private static void executeSQL(String sql, Connection conn) throws Exception { + // Check and ignore empty statements + if ("".equals(sql.trim())) { + return; + } + + Statement statement = null; + try { + logger.debug("SQL : " + sql); + + boolean ret; + int updateCount = 0, updateCountTotal = 0; + statement = conn.createStatement(); + ret = statement.execute(sql); + updateCount = statement.getUpdateCount(); + do { + if (!ret) { + if (updateCount != -1) { + updateCountTotal += updateCount; + } + } + ret = statement.getMoreResults(); + if (ret) { + updateCount = statement.getUpdateCount(); + } + } while (ret); + + logger.debug(sql + " : " + updateCountTotal + " rows affected"); + + SQLWarning warning = conn.getWarnings(); + while (warning != null) { + logger.warn(warning + " sql warning"); + warning = warning.getNextWarning(); + } + conn.clearWarnings(); + } catch (SQLException e) { + if (e.getSQLState().equals("X0Y32")) { + // eliminating the table already exception for the derby + // database + logger.info("Table Already Exists", e); + } else { + throw new Exception("Error occurred while executing : " + sql, e); + } + } finally { + if (statement != null) { + try { + statement.close(); + } catch (SQLException e) { + logger.error("Error occurred while closing result set.", e); + } + } + } + } + + private void startDerbyInServerMode() { + try { + System.setProperty(DERBY_SERVER_MODE_SYS_PROPERTY, "true"); + server = new NetworkServerControl(InetAddress.getByName("0.0.0.0"), + 20000, + jdbcUser, jdbcPassword); + java.io.PrintWriter consoleWriter = new java.io.PrintWriter(System.out, true); + server.start(consoleWriter); + } catch (IOException e) { + logger.error("Unable to start Apache derby in the server mode! Check whether " + + "specified port is available"); + } catch (Exception e) { + logger.error("Unable to start Apache derby in the server mode! Check whether " + + "specified port is available"); + } + + } + + public static int getPort(String jdbcURL){ + try{ + String cleanURI = jdbcURL.substring(5); + URI uri = URI.create(cleanURI); + return uri.getPort(); + } catch (Exception e) { + logger.error(e.getMessage(), e); + return -1; + } + } + + private void startDerbyInEmbeddedMode(){ + try { + Class.forName("org.apache.derby.jdbc.EmbeddedDriver"); + DriverManager.getConnection("jdbc:derby:memory:unit-testing-jpa;create=true").close(); + } catch (ClassNotFoundException e) { + logger.error(e.getMessage(), e); + } catch (SQLException e) { + logger.error(e.getMessage(), e); + } + } + + public void stopDerbyServer() { + try { + server.shutdown(); + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + } +} http://git-wip-us.apache.org/repos/asf/airavata/blob/15d8293c/modules/data-manager/src/test/java/org/apache/airavata/data/manager/utils/DataCatInit.java ---------------------------------------------------------------------- diff --git a/modules/data-manager/src/test/java/org/apache/airavata/data/manager/utils/DataCatInit.java b/modules/data-manager/src/test/java/org/apache/airavata/data/manager/utils/DataCatInit.java new file mode 100644 index 0000000..05165a2 --- /dev/null +++ b/modules/data-manager/src/test/java/org/apache/airavata/data/manager/utils/DataCatInit.java @@ -0,0 +1,315 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +package org.apache.airavata.data.manager.utils; + +import org.apache.airavata.common.exception.ApplicationSettingsException; +import org.apache.airavata.common.utils.ServerSettings; +import org.apache.airavata.registry.core.data.catalog.utils.DataCatalogConstants; +import org.apache.derby.drda.NetworkServerControl; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.InetAddress; +import java.net.URI; +import java.sql.*; +import java.util.StringTokenizer; + +public class DataCatInit { + private static final Logger logger = LoggerFactory.getLogger(DataCatInit.class); + public static final String DERBY_SERVER_MODE_SYS_PROPERTY = "derby.drda.startNetworkServer"; + public String scriptName = "datacatalog-derby.sql"; + private NetworkServerControl server; + private static final String delimiter = ";"; + private String jdbcUrl = null; + private String jdbcDriver = null; + private String jdbcUser = null; + private String jdbcPassword = null; + + public DataCatInit(String scriptName) { + this.scriptName = scriptName; + } + + public static boolean checkStringBufferEndsWith(StringBuffer buffer, String suffix) { + if (suffix.length() > buffer.length()) { + return false; + } + // this loop is done on purpose to avoid memory allocation performance + // problems on various JDKs + // StringBuffer.lastIndexOf() was introduced in jdk 1.4 and + // implementation is ok though does allocation/copying + // StringBuffer.toString().endsWith() does massive memory + // allocation/copying on JDK 1.5 + // See http://issues.apache.org/bugzilla/show_bug.cgi?id=37169 + int endIndex = suffix.length() - 1; + int bufferIndex = buffer.length() - 1; + while (endIndex >= 0) { + if (buffer.charAt(bufferIndex) != suffix.charAt(endIndex)) { + return false; + } + bufferIndex--; + endIndex--; + } + return true; + } + + private static boolean isServerStarted(NetworkServerControl server, int ntries) + { + for (int i = 1; i <= ntries; i ++) + { + try { + Thread.sleep(500); + server.ping(); + return true; + } + catch (Exception e) { + if (i == ntries) + return false; + } + } + return false; + } + + public void initializeDB() { + try{ + jdbcDriver = ServerSettings.getSetting("datacatalog.jdbc.driver"); + jdbcUrl = ServerSettings.getSetting("datacatalog.jdbc.url"); + jdbcUser = ServerSettings.getSetting("datacatalog.jdbc.user"); + jdbcPassword = ServerSettings.getSetting("datacatalog.jdbc.password"); + jdbcUrl = jdbcUrl + "?" + "user=" + jdbcUser + "&" + "password=" + jdbcPassword; + } catch (ApplicationSettingsException e) { + logger.error("Unable to read properties", e); + } + + startDerbyInServerMode(); + if(!isServerStarted(server, 20)){ + throw new RuntimeException("Derby server could not started within five seconds..."); + } + Connection conn = null; + try { + Class.forName(jdbcDriver).newInstance(); + conn = DriverManager.getConnection(jdbcUrl, jdbcUser, jdbcPassword); + if (!isDatabaseStructureCreated(DataCatalogConstants.CONFIGURATION, conn)) { + executeSQLScript(conn); + logger.info("New Database created for Data Catalog !!!"); + } else { + logger.debug("Database already created for Data Catalog!"); + } + } catch (Exception e) { + logger.error(e.getMessage(), e); + throw new RuntimeException("Database failure", e); + } finally { + try { + if (conn != null){ + if (!conn.getAutoCommit()) { + conn.commit(); + } + conn.close(); + } + } catch (SQLException e) { + logger.error(e.getMessage(), e); + } + } + } + + public static boolean isDatabaseStructureCreated(String tableName, Connection conn) { + try { + System.out.println("Running a query to test the database tables existence."); + // check whether the tables are already created with a query + Statement statement = null; + try { + statement = conn.createStatement(); + ResultSet rs = statement.executeQuery("select * from " + tableName); + if (rs != null) { + rs.close(); + } + } finally { + try { + if (statement != null) { + statement.close(); + } + } catch (SQLException e) { + return false; + } + } + } catch (SQLException e) { + return false; + } + + return true; + } + + private void executeSQLScript(Connection conn) throws Exception { + StringBuffer sql = new StringBuffer(); + BufferedReader reader = null; + try{ + + InputStream inputStream = this.getClass().getClassLoader().getResourceAsStream(scriptName); + reader = new BufferedReader(new InputStreamReader(inputStream)); + String line; + while ((line = reader.readLine()) != null) { + line = line.trim(); + if (line.startsWith("//")) { + continue; + } + if (line.startsWith("--")) { + continue; + } + StringTokenizer st = new StringTokenizer(line); + if (st.hasMoreTokens()) { + String token = st.nextToken(); + if ("REM".equalsIgnoreCase(token)) { + continue; + } + } + sql.append(" ").append(line); + + // SQL defines "--" as a comment to EOL + // and in Oracle it may contain a hint + // so we cannot just remove it, instead we must end it + if (line.indexOf("--") >= 0) { + sql.append("\n"); + } + if ((checkStringBufferEndsWith(sql, delimiter))) { + executeSQL(sql.substring(0, sql.length() - delimiter.length()), conn); + sql.replace(0, sql.length(), ""); + } + } + // Catch any statements not followed by ; + if (sql.length() > 0) { + executeSQL(sql.toString(), conn); + } + }catch (IOException e){ + logger.error("Error occurred while executing SQL script for creating Airavata Data Catalog database", e); + throw new Exception("Error occurred while executing SQL script for creating Airavata Data Catalog database", e); + }finally { + if (reader != null) { + reader.close(); + } + } + } + + private static void executeSQL(String sql, Connection conn) throws Exception { + // Check and ignore empty statements + if ("".equals(sql.trim())) { + return; + } + + Statement statement = null; + try { + logger.debug("SQL : " + sql); + + boolean ret; + int updateCount = 0, updateCountTotal = 0; + statement = conn.createStatement(); + ret = statement.execute(sql); + updateCount = statement.getUpdateCount(); + do { + if (!ret) { + if (updateCount != -1) { + updateCountTotal += updateCount; + } + } + ret = statement.getMoreResults(); + if (ret) { + updateCount = statement.getUpdateCount(); + } + } while (ret); + + logger.debug(sql + " : " + updateCountTotal + " rows affected"); + + SQLWarning warning = conn.getWarnings(); + while (warning != null) { + logger.warn(warning + " sql warning"); + warning = warning.getNextWarning(); + } + conn.clearWarnings(); + } catch (SQLException e) { + if (e.getSQLState().equals("X0Y32")) { + // eliminating the table already exception for the derby + // database + logger.info("Table Already Exists", e); + } else { + throw new Exception("Error occurred while executing : " + sql, e); + } + } finally { + if (statement != null) { + try { + statement.close(); + } catch (SQLException e) { + logger.error("Error occurred while closing result set.", e); + } + } + } + } + + private void startDerbyInServerMode() { + try { + System.setProperty(DERBY_SERVER_MODE_SYS_PROPERTY, "true"); + server = new NetworkServerControl(InetAddress.getByName("0.0.0.0"), + 20000, + jdbcUser, jdbcPassword); + java.io.PrintWriter consoleWriter = new java.io.PrintWriter(System.out, true); + server.start(consoleWriter); + } catch (IOException e) { + logger.error("Unable to start Apache derby in the server mode! Check whether " + + "specified port is available"); + } catch (Exception e) { + logger.error("Unable to start Apache derby in the server mode! Check whether " + + "specified port is available"); + } + + } + + public static int getPort(String jdbcURL){ + try{ + String cleanURI = jdbcURL.substring(5); + URI uri = URI.create(cleanURI); + return uri.getPort(); + } catch (Exception e) { + logger.error(e.getMessage(), e); + return -1; + } + } + + private void startDerbyInEmbeddedMode(){ + try { + Class.forName("org.apache.derby.jdbc.EmbeddedDriver"); + DriverManager.getConnection("jdbc:derby:memory:unit-testing-jpa;create=true").close(); + } catch (ClassNotFoundException e) { + logger.error(e.getMessage(), e); + } catch (SQLException e) { + logger.error(e.getMessage(), e); + } + } + + public void stopDerbyServer() { + try { + server.shutdown(); + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + } +}
