Repository: airavata Updated Branches: refs/heads/master efa66f9ba -> 80d202785
Tests are failing with new thrift generated code, need to fix. Committing with build wokring without tetss Project: http://git-wip-us.apache.org/repos/asf/airavata/repo Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/80d20278 Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/80d20278 Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/80d20278 Branch: refs/heads/master Commit: 80d202785f05f76fee6fa6461c38cfd759444740 Parents: efa66f9 Author: smarru <sma...@apache.org> Authored: Wed Dec 16 11:34:09 2015 -0500 Committer: smarru <sma...@apache.org> Committed: Wed Dec 16 11:34:09 2015 -0500 ---------------------------------------------------------------------- .../server/handler/AiravataServerHandler.java | 7 +- .../airavata/data/manager/DataManager.java | 67 ---- .../data/manager/DataManagerException.java | 35 -- .../data/manager/DataManagerFactory.java | 32 -- .../airavata/data/manager/DataManagerImpl.java | 92 ------ .../data/manager/DataManagerFactoryTest.java | 36 --- .../data/manager/DataManagerImptTest.java | 141 -------- .../airavata/data/manager/utils/AppCatInit.java | 320 ------------------- .../data/manager/utils/DataCatInit.java | 315 ------------------ .../airavata-api/data_movement_models.thrift | 154 --------- .../airavata-api/data_resource_models.thrift | 45 --- 11 files changed, 4 insertions(+), 1240 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/airavata/blob/80d20278/airavata-api/airavata-api-server/src/main/java/org/apache/airavata/api/server/handler/AiravataServerHandler.java ---------------------------------------------------------------------- diff --git a/airavata-api/airavata-api-server/src/main/java/org/apache/airavata/api/server/handler/AiravataServerHandler.java b/airavata-api/airavata-api-server/src/main/java/org/apache/airavata/api/server/handler/AiravataServerHandler.java index 52bfa80..d9121d8 100644 --- a/airavata-api/airavata-api-server/src/main/java/org/apache/airavata/api/server/handler/AiravataServerHandler.java +++ b/airavata-api/airavata-api-server/src/main/java/org/apache/airavata/api/server/handler/AiravataServerHandler.java @@ -72,9 +72,10 @@ import org.apache.airavata.registry.core.experiment.catalog.ExpCatResourceUtils; import org.apache.airavata.registry.core.experiment.catalog.impl.RegistryFactory; import org.apache.airavata.registry.cpi.*; import org.apache.airavata.registry.cpi.utils.Constants; -import org.apache.airavata.data.catalog.core.DataManagerFactory; -import org.apache.airavata.data.catalog.cpi.DataManager; -import org.apache.airavata.data.catalog.cpi.DataManagerException; + +import org.apache.airavata.replica.catalog.core.DataManagerFactory; +import org.apache.airavata.replica.catalog.cpi.DataManager; +import org.apache.airavata.replica.catalog.cpi.DataManagerException; import org.apache.thrift.TException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; http://git-wip-us.apache.org/repos/asf/airavata/blob/80d20278/modules/data-manager/src/main/java/org/apache/airavata/data/manager/DataManager.java ---------------------------------------------------------------------- diff --git a/modules/data-manager/src/main/java/org/apache/airavata/data/manager/DataManager.java b/modules/data-manager/src/main/java/org/apache/airavata/data/manager/DataManager.java deleted file mode 100644 index 7b046d5..0000000 --- a/modules/data-manager/src/main/java/org/apache/airavata/data/manager/DataManager.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - */ -package org.apache.airavata.data.manager; - -import org.apache.airavata.model.data.resource.DataResourceModel; - -public interface DataManager { - - /** - * To create a replica entry for an already existing file(s). This is how the system comes to know about already - * existing resources - * @param resource - * @return - */ - String publishDataResource(DataResourceModel resource) throws DataManagerException; - - /** - * To remove a resource entry from the replica catalog - * @param resourceId - * @return - */ - boolean removeDataResource(String resourceId) throws DataManagerException; - - - /** - * To update an existing data resource model - * @param dataResourceModel - * @return - * @throws DataManagerException - */ - boolean updateDataResource(DataResourceModel dataResourceModel) throws DataManagerException; - - /** - * To retrieve a resource object providing the resourceId - * @param resourceId - * @return - */ - DataResourceModel getDataResource(String resourceId) throws DataManagerException; - - /** - * To copy an already existing resource to a specified location. After successful copying the new location will be - * added to the available replica locations of the resource - * @param resourceId - * @param destLocation - * @return - */ - boolean copyDataResource(String resourceId, String destLocation) throws DataManagerException; - -} http://git-wip-us.apache.org/repos/asf/airavata/blob/80d20278/modules/data-manager/src/main/java/org/apache/airavata/data/manager/DataManagerException.java ---------------------------------------------------------------------- diff --git a/modules/data-manager/src/main/java/org/apache/airavata/data/manager/DataManagerException.java b/modules/data-manager/src/main/java/org/apache/airavata/data/manager/DataManagerException.java deleted file mode 100644 index 36a1a13..0000000 --- a/modules/data-manager/src/main/java/org/apache/airavata/data/manager/DataManagerException.java +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.airavata.data.manager; - -public class DataManagerException extends Exception{ - - public DataManagerException(Throwable e) { - super(e); - } - - public DataManagerException(String message) { - super(message, null); - } - - public DataManagerException(String message, Throwable e) { - super(message, e); - } -} http://git-wip-us.apache.org/repos/asf/airavata/blob/80d20278/modules/data-manager/src/main/java/org/apache/airavata/data/manager/DataManagerFactory.java ---------------------------------------------------------------------- diff --git a/modules/data-manager/src/main/java/org/apache/airavata/data/manager/DataManagerFactory.java b/modules/data-manager/src/main/java/org/apache/airavata/data/manager/DataManagerFactory.java deleted file mode 100644 index e1fb18a..0000000 --- a/modules/data-manager/src/main/java/org/apache/airavata/data/manager/DataManagerFactory.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * -*/ -package org.apache.airavata.data.manager; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class DataManagerFactory { - private final static Logger logger = LoggerFactory.getLogger(DataManagerFactory.class); - - public static DataManager getDataManager(){ - return new DataManagerImpl(); - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata/blob/80d20278/modules/data-manager/src/main/java/org/apache/airavata/data/manager/DataManagerImpl.java ---------------------------------------------------------------------- diff --git a/modules/data-manager/src/main/java/org/apache/airavata/data/manager/DataManagerImpl.java b/modules/data-manager/src/main/java/org/apache/airavata/data/manager/DataManagerImpl.java deleted file mode 100644 index a8b1ae6..0000000 --- a/modules/data-manager/src/main/java/org/apache/airavata/data/manager/DataManagerImpl.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * -*/ -package org.apache.airavata.data.manager; - -import org.apache.airavata.model.data.resource.DataResourceModel; -import org.apache.airavata.registry.core.experiment.catalog.impl.RegistryFactory; -import org.apache.airavata.registry.cpi.DataCatalog; -import org.apache.airavata.registry.cpi.DataCatalogException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class DataManagerImpl implements DataManager{ - private final static Logger logger = LoggerFactory.getLogger(DataManagerImpl.class); - - private DataCatalog dataCatalog; - - public DataManagerImpl() throws DataManagerException { - try { - this.dataCatalog = RegistryFactory.getDataCatalog(); - } catch (DataCatalogException e) { - logger.error(e.getMessage(), e); - throw new DataManagerException(e); - } - } - - @Override - public String publishDataResource(DataResourceModel resourceModel) throws DataManagerException{ - try { - String resourceId = dataCatalog.publishResource(resourceModel); - return resourceId; - } catch (DataCatalogException e) { - logger.error(e.getMessage(), e); - throw new DataManagerException(e); - } - } - - @Override - public boolean removeDataResource(String resourceId) throws DataManagerException { - try { - boolean result = dataCatalog.removeResource(resourceId); - return result; - } catch (DataCatalogException e) { - logger.error(e.getMessage(), e); - throw new DataManagerException(e); - } - } - - @Override - public boolean updateDataResource(DataResourceModel dataResourceModel) throws DataManagerException { - try { - boolean result = dataCatalog.updateResource(dataResourceModel); - return result; - } catch (DataCatalogException e) { - logger.error(e.getMessage(), e); - throw new DataManagerException(e); - } - } - - @Override - public DataResourceModel getDataResource(String resourceId) throws DataManagerException { - try { - DataResourceModel dataResource = dataCatalog.getResource(resourceId); - return dataResource; - } catch (DataCatalogException e) { - logger.error(e.getMessage(), e); - throw new DataManagerException(e); - } - } - - @Override - public boolean copyDataResource(String resourceId, String destLocation) throws DataManagerException{ - return false; - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata/blob/80d20278/modules/data-manager/src/test/java/org/apache/airavata/data/manager/DataManagerFactoryTest.java ---------------------------------------------------------------------- diff --git a/modules/data-manager/src/test/java/org/apache/airavata/data/manager/DataManagerFactoryTest.java b/modules/data-manager/src/test/java/org/apache/airavata/data/manager/DataManagerFactoryTest.java deleted file mode 100644 index aa10724..0000000 --- a/modules/data-manager/src/test/java/org/apache/airavata/data/manager/DataManagerFactoryTest.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * -*/ -package org.apache.airavata.data.manager; - -import junit.framework.Assert; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class DataManagerFactoryTest { - private final static Logger logger = LoggerFactory.getLogger(DataManagerFactoryTest.class); - - @Test - public void testCreateDataManager(){ - DataManager dataManager = DataManagerFactory.getDataManager(); - Assert.assertNotNull(dataManager); - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata/blob/80d20278/modules/data-manager/src/test/java/org/apache/airavata/data/manager/DataManagerImptTest.java ---------------------------------------------------------------------- diff --git a/modules/data-manager/src/test/java/org/apache/airavata/data/manager/DataManagerImptTest.java b/modules/data-manager/src/test/java/org/apache/airavata/data/manager/DataManagerImptTest.java deleted file mode 100644 index b6bc751..0000000 --- a/modules/data-manager/src/test/java/org/apache/airavata/data/manager/DataManagerImptTest.java +++ /dev/null @@ -1,141 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * -*/ -package org.apache.airavata.data.manager; - -import org.apache.airavata.data.manager.utils.AppCatInit; -import org.apache.airavata.data.manager.utils.DataCatInit; -import org.apache.airavata.model.data.resource.DataReplicaLocationModel; -import org.apache.airavata.model.data.resource.DataResourceModel; -import org.apache.airavata.registry.core.experiment.catalog.impl.RegistryFactory; -import org.apache.airavata.registry.cpi.DataCatalog; -import org.apache.airavata.registry.cpi.DataCatalogException; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.List; -import java.util.UUID; - -public class DataManagerImptTest { - private final static Logger logger = LoggerFactory.getLogger(DataManagerImptTest.class); - private static AppCatInit appCatInit; - private static DataCatInit dataCatInit; - private static DataManager dataManager; - private static DataResourceModel dataResourceModel; - - @BeforeClass - public static void setUp() { - try { - System.out.println("********** SET UP ************"); - appCatInit = new AppCatInit("appcatalog-derby.sql"); - appCatInit.initializeDB(); - dataCatInit = new DataCatInit("datacatalog-derby.sql"); - dataCatInit.initializeDB(); - dataManager = DataManagerFactory.getDataManager(); - dataResourceModel = new DataResourceModel(); - dataResourceModel.setResourceName("test-file.txt"); - List<DataReplicaLocationModel> replicaLocationModelList = new ArrayList<>(); - DataReplicaLocationModel dataReplicaLocationModel = new DataReplicaLocationModel(); - dataReplicaLocationModel.setReplicaName("1-st-replica"); - ArrayList<String> dataLocations = new ArrayList<>(); - dataLocations.add("scp://g75.iu.xsede.org:/var/www/portal/experimentData/test-file.txt"); - dataReplicaLocationModel.setDataLocations(dataLocations); - replicaLocationModelList.add(dataReplicaLocationModel); - dataResourceModel.setReplicaLocations(replicaLocationModelList); - } catch (DataCatalogException e) { - logger.error(e.getMessage(), e); - } - } - - @AfterClass - public static void tearDown() throws Exception { - System.out.println("********** TEAR DOWN ************"); - appCatInit.stopDerbyServer(); - dataCatInit.stopDerbyServer(); - } - - @Test - public void testPublishDataResource(){ - try { - String resourceId = dataManager.publishDataResource(dataResourceModel); - org.junit.Assert.assertNotNull(resourceId); - } catch (DataManagerException e) { - e.printStackTrace(); - } - } - - @Test - public void testRemoveDataResource(){ - try { - boolean result = dataManager.removeDataResource("234234234"); - Assert.assertFalse(result); - String resourceId = dataManager.publishDataResource(dataResourceModel); - Assert.assertNotNull(resourceId); - result = dataManager.removeDataResource(resourceId); - Assert.assertTrue(result); - result = dataManager.removeDataResource(resourceId); - Assert.assertFalse(result); - } catch (DataManagerException e) { - e.printStackTrace(); - } - } - - @Test - public void testGetDataResource(){ - try { - String resourceId = dataManager.publishDataResource(dataResourceModel); - Assert.assertNotNull(resourceId); - DataResourceModel persistedCopy = dataManager.getDataResource(resourceId); - Assert.assertNotNull(persistedCopy); - Assert.assertTrue(persistedCopy.getReplicaLocations().size()==1); - } catch (DataManagerException e) { - e.printStackTrace(); - } - } - - @Test - public void testUpdateDataResource(){ - try { - dataResourceModel.setResourceId(UUID.randomUUID().toString()); - boolean result = dataManager.updateDataResource(dataResourceModel); - Assert.assertFalse(result); - dataManager.publishDataResource(dataResourceModel); - DataReplicaLocationModel dataReplicaLocationModel = new DataReplicaLocationModel(); - dataReplicaLocationModel.setReplicaName("2-nd-replica"); - ArrayList<String> dataLocations = new ArrayList<>(); - dataLocations.add("scp://g175.iu.xsede.org:/var/www/portal/experimentData/test-file.txt"); - dataResourceModel.getReplicaLocations().add(dataReplicaLocationModel); - dataManager.updateDataResource(dataResourceModel); - dataResourceModel = dataManager.getDataResource(dataResourceModel.getResourceId()); - Assert.assertTrue(dataResourceModel.getReplicaLocations().size()==2); - dataResourceModel.getReplicaLocations().remove(1); - dataManager.updateDataResource(dataResourceModel); - dataResourceModel = dataManager.getDataResource(dataResourceModel.getResourceId()); - Assert.assertTrue(dataResourceModel.getReplicaLocations().size()==1); - } catch (DataManagerException e) { - e.printStackTrace(); - } - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/airavata/blob/80d20278/modules/data-manager/src/test/java/org/apache/airavata/data/manager/utils/AppCatInit.java ---------------------------------------------------------------------- diff --git a/modules/data-manager/src/test/java/org/apache/airavata/data/manager/utils/AppCatInit.java b/modules/data-manager/src/test/java/org/apache/airavata/data/manager/utils/AppCatInit.java deleted file mode 100644 index 2433b78..0000000 --- a/modules/data-manager/src/test/java/org/apache/airavata/data/manager/utils/AppCatInit.java +++ /dev/null @@ -1,320 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - */ - -package org.apache.airavata.data.manager.utils; - -import org.apache.airavata.common.exception.ApplicationSettingsException; -import org.apache.airavata.common.utils.ServerSettings; -import org.apache.derby.drda.NetworkServerControl; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.net.InetAddress; -import java.net.URI; -import java.sql.*; -import java.util.StringTokenizer; - -public class AppCatInit { - private static final Logger logger = LoggerFactory.getLogger(AppCatInit.class); - public static final String DERBY_SERVER_MODE_SYS_PROPERTY = "derby.drda.startNetworkServer"; - public String scriptName = "appcatalog-derby.sql"; - private NetworkServerControl server; - private static final String delimiter = ";"; - public static final String COMPUTE_RESOURCE_TABLE = "COMPUTE_RESOURCE"; - private String jdbcUrl = null; - private String jdbcDriver = null; - private String jdbcUser = null; - private String jdbcPassword = null; - - public AppCatInit(String scriptName) { - this.scriptName = scriptName; - } - - public static boolean checkStringBufferEndsWith(StringBuffer buffer, String suffix) { - if (suffix.length() > buffer.length()) { - return false; - } - // this loop is done on purpose to avoid memory allocation performance - // problems on various JDKs - // StringBuffer.lastIndexOf() was introduced in jdk 1.4 and - // implementation is ok though does allocation/copying - // StringBuffer.toString().endsWith() does massive memory - // allocation/copying on JDK 1.5 - // See http://issues.apache.org/bugzilla/show_bug.cgi?id=37169 - int endIndex = suffix.length() - 1; - int bufferIndex = buffer.length() - 1; - while (endIndex >= 0) { - if (buffer.charAt(bufferIndex) != suffix.charAt(endIndex)) { - return false; - } - bufferIndex--; - endIndex--; - } - return true; - } - - private static boolean isServerStarted(NetworkServerControl server, int ntries) - { - for (int i = 1; i <= ntries; i ++) - { - try { - Thread.sleep(500); - server.ping(); - return true; - } - catch (Exception e) { - if (i == ntries) - return false; - } - } - return false; - } - - public void initializeDB() { - - try{ - jdbcDriver = ServerSettings.getSetting("appcatalog.jdbc.driver"); - jdbcUrl = ServerSettings.getSetting("appcatalog.jdbc.url"); - jdbcUser = ServerSettings.getSetting("appcatalog.jdbc.user"); - jdbcPassword = ServerSettings.getSetting("appcatalog.jdbc.password"); - jdbcUrl = jdbcUrl + "?" + "user=" + jdbcUser + "&" + "password=" + jdbcPassword; - } catch (ApplicationSettingsException e) { - logger.error("Unable to read properties", e); - } - - startDerbyInServerMode(); - if(!isServerStarted(server, 20)){ - throw new RuntimeException("Derby server cound not started within five seconds..."); - } -// startDerbyInEmbeddedMode(); - - Connection conn = null; - try { - Class.forName(jdbcDriver).newInstance(); - conn = DriverManager.getConnection(jdbcUrl, jdbcUser, jdbcPassword); - if (!isDatabaseStructureCreated(COMPUTE_RESOURCE_TABLE, conn)) { - executeSQLScript(conn); - logger.info("New Database created for App Catalog !!!"); - } else { - logger.debug("Database already created for App Catalog!"); - } - } catch (Exception e) { - logger.error(e.getMessage(), e); - throw new RuntimeException("Database failure", e); - } finally { - try { - if (conn != null){ - if (!conn.getAutoCommit()) { - conn.commit(); - } - conn.close(); - } - } catch (SQLException e) { - logger.error(e.getMessage(), e); - } - } - } - - public static boolean isDatabaseStructureCreated(String tableName, Connection conn) { - try { - System.out.println("Running a query to test the database tables existence."); - // check whether the tables are already created with a query - Statement statement = null; - try { - statement = conn.createStatement(); - ResultSet rs = statement.executeQuery("select * from " + tableName); - if (rs != null) { - rs.close(); - } - } finally { - try { - if (statement != null) { - statement.close(); - } - } catch (SQLException e) { - return false; - } - } - } catch (SQLException e) { - return false; - } - - return true; - } - - private void executeSQLScript(Connection conn) throws Exception { - StringBuffer sql = new StringBuffer(); - BufferedReader reader = null; - try{ - - InputStream inputStream = this.getClass().getClassLoader().getResourceAsStream(scriptName); - reader = new BufferedReader(new InputStreamReader(inputStream)); - String line; - while ((line = reader.readLine()) != null) { - line = line.trim(); - if (line.startsWith("//")) { - continue; - } - if (line.startsWith("--")) { - continue; - } - StringTokenizer st = new StringTokenizer(line); - if (st.hasMoreTokens()) { - String token = st.nextToken(); - if ("REM".equalsIgnoreCase(token)) { - continue; - } - } - sql.append(" ").append(line); - - // SQL defines "--" as a comment to EOL - // and in Oracle it may contain a hint - // so we cannot just remove it, instead we must end it - if (line.indexOf("--") >= 0) { - sql.append("\n"); - } - if ((checkStringBufferEndsWith(sql, delimiter))) { - executeSQL(sql.substring(0, sql.length() - delimiter.length()), conn); - sql.replace(0, sql.length(), ""); - } - } - // Catch any statements not followed by ; - if (sql.length() > 0) { - executeSQL(sql.toString(), conn); - } - }catch (IOException e){ - logger.error("Error occurred while executing SQL script for creating Airavata database", e); - throw new Exception("Error occurred while executing SQL script for creating Airavata database", e); - }finally { - if (reader != null) { - reader.close(); - } - - } - - } - - private static void executeSQL(String sql, Connection conn) throws Exception { - // Check and ignore empty statements - if ("".equals(sql.trim())) { - return; - } - - Statement statement = null; - try { - logger.debug("SQL : " + sql); - - boolean ret; - int updateCount = 0, updateCountTotal = 0; - statement = conn.createStatement(); - ret = statement.execute(sql); - updateCount = statement.getUpdateCount(); - do { - if (!ret) { - if (updateCount != -1) { - updateCountTotal += updateCount; - } - } - ret = statement.getMoreResults(); - if (ret) { - updateCount = statement.getUpdateCount(); - } - } while (ret); - - logger.debug(sql + " : " + updateCountTotal + " rows affected"); - - SQLWarning warning = conn.getWarnings(); - while (warning != null) { - logger.warn(warning + " sql warning"); - warning = warning.getNextWarning(); - } - conn.clearWarnings(); - } catch (SQLException e) { - if (e.getSQLState().equals("X0Y32")) { - // eliminating the table already exception for the derby - // database - logger.info("Table Already Exists", e); - } else { - throw new Exception("Error occurred while executing : " + sql, e); - } - } finally { - if (statement != null) { - try { - statement.close(); - } catch (SQLException e) { - logger.error("Error occurred while closing result set.", e); - } - } - } - } - - private void startDerbyInServerMode() { - try { - System.setProperty(DERBY_SERVER_MODE_SYS_PROPERTY, "true"); - server = new NetworkServerControl(InetAddress.getByName("0.0.0.0"), - 20000, - jdbcUser, jdbcPassword); - java.io.PrintWriter consoleWriter = new java.io.PrintWriter(System.out, true); - server.start(consoleWriter); - } catch (IOException e) { - logger.error("Unable to start Apache derby in the server mode! Check whether " + - "specified port is available"); - } catch (Exception e) { - logger.error("Unable to start Apache derby in the server mode! Check whether " + - "specified port is available"); - } - - } - - public static int getPort(String jdbcURL){ - try{ - String cleanURI = jdbcURL.substring(5); - URI uri = URI.create(cleanURI); - return uri.getPort(); - } catch (Exception e) { - logger.error(e.getMessage(), e); - return -1; - } - } - - private void startDerbyInEmbeddedMode(){ - try { - Class.forName("org.apache.derby.jdbc.EmbeddedDriver"); - DriverManager.getConnection("jdbc:derby:memory:unit-testing-jpa;create=true").close(); - } catch (ClassNotFoundException e) { - logger.error(e.getMessage(), e); - } catch (SQLException e) { - logger.error(e.getMessage(), e); - } - } - - public void stopDerbyServer() { - try { - server.shutdown(); - } catch (Exception e) { - logger.error(e.getMessage(), e); - } - } -} http://git-wip-us.apache.org/repos/asf/airavata/blob/80d20278/modules/data-manager/src/test/java/org/apache/airavata/data/manager/utils/DataCatInit.java ---------------------------------------------------------------------- diff --git a/modules/data-manager/src/test/java/org/apache/airavata/data/manager/utils/DataCatInit.java b/modules/data-manager/src/test/java/org/apache/airavata/data/manager/utils/DataCatInit.java deleted file mode 100644 index 05165a2..0000000 --- a/modules/data-manager/src/test/java/org/apache/airavata/data/manager/utils/DataCatInit.java +++ /dev/null @@ -1,315 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - */ - -package org.apache.airavata.data.manager.utils; - -import org.apache.airavata.common.exception.ApplicationSettingsException; -import org.apache.airavata.common.utils.ServerSettings; -import org.apache.airavata.registry.core.data.catalog.utils.DataCatalogConstants; -import org.apache.derby.drda.NetworkServerControl; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.net.InetAddress; -import java.net.URI; -import java.sql.*; -import java.util.StringTokenizer; - -public class DataCatInit { - private static final Logger logger = LoggerFactory.getLogger(DataCatInit.class); - public static final String DERBY_SERVER_MODE_SYS_PROPERTY = "derby.drda.startNetworkServer"; - public String scriptName = "datacatalog-derby.sql"; - private NetworkServerControl server; - private static final String delimiter = ";"; - private String jdbcUrl = null; - private String jdbcDriver = null; - private String jdbcUser = null; - private String jdbcPassword = null; - - public DataCatInit(String scriptName) { - this.scriptName = scriptName; - } - - public static boolean checkStringBufferEndsWith(StringBuffer buffer, String suffix) { - if (suffix.length() > buffer.length()) { - return false; - } - // this loop is done on purpose to avoid memory allocation performance - // problems on various JDKs - // StringBuffer.lastIndexOf() was introduced in jdk 1.4 and - // implementation is ok though does allocation/copying - // StringBuffer.toString().endsWith() does massive memory - // allocation/copying on JDK 1.5 - // See http://issues.apache.org/bugzilla/show_bug.cgi?id=37169 - int endIndex = suffix.length() - 1; - int bufferIndex = buffer.length() - 1; - while (endIndex >= 0) { - if (buffer.charAt(bufferIndex) != suffix.charAt(endIndex)) { - return false; - } - bufferIndex--; - endIndex--; - } - return true; - } - - private static boolean isServerStarted(NetworkServerControl server, int ntries) - { - for (int i = 1; i <= ntries; i ++) - { - try { - Thread.sleep(500); - server.ping(); - return true; - } - catch (Exception e) { - if (i == ntries) - return false; - } - } - return false; - } - - public void initializeDB() { - try{ - jdbcDriver = ServerSettings.getSetting("datacatalog.jdbc.driver"); - jdbcUrl = ServerSettings.getSetting("datacatalog.jdbc.url"); - jdbcUser = ServerSettings.getSetting("datacatalog.jdbc.user"); - jdbcPassword = ServerSettings.getSetting("datacatalog.jdbc.password"); - jdbcUrl = jdbcUrl + "?" + "user=" + jdbcUser + "&" + "password=" + jdbcPassword; - } catch (ApplicationSettingsException e) { - logger.error("Unable to read properties", e); - } - - startDerbyInServerMode(); - if(!isServerStarted(server, 20)){ - throw new RuntimeException("Derby server could not started within five seconds..."); - } - Connection conn = null; - try { - Class.forName(jdbcDriver).newInstance(); - conn = DriverManager.getConnection(jdbcUrl, jdbcUser, jdbcPassword); - if (!isDatabaseStructureCreated(DataCatalogConstants.CONFIGURATION, conn)) { - executeSQLScript(conn); - logger.info("New Database created for Data Catalog !!!"); - } else { - logger.debug("Database already created for Data Catalog!"); - } - } catch (Exception e) { - logger.error(e.getMessage(), e); - throw new RuntimeException("Database failure", e); - } finally { - try { - if (conn != null){ - if (!conn.getAutoCommit()) { - conn.commit(); - } - conn.close(); - } - } catch (SQLException e) { - logger.error(e.getMessage(), e); - } - } - } - - public static boolean isDatabaseStructureCreated(String tableName, Connection conn) { - try { - System.out.println("Running a query to test the database tables existence."); - // check whether the tables are already created with a query - Statement statement = null; - try { - statement = conn.createStatement(); - ResultSet rs = statement.executeQuery("select * from " + tableName); - if (rs != null) { - rs.close(); - } - } finally { - try { - if (statement != null) { - statement.close(); - } - } catch (SQLException e) { - return false; - } - } - } catch (SQLException e) { - return false; - } - - return true; - } - - private void executeSQLScript(Connection conn) throws Exception { - StringBuffer sql = new StringBuffer(); - BufferedReader reader = null; - try{ - - InputStream inputStream = this.getClass().getClassLoader().getResourceAsStream(scriptName); - reader = new BufferedReader(new InputStreamReader(inputStream)); - String line; - while ((line = reader.readLine()) != null) { - line = line.trim(); - if (line.startsWith("//")) { - continue; - } - if (line.startsWith("--")) { - continue; - } - StringTokenizer st = new StringTokenizer(line); - if (st.hasMoreTokens()) { - String token = st.nextToken(); - if ("REM".equalsIgnoreCase(token)) { - continue; - } - } - sql.append(" ").append(line); - - // SQL defines "--" as a comment to EOL - // and in Oracle it may contain a hint - // so we cannot just remove it, instead we must end it - if (line.indexOf("--") >= 0) { - sql.append("\n"); - } - if ((checkStringBufferEndsWith(sql, delimiter))) { - executeSQL(sql.substring(0, sql.length() - delimiter.length()), conn); - sql.replace(0, sql.length(), ""); - } - } - // Catch any statements not followed by ; - if (sql.length() > 0) { - executeSQL(sql.toString(), conn); - } - }catch (IOException e){ - logger.error("Error occurred while executing SQL script for creating Airavata Data Catalog database", e); - throw new Exception("Error occurred while executing SQL script for creating Airavata Data Catalog database", e); - }finally { - if (reader != null) { - reader.close(); - } - } - } - - private static void executeSQL(String sql, Connection conn) throws Exception { - // Check and ignore empty statements - if ("".equals(sql.trim())) { - return; - } - - Statement statement = null; - try { - logger.debug("SQL : " + sql); - - boolean ret; - int updateCount = 0, updateCountTotal = 0; - statement = conn.createStatement(); - ret = statement.execute(sql); - updateCount = statement.getUpdateCount(); - do { - if (!ret) { - if (updateCount != -1) { - updateCountTotal += updateCount; - } - } - ret = statement.getMoreResults(); - if (ret) { - updateCount = statement.getUpdateCount(); - } - } while (ret); - - logger.debug(sql + " : " + updateCountTotal + " rows affected"); - - SQLWarning warning = conn.getWarnings(); - while (warning != null) { - logger.warn(warning + " sql warning"); - warning = warning.getNextWarning(); - } - conn.clearWarnings(); - } catch (SQLException e) { - if (e.getSQLState().equals("X0Y32")) { - // eliminating the table already exception for the derby - // database - logger.info("Table Already Exists", e); - } else { - throw new Exception("Error occurred while executing : " + sql, e); - } - } finally { - if (statement != null) { - try { - statement.close(); - } catch (SQLException e) { - logger.error("Error occurred while closing result set.", e); - } - } - } - } - - private void startDerbyInServerMode() { - try { - System.setProperty(DERBY_SERVER_MODE_SYS_PROPERTY, "true"); - server = new NetworkServerControl(InetAddress.getByName("0.0.0.0"), - 20000, - jdbcUser, jdbcPassword); - java.io.PrintWriter consoleWriter = new java.io.PrintWriter(System.out, true); - server.start(consoleWriter); - } catch (IOException e) { - logger.error("Unable to start Apache derby in the server mode! Check whether " + - "specified port is available"); - } catch (Exception e) { - logger.error("Unable to start Apache derby in the server mode! Check whether " + - "specified port is available"); - } - - } - - public static int getPort(String jdbcURL){ - try{ - String cleanURI = jdbcURL.substring(5); - URI uri = URI.create(cleanURI); - return uri.getPort(); - } catch (Exception e) { - logger.error(e.getMessage(), e); - return -1; - } - } - - private void startDerbyInEmbeddedMode(){ - try { - Class.forName("org.apache.derby.jdbc.EmbeddedDriver"); - DriverManager.getConnection("jdbc:derby:memory:unit-testing-jpa;create=true").close(); - } catch (ClassNotFoundException e) { - logger.error(e.getMessage(), e); - } catch (SQLException e) { - logger.error(e.getMessage(), e); - } - } - - public void stopDerbyServer() { - try { - server.shutdown(); - } catch (Exception e) { - logger.error(e.getMessage(), e); - } - } -} http://git-wip-us.apache.org/repos/asf/airavata/blob/80d20278/thrift-interface-descriptions/airavata-api/data_movement_models.thrift ---------------------------------------------------------------------- diff --git a/thrift-interface-descriptions/airavata-api/data_movement_models.thrift b/thrift-interface-descriptions/airavata-api/data_movement_models.thrift deleted file mode 100644 index 677d790..0000000 --- a/thrift-interface-descriptions/airavata-api/data_movement_models.thrift +++ /dev/null @@ -1,154 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - */ - -include "airavata_commons.thrift" - -namespace java org.apache.airavata.model.data.movement -namespace php Airavata.Model.Data.Movement -namespace cpp apache.airavata.model.data.movement -namespace py apache.airavata.model.data.movement - -enum DMType { - COMPUTE_RESOURCE, - STORAGE_RESOURCE -} - -/** - * Enumeration of security authentication and authorization mechanisms supported by Airavata. This enumeration just - * describes the supported mechanism. The corresponding security credentials are registered with Airavata Credential - * store. - * - * USERNAME_PASSWORD: - * A User Name. - * - * SSH_KEYS: - * SSH Keys - * - * FIXME: Change GSI to a more precise generic security protocol - X509 - * -*/ -enum SecurityProtocol { - USERNAME_PASSWORD, - SSH_KEYS, - GSI, - KERBEROS, - OAUTH, - LOCAL -} - - -/** - * Enumeration of data movement supported by Airavata - * - * SCP: - * Job manager supporting the Portal Batch System (PBS) protocol. Some examples include TORQUE, PBSPro, Grid Engine. - * - * SFTP: - * The Simple Linux Utility for Resource Management is a open source workload manager. - * - * GridFTP: - * Globus File Transfer Protocol - * - * UNICORE_STORAGE_SERVICE: - * Storage Service Provided by Unicore - * -*/ -enum DataMovementProtocol { - LOCAL, - SCP, - SFTP, - GridFTP, - UNICORE_STORAGE_SERVICE -} - -/** - * Data Movement through Secured Copy - * - * alternativeSCPHostName: - * If the login to scp is different than the hostname itself, specify it here - * - * sshPort: - * If a non-default port needs to used, specify it. -*/ -struct SCPDataMovement { - 1: required string dataMovementInterfaceId = airavata_commons.DEFAULT_ID, - 2: required SecurityProtocol securityProtocol, - 3: optional string alternativeSCPHostName, - 4: optional i32 sshPort = 22 -} - -/** - * Data Movement through GridFTP - * - * alternativeSCPHostName: - * If the login to scp is different than the hostname itself, specify it here - * - * sshPort: - * If a non-default port needs to used, specify it. -*/ -struct GridFTPDataMovement { - 1: required string dataMovementInterfaceId = airavata_commons.DEFAULT_ID, - 2: required SecurityProtocol securityProtocol, - 3: required list<string> gridFTPEndPoints -} - -/** - * Data Movement through UnicoreStorage - * - * unicoreEndPointURL: - * unicoreGateway End Point. The provider will query this service to fetch required service end points. -*/ -struct UnicoreDataMovement { - 1: required string dataMovementInterfaceId = airavata_commons.DEFAULT_ID, - 2: required SecurityProtocol securityProtocol, - 3: required string unicoreEndPointURL -} - -/** - * LOCAL - * - * alternativeSCPHostName: - * If the login to scp is different than the hostname itself, specify it here - * - * sshPort: - * If a non-defualt port needs to used, specify it. -*/ -struct LOCALDataMovement { - 1: required string dataMovementInterfaceId = airavata_commons.DEFAULT_ID, -} - -/** - * Data Movement Interfaces - * - * dataMovementInterfaceId: The Data Movement Interface has to be previously registered and referenced here. - * - * priorityOrder: - * For resources with multiple interfaces, the priority order should be selected. - * Lower the numerical number, higher the priority - * -*/ -struct DataMovementInterface { - 1: required string dataMovementInterfaceId, - 2: required DataMovementProtocol dataMovementProtocol, - 3: required i32 priorityOrder = 0, -} - - - http://git-wip-us.apache.org/repos/asf/airavata/blob/80d20278/thrift-interface-descriptions/airavata-api/data_resource_models.thrift ---------------------------------------------------------------------- diff --git a/thrift-interface-descriptions/airavata-api/data_resource_models.thrift b/thrift-interface-descriptions/airavata-api/data_resource_models.thrift deleted file mode 100644 index 4130eca..0000000 --- a/thrift-interface-descriptions/airavata-api/data_resource_models.thrift +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - */ - - namespace java org.apache.airavata.model.data.resource - namespace php Airavata.Model.Data.Resource - namespace cpp apache.airavata.model.data.resource - namespace py apache.airavata.model.data.resource - -struct DataResourceModel { - 1: optional string resourceId, - 2: optional string resourceName, - 3: optional string resourceDescription, - 4: optional i32 resourceSize, - 5: optional i64 creationTime, - 6: optional i64 lastModifiedTime, - 7: list<DataReplicaLocationModel> replicaLocations -} - -struct DataReplicaLocationModel { - 1: optional string replicaId, - 2: optional string resourceId, - 3: optional string replicaName, - 4: optional string replicaDescription, - 5: optional i64 creationTime, - 6: optional i64 lastModifiedTime, - 7: optional list<string> dataLocations -} -
