Repository: falcon Updated Branches: refs/heads/master 9979a1fd8 -> ad18b024f
http://git-wip-us.apache.org/repos/asf/falcon/blob/ad18b024/addons/adf/src/main/java/org/apache/falcon/adfservice/util/ADFJsonConstants.java ---------------------------------------------------------------------- diff --git a/addons/adf/src/main/java/org/apache/falcon/adfservice/util/ADFJsonConstants.java b/addons/adf/src/main/java/org/apache/falcon/adfservice/util/ADFJsonConstants.java new file mode 100644 index 0000000..9e48685 --- /dev/null +++ b/addons/adf/src/main/java/org/apache/falcon/adfservice/util/ADFJsonConstants.java @@ -0,0 +1,73 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.falcon.adfservice.util; + +/** + * ADF JSON Constants in ADF request. + */ +public final class ADFJsonConstants { + + public static final String ADF_REQUEST_ACTIVITY = "activity"; + public static final String ADF_REQUEST_TRANSFORMATION = "transformation"; + public static final String ADF_REQUEST_TYPE = "type"; + public static final String ADF_REQUEST_JOBID = "jobId"; + public static final String ADF_REQUEST_START_TIME = "dataSliceStart"; + public static final String ADF_REQUEST_END_TIME = "dataSliceEnd"; + public static final String ADF_REQUEST_SCHEDULER = "scheduler"; + public static final String ADF_REQUEST_POLICY = "policy"; + public static final String ADF_REQUEST_TIMEOUT = "timeout"; + public static final String ADF_REQUEST_FREQUENCY = "frequency"; + public static final String ADF_REQUEST_INTERVAL = "interval"; + public static final String ADF_REQUEST_LINKED_SERVICES = "linkedServices"; + public static final String ADF_REQUEST_NAME = "name"; + public static final String ADF_REQUEST_INPUTS = "inputs"; + public static final String ADF_REQUEST_OUTPUTS = "outputs"; + public static final String ADF_REQUEST_TABLES = "tables"; + public static final String ADF_REQUEST_PROPERTIES = "properties"; + public static final String ADF_REQUEST_EXTENDED_PROPERTIES = "extendedProperties"; + public static final String ADF_REQUEST_CLUSTER_NAME = "clusterName"; + public static final String ADF_REQUEST_RUN_ON_BEHALF_USER = "runOnBehalf"; + public static final String ADF_REQUEST_LOCATION = "location"; + public static final String ADF_REQUEST_FOLDER_PATH = "folderPath"; + public static final String ADF_REQUEST_SCRIPT = "script"; + public static final String ADF_REQUEST_SCRIPT_PATH = "scriptPath"; + public static final String ADF_REQUEST_LINKED_SERVICE_NAME = "linkedServiceName"; + public static final String ADF_REQUEST_TABLE_NAME = "tableName"; + public static final String ADF_REQUEST_TABLE_PARTITION = "partitionedBy"; + public static final String ADF_REQUEST_LOCATION_TYPE_AZURE_BLOB = "AzureBlobLocation"; + public static final String ADF_REQUEST_CONNECTION_STRING = "connectionString"; + public static final String ADF_REQUEST_BLOB_ACCOUNT_NAME = "AccountName="; + + public static final String ADF_STATUS_PROTOCOL = "TransportProtocolVersion"; + public static final String ADF_STATUS_JOBID = "JobId"; + public static final String ADF_STATUS_STATUS = "Status"; + public static final String ADF_STATUS_PROGRESS = "Progress"; + public static final String ADF_STATUS_LOG_URL = "LogURL"; + public static final String ADF_STATUS_ERROR_TYPE = "ErrorType"; + public static final String ADF_STATUS_ERROR_MESSAGE = "ErrorMessage"; + public static final String ADF_STATUS_PROTOCOL_NAME = "2.0-preview"; + public static final String ADF_STATUS_ERROR_TYPE_VALUE = "UserError"; + public static final String ADF_STATUS_SUCCEEDED = "Succeeded"; + public static final String ADF_STATUS_FAILED = "Failed"; + public static final String ADF_STATUS_EXECUTING = "Executing"; + public static final String ADF_STATUS_CANCELED = "Canceled"; + + private ADFJsonConstants() { + } +} http://git-wip-us.apache.org/repos/asf/falcon/blob/ad18b024/addons/adf/src/main/java/org/apache/falcon/adfservice/util/FSUtils.java ---------------------------------------------------------------------- diff --git a/addons/adf/src/main/java/org/apache/falcon/adfservice/util/FSUtils.java b/addons/adf/src/main/java/org/apache/falcon/adfservice/util/FSUtils.java new file mode 100644 index 0000000..58abfbf --- /dev/null +++ b/addons/adf/src/main/java/org/apache/falcon/adfservice/util/FSUtils.java @@ -0,0 +1,102 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.falcon.adfservice.util; + +import org.apache.commons.io.IOUtils; +import org.apache.falcon.FalconException; +import org.apache.falcon.hadoop.HadoopClientFactory; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.net.URISyntaxException; + +/** + * Utility for file operations. + */ +public final class FSUtils { + private static final Logger LOG = LoggerFactory.getLogger(FSUtils.class); + private FSUtils() { + } + + public static String readHDFSFile(final String filePath, final String fileName) + throws URISyntaxException, FalconException { + BufferedReader br = null; + try { + Path path = new Path(filePath, fileName); + FileSystem fs = HadoopClientFactory.get().createProxiedFileSystem(path.toUri()); + br = new BufferedReader(new InputStreamReader(fs.open(path))); + StringBuilder fileContent = new StringBuilder(); + String line; + while (true) { + line = br.readLine(); + if (line == null) { + break; + } + fileContent.append(line); + } + return fileContent.toString(); + } catch (IOException e) { + throw new FalconException("Error reading file from hdfs: " + filePath + fileName + ": " + e.toString(), e); + } finally { + IOUtils.closeQuietly(br); + } + } + + public static String createFile(final Path path, + final String content) throws FalconException { + OutputStream out = null; + try { + FileSystem fs = HadoopClientFactory.get().createProxiedFileSystem(path.toUri()); + out = fs.create(path); + out.write(content.getBytes()); + } catch (IOException e) { + throw new FalconException("Error preparing script file: " + path, e); + } finally { + IOUtils.closeQuietly(out); + } + return path.toString(); + } + + public static void createDir(final Path dirPath) throws FalconException { + FileSystem fs = HadoopClientFactory.get().createProxiedFileSystem(dirPath.toUri()); + try { + if (!fs.exists(dirPath)) { + LOG.info("Creating directory: {}", dirPath); + HadoopClientFactory.mkdirsWithDefaultPerms(fs, dirPath); + } + } catch (IOException e) { + throw new FalconException("Error creating directory: " + dirPath, e); + } + } + + public static void removeDir(final Path dirPath) throws FalconException { + FileSystem fs = HadoopClientFactory.get().createProxiedFileSystem(dirPath.toUri()); + try { + fs.delete(dirPath, true); + } catch (IOException e) { + throw new FalconException("Error creating directory: " + dirPath, e); + } + } +} http://git-wip-us.apache.org/repos/asf/falcon/blob/ad18b024/addons/hivedr/pom.xml ---------------------------------------------------------------------- diff --git a/addons/hivedr/pom.xml b/addons/hivedr/pom.xml index a4d6be8..37dc5c9 100644 --- a/addons/hivedr/pom.xml +++ b/addons/hivedr/pom.xml @@ -64,7 +64,12 @@ <groupId>org.apache.hive</groupId> <artifactId>hive-jdbc</artifactId> <version>${hive.version}</version> - <classifier>standalone</classifier> + <exclusions> + <exclusion> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpclient</artifactId> + </exclusion> + </exclusions> </dependency> <dependency> <groupId>org.testng</groupId> http://git-wip-us.apache.org/repos/asf/falcon/blob/ad18b024/common/src/main/resources/startup.properties ---------------------------------------------------------------------- diff --git a/common/src/main/resources/startup.properties b/common/src/main/resources/startup.properties index 3f1ed03..2497cce 100644 --- a/common/src/main/resources/startup.properties +++ b/common/src/main/resources/startup.properties @@ -42,7 +42,8 @@ org.apache.falcon.metadata.MetadataMappingService,\ org.apache.falcon.service.LogCleanupService,\ org.apache.falcon.service.GroupsService,\ - org.apache.falcon.service.ProxyUserService + org.apache.falcon.service.ProxyUserService,\ + org.apache.falcon.adfservice.ADFProviderService ## If you wish to use Falcon native scheduler add the commented out services below to application.services ## # org.apache.falcon.notification.service.impl.JobCompletionService,\ # org.apache.falcon.notification.service.impl.SchedulerService,\ @@ -234,6 +235,33 @@ it.workflow.execution.listeners=org.apache.falcon.catalog.CatalogPartitionHandle ######### Authorization Properties ######### +######### ADF Configurations start ######### + +# A String object that represents the namespace +*.microsoft.windowsazure.services.servicebus.namespace= + +# Request and status queues on the namespace +*.microsoft.windowsazure.services.servicebus.requestqueuename= +*.microsoft.windowsazure.services.servicebus.statusqueuename= + +# A String object that contains the SAS key name +*.microsoft.windowsazure.services.servicebus.sasKeyName= + +# A String object that contains the SAS key +*.microsoft.windowsazure.services.servicebus.sasKey= + +# A String object containing the base URI that is added to your Service Bus namespace to form the URI to connect +# to the Service Bus service. To access the default public Azure service, pass ".servicebus.windows.net" +*.microsoft.windowsazure.services.servicebus.serviceBusRootUri= + +# Service bus polling frequency +*.microsoft.windowsazure.services.servicebus.polling.frequency= + +# Super user +*.microsoft.windowsazure.services.servicebus.superuser= + +######### ADF Configurations end ########### + ######### SMTP Properties ######## # Setting SMTP hostname http://git-wip-us.apache.org/repos/asf/falcon/blob/ad18b024/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index fbf5913..27d2fc2 100644 --- a/pom.xml +++ b/pom.xml @@ -402,6 +402,12 @@ <module>addons/hivedr</module> </modules> </profile> + <profile> + <id>adf</id> + <modules> + <module>addons/adf</module> + </modules> + </profile> </profiles> <modules> @@ -539,13 +545,13 @@ <dependency> <groupId>org.codehaus.jackson</groupId> <artifactId>jackson-core-asl</artifactId> - <version>1.5.2</version> + <version>1.9.2</version> </dependency> <dependency> <groupId>org.codehaus.jackson</groupId> <artifactId>jackson-mapper-asl</artifactId> - <version>1.5.2</version> + <version>1.9.2</version> </dependency> <dependency> @@ -965,6 +971,10 @@ <groupId>org.apache.derby</groupId> <artifactId>derby</artifactId> </exclusion> + <exclusion> <!-- conflict with hadoop-auth --> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpclient</artifactId> + </exclusion> </exclusions> </dependency> @@ -977,6 +987,10 @@ <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-client</artifactId> </exclusion> + <exclusion> <!-- conflict with hadoop-auth --> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpclient</artifactId> + </exclusion> </exclusions> </dependency> @@ -1034,6 +1048,18 @@ </dependency> <dependency> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpclient</artifactId> + <version>4.3.3</version> + </dependency> + + <dependency> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpcore</artifactId> + <version>4.3.2</version> + </dependency> + + <dependency> <groupId>com.tinkerpop.blueprints</groupId> <artifactId>blueprints-core</artifactId> <version>2.5.0</version> http://git-wip-us.apache.org/repos/asf/falcon/blob/ad18b024/src/conf/startup.properties ---------------------------------------------------------------------- diff --git a/src/conf/startup.properties b/src/conf/startup.properties index b1a340a..faaedfa 100644 --- a/src/conf/startup.properties +++ b/src/conf/startup.properties @@ -50,7 +50,8 @@ org.apache.falcon.metadata.MetadataMappingService,\ org.apache.falcon.service.LogCleanupService,\ org.apache.falcon.service.GroupsService,\ - org.apache.falcon.service.ProxyUserService + org.apache.falcon.service.ProxyUserService,\ + org.apache.falcon.adfservice.ADFProviderService ## If you wish to use Falcon native scheduler uncomment out below application services and comment out above application services ## #*.application.services=org.apache.falcon.security.AuthenticationInitializationService,\ # org.apache.falcon.workflow.WorkflowJobEndNotificationService, \ @@ -251,6 +252,33 @@ prism.configstore.listeners=org.apache.falcon.entity.v0.EntityGraph,\ ######### Authorization Properties ######### +######### ADF Configurations start ######### + +# A String object that represents the namespace +*.microsoft.windowsazure.services.servicebus.namespace= + +# Request and status queues on the namespace +*.microsoft.windowsazure.services.servicebus.requestqueuename= +*.microsoft.windowsazure.services.servicebus.statusqueuename= + +# A String object that contains the SAS key name +*.microsoft.windowsazure.services.servicebus.sasKeyName= + +# A String object that contains the SAS key +*.microsoft.windowsazure.services.servicebus.sasKey= + +# A String object containing the base URI that is added to your Service Bus namespace to form the URI to connect +# to the Service Bus service. To access the default public Azure service, pass ".servicebus.windows.net" +*.microsoft.windowsazure.services.servicebus.serviceBusRootUri= + +# Service bus polling frequency +*.microsoft.windowsazure.services.servicebus.polling.frequency= + +# Super user +*.microsoft.windowsazure.services.servicebus.superuser= + +######### ADF Configurations end ########### + ######### SMTP Properties ######## # Setting SMTP hostname http://git-wip-us.apache.org/repos/asf/falcon/blob/ad18b024/webapp/pom.xml ---------------------------------------------------------------------- diff --git a/webapp/pom.xml b/webapp/pom.xml index 3996966..06aa897 100644 --- a/webapp/pom.xml +++ b/webapp/pom.xml @@ -71,6 +71,10 @@ <artifactId>hadoop-client</artifactId> <scope>test</scope> </dependency> + <dependency> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpclient</artifactId> + </dependency> </dependencies> </profile> <profile> @@ -83,6 +87,16 @@ </dependency> </dependencies> </profile> + <profile> + <id>adf</id> + <dependencies> + <dependency> + <groupId>org.apache.falcon</groupId> + <artifactId>falcon-adf</artifactId> + <version>${project.version}</version> + </dependency> + </dependencies> + </profile> </profiles> <dependencies>
