http://git-wip-us.apache.org/repos/asf/falcon/blob/4612c3f7/trunk/releases/0.10/src/site/twiki/restapi/EntityResume.twiki ---------------------------------------------------------------------- diff --git a/trunk/releases/0.10/src/site/twiki/restapi/EntityResume.twiki b/trunk/releases/0.10/src/site/twiki/restapi/EntityResume.twiki new file mode 100644 index 0000000..d0bbe41 --- /dev/null +++ b/trunk/releases/0.10/src/site/twiki/restapi/EntityResume.twiki @@ -0,0 +1,30 @@ +---++ POST /api/entities/resume/:entity-type/:entity-name + * <a href="#Description">Description</a> + * <a href="#Parameters">Parameters</a> + * <a href="#Results">Results</a> + * <a href="#Examples">Examples</a> + +---++ Description +Resume a supended entity. + +---++ Parameters + * :entity-type can either be a feed or a process. + * :entity-name is name of the entity. + * doAs <optional query param> allows the current user to impersonate the user passed in doAs when interacting with the Falcon system. + +---++ Results +Result of the resume command. + +---++ Examples +---+++ Rest Call +<verbatim> +POST http://localhost:15000/api/entities/resume/process/SampleProcess?doAs=joe +</verbatim> +---+++ Result +<verbatim> +{ + "requestId": "default\/106582a9-130f-4903-8b8f-f95d7b286c30\n", + "message": "default\/SampleProcess(process) resumed successfully\n", + "status": "SUCCEEDED" +} +</verbatim>
http://git-wip-us.apache.org/repos/asf/falcon/blob/4612c3f7/trunk/releases/0.10/src/site/twiki/restapi/EntitySchedule.twiki ---------------------------------------------------------------------- diff --git a/trunk/releases/0.10/src/site/twiki/restapi/EntitySchedule.twiki b/trunk/releases/0.10/src/site/twiki/restapi/EntitySchedule.twiki new file mode 100644 index 0000000..0dede9b --- /dev/null +++ b/trunk/releases/0.10/src/site/twiki/restapi/EntitySchedule.twiki @@ -0,0 +1,100 @@ +---++ POST /api/entities/schedule/:entity-type/:entity-name + * <a href="#Description">Description</a> + * <a href="#Parameters">Parameters</a> + * <a href="#Results">Results</a> + * <a href="#Examples">Examples</a> + +---++ Description +Schedule an entity. + +---++ Parameters + * :entity-type can either be a feed or a process. + * :entity-name is name of the entity. + * skipDryRun : Optional query param, Falcon skips oozie dryrun when value is set to true. + * doAs <optional query param> allows the current user to impersonate the user passed in doAs when interacting with the Falcon system. + * properties <key1:val1,...,keyN:valN> : Optional query param, supplies a set of key-value pairs that will be available to the entity in the coordinator configuration. These values will not override properties with the same name predefined in the entity specification. For example, to change the scheduler used for scheduling the entity you would set the property _falcon.scheduler_ in the properties parameter to _native_ to use the Falcon Scheduler or to _oozie_ to use the Oozie Scheduler. + + +---++ Results +Result of the schedule command. + +---++ Examples +---+++ Oozie Workflow +<verbatim> +<workflow-app xmlns="uri:oozie:workflow:0.4" name="aggregator-wf"> + <start to="aggregator" /> + <action name="aggregator"> + <java> + <job-tracker>${jobTracker}</job-tracker> + <name-node>${nameNode}</name-node> + <configuration> + <property> + <name>mapred.job.queue.name</name> + <value>${queueName}</value> + </property> + </configuration> + <main-class>com.company.hadoop.AggregatorJob</main-class> + <java-opts>-Dframework.instrumentation.host=${instrumentationServer}</java-opts> + <arg>--input.path=${inputBasePath}</arg> + <arg>--output.path=${outputBasePath}</arg> + </java> + <ok to="end" /> + <error to="fail" /> + </action> + <kill name="fail"> + <message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message> + </kill> +</workflow-app> +</verbatim> +---+++ Submitted Process +<verbatim> +<?xml version="1.0" encoding="UTF-8"?> +<!-- Daily sample process. Runs at 6th hour every day. Input - last day's hourly data. Generates output for yesterday --> +<process xmlns="uri:falcon:process:0.1" name="SampleProcess" > + <clusters> + <cluster name="primary-cluster"> + <validity start="2012-04-03T06:00Z" end="2022-12-30T00:00Z" /> + </cluster> + </clusters> + + <parallel>1</parallel> + <order>FIFO</order> + <frequency>hours(1)</frequency> + + <inputs> + <input name="input" feed="SampleInput" start="yesterday(0,0)" end="today(-1,0)" /> + </inputs> + + <outputs> + <output name="output" feed="SampleOutput" instance="yesterday(0,0)" /> + </outputs> + + <properties> + <property name="queueName" value="default" /> + <property name="ssh.host" value="localhost" /> + <property name="fileTimestamp" value="${coord:formatTime(coord:nominalTime(), 'yyyy-MM-dd')}" /> + <property name="instrumentationServer" value="${coord:conf('instrumentation.host')}" /> + </properties> + + <workflow engine="oozie" path="/examples/apps/aggregator" /> + <retry policy="exp-backoff" delay="minutes(5)" attempts="3" /> + + <late-process policy="exp-backoff" delay="hours(1)"> + <late-input input="input" workflow-path="/projects/bootcamp/workflow/lateinput" /> + </late-process> +</process> +</verbatim> +---+++ Rest Call +<verbatim> +POST http://localhost:15000/api/entities/schedule/process/SampleProcess?skipDryRun=false&doAs=joe&properties=instrumentation.host:intrumentation.localdomain +</verbatim> +---+++ Result +<verbatim> +{ + "requestId": "default\/ee735c95-98bd-41b8-a705-2e78bcfcdcd9\n", + "message": "default\/SampleProcess(process) scheduled successfully\n", + "status": "SUCCEEDED" +} +</verbatim> +---+++ Notes +In this example, the value of _framework.instrumentation.host_ in the Oozie workflow will be _intrumentation.localdomain_ which is the property passed when the process is scheduled. http://git-wip-us.apache.org/repos/asf/falcon/blob/4612c3f7/trunk/releases/0.10/src/site/twiki/restapi/EntityStatus.twiki ---------------------------------------------------------------------- diff --git a/trunk/releases/0.10/src/site/twiki/restapi/EntityStatus.twiki b/trunk/releases/0.10/src/site/twiki/restapi/EntityStatus.twiki new file mode 100644 index 0000000..188019d --- /dev/null +++ b/trunk/releases/0.10/src/site/twiki/restapi/EntityStatus.twiki @@ -0,0 +1,30 @@ +---++ GET /api/entities/status/:entity-type/:entity-name + * <a href="#Description">Description</a> + * <a href="#Parameters">Parameters</a> + * <a href="#Results">Results</a> + * <a href="#Examples">Examples</a> + +---++ Description +Get status of the entity. + +---++ Parameters + * :entity-type can be cluster, feed or process. + * :entity-name is name of the entity. + * doAs <optional query param> allows the current user to impersonate the user passed in doAs when interacting with the Falcon system. + +---++ Results +Status of the entity. + +---++ Examples +---+++ Rest Call +<verbatim> +GET http://localhost:15000/api/entities/status/process/SampleProcess?doAs=joe +</verbatim> +---+++ Result +<verbatim> +{ + "requestId": "default\/4d35b382-852a-4bc7-9972-b9db3493322a\n", + "message": "default\/SUBMITTED\n", + "status": "SUCCEEDED" +} +</verbatim> http://git-wip-us.apache.org/repos/asf/falcon/blob/4612c3f7/trunk/releases/0.10/src/site/twiki/restapi/EntitySubmit.twiki ---------------------------------------------------------------------- diff --git a/trunk/releases/0.10/src/site/twiki/restapi/EntitySubmit.twiki b/trunk/releases/0.10/src/site/twiki/restapi/EntitySubmit.twiki new file mode 100644 index 0000000..a8dc9d7 --- /dev/null +++ b/trunk/releases/0.10/src/site/twiki/restapi/EntitySubmit.twiki @@ -0,0 +1,105 @@ +---++ POST api/entities/submit/:entity-type + * <a href="#Description">Description</a> + * <a href="#Parameters">Parameters</a> + * <a href="#Results">Results</a> + * <a href="#Examples">Examples</a> + +---++ Description +Submit the given entity. + +---++ Parameters + * :entity-type can be cluster, feed or process. + * doAs <optional query param> allows the current user to impersonate the user passed in doAs when interacting with the Falcon system. + +---++ Results +Result of the submission. + +---++ Examples +---+++ Rest Call +<verbatim> +POST http://localhost:15000/api/entities/submit/feed +<?xml version="1.0" encoding="UTF-8"?> +<!-- Hourly sample input data --> + +<feed description="sample input data" + name="SampleInput" xmlns="uri:falcon:feed:0.1" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";> + <groups>group</groups> + <frequency>hours(1)</frequency> + <late-arrival cut-off="hours(6)" /> + <clusters> + <cluster name="primary-cluster" type="source"> + <!--validity start="2009-01-01T00:00Z" end="2099-12-31T00:00Z" timezone="UTC" /--> + <validity start="2009-01-01T00:00Z" end="2099-12-31T00:00Z" /> + <retention limit="months(24)" action="delete" /> + </cluster> + </clusters> + + <locations> + <location type="data" path="/projects/bootcamp/data/${YEAR}-${MONTH}-${DAY}-${HOUR}/SampleInput" /> + <location type="stats" path="/projects/bootcamp/stats/SampleInput" /> + <location type="meta" path="/projects/bootcamp/meta/SampleInput" /> + </locations> + + <ACL owner="suser" group="users" permission="0755" /> + + <schema location="/none" provider="none" /> +</feed> +</verbatim> + +---+++ Result +<verbatim> +{ + "requestId": "default\/d72a41f7-6420-487b-8199-62d66e492e35\n", + "message": "default\/Submit successful (feed) SampleInput\n", + "status": "SUCCEEDED" +} +</verbatim> + +---+++ Rest Call +<verbatim> +POST http://localhost:15000/api/entities/submit/process?doAs=joe +<?xml version="1.0" encoding="UTF-8"?> +<!-- Daily sample process. Runs at 6th hour every day. Input - last day's hourly data. Generates output for yesterday --> +<process xmlns="uri:falcon:process:0.1" name="SampleProcess" > + <clusters> + <cluster name="primary-cluster"> + <validity start="2012-04-03T06:00Z" end="2022-12-30T00:00Z" /> + </cluster> + </clusters> + + <parallel>1</parallel> + <order>FIFO</order> + <frequency>hours(1)</frequency> + + <inputs> + <input name="input" feed="SampleInput" start="yesterday(0,0)" end="today(-1,0)" /> + </inputs> + + <outputs> + <output name="output" feed="SampleOutput" instance="yesterday(0,0)" /> + </outputs> + + <properties> + <property name="queueName" value="default" /> + <property name="ssh.host" value="localhost" /> + <property name="fileTimestamp" value="${coord:formatTime(coord:nominalTime(), 'yyyy-MM-dd')}" /> + </properties> + + <workflow engine="oozie" path="/examples/apps/aggregator" /> + <retry policy="exp-backoff" delay="minutes(5)" attempts="3" /> + + <late-process policy="exp-backoff" delay="hours(1)"> + <late-input input="input" workflow-path="/projects/bootcamp/workflow/lateinput" /> + </late-process> +</process> +</verbatim> + +---+++ Result +<verbatim> +{ + "requestId": "default\/e5cc8230-f356-4566-9b65-536abdff8aa3\n", + "message": "default\/Submit successful (process) SampleProcess\n", + "status": "SUCCEEDED" +} +</verbatim> http://git-wip-us.apache.org/repos/asf/falcon/blob/4612c3f7/trunk/releases/0.10/src/site/twiki/restapi/EntitySubmitAndSchedule.twiki ---------------------------------------------------------------------- diff --git a/trunk/releases/0.10/src/site/twiki/restapi/EntitySubmitAndSchedule.twiki b/trunk/releases/0.10/src/site/twiki/restapi/EntitySubmitAndSchedule.twiki new file mode 100644 index 0000000..3cc23e9 --- /dev/null +++ b/trunk/releases/0.10/src/site/twiki/restapi/EntitySubmitAndSchedule.twiki @@ -0,0 +1,64 @@ +---++ POST /api/entities/submitAndSchedule/:entity-type + * <a href="#Description">Description</a> + * <a href="#Parameters">Parameters</a> + * <a href="#Results">Results</a> + * <a href="#Examples">Examples</a> + +---++ Description +Submits and schedules an entity. + +---++ Parameters + * :entity-type can either be a feed or a process. + * skipDryRun : Optional query param, Falcon skips oozie dryrun when value is set to true. + * doAs <optional query param> allows the current user to impersonate the user passed in doAs when interacting with the Falcon system. + +---++ Results +Result of the submit and schedule command. + +---++ Examples +---+++ Rest Call +<verbatim> +POST http://localhost:15000/api/entities/submitAndSchedule/process?skipDryRun=false&doAs=joe +<?xml version="1.0" encoding="UTF-8"?> +<!-- Daily sample process. Runs at 6th hour every day. Input - last day's hourly data. Generates output for yesterday --> +<process xmlns="uri:falcon:process:0.1" name="SampleProcess" > + <clusters> + <cluster name="primary-cluster"> + <validity start="2012-04-03T06:00Z" end="2022-12-30T00:00Z" /> + </cluster> + </clusters> + + <parallel>1</parallel> + <order>FIFO</order> + <frequency>hours(1)</frequency> + + <inputs> + <input name="input" feed="SampleInput" start="yesterday(0,0)" end="today(-1,0)" /> + </inputs> + + <outputs> + <output name="output" feed="SampleOutput" instance="yesterday(0,0)" /> + </outputs> + + <properties> + <property name="queueName" value="default" /> + <property name="ssh.host" value="localhost" /> + <property name="fileTimestamp" value="${coord:formatTime(coord:nominalTime(), 'yyyy-MM-dd')}" /> + </properties> + + <workflow engine="oozie" path="/examples/apps/aggregator" /> + <retry policy="exp-backoff" delay="minutes(5)" attempts="3" /> + + <late-process policy="exp-backoff" delay="hours(1)"> + <late-input input="input" workflow-path="/projects/bootcamp/workflow/lateinput" /> + </late-process> +</process> +</verbatim> +---+++ Result +<verbatim> +{ + "requestId": "schedule\/default\/b5b40931-175b-4b15-8f2b-02ef2e66f06b\n\nsubmit\/default\/b5b40931-175b-4b15-8f2b-02ef2e66f06b\n\n", + "message": "schedule\/default\/SampleProcess(process) scheduled successfully\n\nsubmit\/default\/Submit successful (process) SampleProcess\n\n", + "status": "SUCCEEDED" +} +</verbatim> http://git-wip-us.apache.org/repos/asf/falcon/blob/4612c3f7/trunk/releases/0.10/src/site/twiki/restapi/EntitySummary.twiki ---------------------------------------------------------------------- diff --git a/trunk/releases/0.10/src/site/twiki/restapi/EntitySummary.twiki b/trunk/releases/0.10/src/site/twiki/restapi/EntitySummary.twiki new file mode 100644 index 0000000..763c2a7 --- /dev/null +++ b/trunk/releases/0.10/src/site/twiki/restapi/EntitySummary.twiki @@ -0,0 +1,74 @@ +---++ GET /api/entities/summary/:entity-type + * <a href="#Description">Description</a> + * <a href="#Parameters">Parameters</a> + * <a href="#Results">Results</a> + * <a href="#Examples">Examples</a> + +---++ Description +Given an EntityType and cluster, get list of entities along with summary of N recent instances of each entity + +---++ Parameters + * :entity-type Valid options are feed or process. + * cluster Show entities that belong to this cluster. + * start <optional param> Show entity summaries from this date. Date format is yyyy-MM-dd'T'HH:mm'Z'. + * By default, it is set to (end - 2 days). + * end <optional param> Show entity summary up to this date. Date format is yyyy-MM-dd'T'HH:mm'Z'. + * Default is set to now. + * fields <optional param> Fields of entity that the user wants to view, separated by commas. + * Valid options are STATUS, TAGS, PIPELINES. + * filterBy <optional param> Filter results by list of field:value pairs. Example: filterBy=STATUS:RUNNING,PIPELINES:clickLogs + * Supported filter fields are NAME, STATUS, PIPELINES, CLUSTER. + * Query will do an AND among filterBy fields. + * tags <optional param> Return list of entities that have specified tags, separated by a comma. Query will do AND on tag values. + * Example: [email protected],[email protected] + * orderBy <optional param> Field by which results should be ordered. + * Supports ordering by "name". + * sortOrder <optional param> Valid options are "asc" and "desc" + * offset <optional param> Show results from the offset, used for pagination. Defaults to 0. + * numResults <optional param> Number of results to show per request, used for pagination. Only integers > 0 are valid, Default is 10. + * numInstances <optional param> Number of recent instances to show per entity. Only integers > 0 are valid, Default is 7. + * doAs <optional query param> allows the current user to impersonate the user passed in doAs when interacting with the Falcon system. + +---++ Results +Show entities along with summary of N instances for each entity. + +---++ Examples +---+++ Rest Call +<verbatim> +GET http://localhost:15000/api/entities/summary/feed?cluster=primary-cluster&filterBy=STATUS:RUNNING&fields=status&[email protected]&orderBy=name&offset=0&numResults=1&numInstances=2&doAs=joe +</verbatim> +---+++ Result +<verbatim> +{ + "entitySummary": [ + { + "name" : "SampleOutput", + "type" : "feed", + "status": "RUNNING", + "instances": [ + { + "details": "", + "endTime": "2013-10-21T14:40:26-07:00", + "startTime": "2013-10-21T14:39:56-07:00", + "cluster": "primary-cluster", + "logFile": "http:\/\/localhost:11000\/oozie?job=0000070-131021115933395-oozie-rgau-W", + "status": "RUNNING", + "instance": "2012-04-03T07:00Z" + }, + { + "details": "", + "endTime": "2013-10-21T14:42:27-07:00", + "startTime": "2013-10-21T14:41:57-07:00", + "cluster": "primary-cluster", + "logFile": "http:\/\/localhost:11000\/oozie?job=0000070-131021115933397-oozie-rgau-W", + "status": "RUNNING", + "instance": "2012-04-03T08:00Z" + }, + ] + } + ] + "requestId": "default\/e15bb378-d09f-4911-9df2-5334a45153d2\n", + "message": "default\/STATUS\n", + "status": "SUCCEEDED" +} +</verbatim> http://git-wip-us.apache.org/repos/asf/falcon/blob/4612c3f7/trunk/releases/0.10/src/site/twiki/restapi/EntitySuspend.twiki ---------------------------------------------------------------------- diff --git a/trunk/releases/0.10/src/site/twiki/restapi/EntitySuspend.twiki b/trunk/releases/0.10/src/site/twiki/restapi/EntitySuspend.twiki new file mode 100644 index 0000000..b322b27 --- /dev/null +++ b/trunk/releases/0.10/src/site/twiki/restapi/EntitySuspend.twiki @@ -0,0 +1,30 @@ +---++ POST /api/entities/suspend/:entity-type/:entity-name + * <a href="#Description">Description</a> + * <a href="#Parameters">Parameters</a> + * <a href="#Results">Results</a> + * <a href="#Examples">Examples</a> + +---++ Description +Suspend an entity. + +---++ Parameters + * :entity-type can either be a feed or a process. + * :entity-name is name of the entity. + * doAs <optional query param> allows the current user to impersonate the user passed in doAs when interacting with the Falcon system. + +---++ Results +Status of the entity. + +---++ Examples +---+++ Rest Call +<verbatim> +POST http://localhost:15000/api/entities/suspend/process/SampleProcess?doAs=joe +</verbatim> +---+++ Result +<verbatim> +{ + "requestId": "default\/fe5f2b6c-1f2e-49fc-af3a-342079f0b46b\n", + "message": "default\/SampleProcess(process) suspended successfully\n", + "status": "SUCCEEDED" +} +</verbatim> http://git-wip-us.apache.org/repos/asf/falcon/blob/4612c3f7/trunk/releases/0.10/src/site/twiki/restapi/EntityTouch.twiki ---------------------------------------------------------------------- diff --git a/trunk/releases/0.10/src/site/twiki/restapi/EntityTouch.twiki b/trunk/releases/0.10/src/site/twiki/restapi/EntityTouch.twiki new file mode 100644 index 0000000..5b58ce2 --- /dev/null +++ b/trunk/releases/0.10/src/site/twiki/restapi/EntityTouch.twiki @@ -0,0 +1,31 @@ +---++ POST api/entities/touch/:entity-type/:entity-name + * <a href="#Description">Description</a> + * <a href="#Parameters">Parameters</a> + * <a href="#Results">Results</a> + * <a href="#Examples">Examples</a> + +---++ Description +Force updates the entity. + +---++ Parameters + * :entity-type can be feed or process. + * :entity-name is name of the feed or process. + * skipDryRun : Optional query param, Falcon skips oozie dryrun when value is set to true. + * doAs <optional query param> allows the current user to impersonate the user passed in doAs when interacting with the Falcon system. + +---++ Results +Result of the validation. + +---++ Examples +---+++ Rest Call +<verbatim> +POST http://localhost:15000/api/entities/touch/process/SampleProcess?skipDryRun=true&doAs=joe +</verbatim> +---+++ Result +<verbatim> +{ + "requestId": "touch\/default\/d6aaa328-6836-4818-a212-515bb43d8b86\n\n", + "message": "touch\/default\/SampleProcess updated successfully\n\n", + "status": "SUCCEEDED" +} +</verbatim> http://git-wip-us.apache.org/repos/asf/falcon/blob/4612c3f7/trunk/releases/0.10/src/site/twiki/restapi/EntityUpdate.twiki ---------------------------------------------------------------------- diff --git a/trunk/releases/0.10/src/site/twiki/restapi/EntityUpdate.twiki b/trunk/releases/0.10/src/site/twiki/restapi/EntityUpdate.twiki new file mode 100644 index 0000000..cbf33db --- /dev/null +++ b/trunk/releases/0.10/src/site/twiki/restapi/EntityUpdate.twiki @@ -0,0 +1,66 @@ +---++ POST api/entities/update/:entity-type/:entity-name + * <a href="#Description">Description</a> + * <a href="#Parameters">Parameters</a> + * <a href="#Results">Results</a> + * <a href="#Examples">Examples</a> + +---++ Description +Updates the submitted entity. + +---++ Parameters + * :entity-type can be cluster, feed or process. + * :entity-name is name of the cluster, feed or process. + * skipDryRun : Optional query param, Falcon skips oozie dryrun when value is set to true. + * doAs <optional query param> allows the current user to impersonate the user passed in doAs when interacting with the Falcon system. + +---++ Results +Result of the validation. + +---++ Examples +---+++ Rest Call +<verbatim> +POST http://localhost:15000/api/entities/update/process/SampleProcess?skipDryRun=false&doAs=joe +<?xml version="1.0" encoding="UTF-8"?> +<!-- Daily sample process. Runs at 6th hour every day. Input - last day's hourly data. Generates output for yesterday --> +<process xmlns="uri:falcon:process:0.1" name="SampleProcess" > + <clusters> + <cluster name="primary-cluster"> + <validity start="2012-04-03T06:00Z" end="2022-12-30T00:00Z" /> + </cluster> + </clusters> + + <parallel>1</parallel> + <order>FIFO</order> + <frequency>hours(1)</frequency> + + <inputs> + <input name="input" feed="SampleInput" start="yesterday(0,0)" end="today(-1,0)" /> + </inputs> + + <outputs> + <output name="output" feed="SampleOutput" instance="yesterday(0,0)" /> + </outputs> + + <properties> + <property name="queueName" value="default" /> + <property name="ssh.host" value="localhost" /> + <property name="fileTimestamp" value="${coord:formatTime(coord:nominalTime(), 'yyyy-MM-dd')}" /> + </properties> + + <workflow engine="oozie" path="/examples/apps/aggregator" /> + <retry policy="exp-backoff" delay="minutes(5)" attempts="3" /> + + <late-process policy="exp-backoff" delay="hours(1)"> + <late-input input="input" workflow-path="/projects/bootcamp/workflow/lateinput" /> + </late-process> +</process> +</verbatim> +---+++ Result +<verbatim> +{ + "requestId": "update\/default\/d6aaa328-6836-4818-a212-515bb43d8b86\n\n", + "message": "update\/default\/SampleProcess updated successfully\n\n", + "status": "SUCCEEDED" +} +</verbatim> + http://git-wip-us.apache.org/repos/asf/falcon/blob/4612c3f7/trunk/releases/0.10/src/site/twiki/restapi/EntityValidate.twiki ---------------------------------------------------------------------- diff --git a/trunk/releases/0.10/src/site/twiki/restapi/EntityValidate.twiki b/trunk/releases/0.10/src/site/twiki/restapi/EntityValidate.twiki new file mode 100644 index 0000000..054b083 --- /dev/null +++ b/trunk/releases/0.10/src/site/twiki/restapi/EntityValidate.twiki @@ -0,0 +1,170 @@ +---++ POST api/entities/validate/entity-type + * <a href="#Description">Description</a> + * <a href="#Parameters">Parameters</a> + * <a href="#Results">Results</a> + * <a href="#Examples">Examples</a> + +---++ Description +Validates the submitted entity. + +---++ Parameters + * :entity-type can be cluster, feed or process. + * skipDryRun : Optional query param, Falcon skips oozie dryrun when value is set to true. + * doAs <optional query param> allows the current user to impersonate the user passed in doAs when interacting with the Falcon system. + +---++ Results +Result of the validation. + +---++ Examples +---+++ Rest Call +<verbatim> +POST http://localhost:15000/api/entities/validate/cluster +<?xml version="1.0" encoding="UTF-8" standalone="yes"?> +<cluster xmlns="uri:falcon:cluster:0.1" name="primary-cluster" description="Primary Cluster" colo="west-coast"> + <interfaces> + <interface type="readonly" endpoint="hftp://localhost:50070"; version="1.1.1"/> + <interface type="write" endpoint="hdfs://localhost:9000" version="1.1.1"/> + <interface type="execute" endpoint="localhost:9001" version="1.1.1"/> + <interface type="workflow" endpoint="http://localhost:11000/oozie/"; version="4.0.0"/> + <interface type="messaging" endpoint="tcp://localhost:61616?daemon=true" version="5.4.3"/> + </interfaces> + <locations> + <location name="staging" path="/apps/falcon/staging"/> + <location name="temp" path="/tmp"/> + <location name="working" path="/apps/falcon/working"/> + </locations> +</cluster> +</verbatim> +---+++ Result +<verbatim> +{ + "requestId": "dd3f6c3a-a6f1-4c50-97fb-3f9a3f698e10", + "message": "Validated successfully (CLUSTER) primary-cluster", + "status": "SUCCEEDED" +} +</verbatim> + +---+++ Rest Call +<verbatim> +POST http://localhost:15000/api/entities/validate/feed?skipDryRun=true +<?xml version="1.0" encoding="UTF-8"?> +<!-- Hourly sample input data --> + +<feed description="sample input data" + name="SampleInput" xmlns="uri:falcon:feed:0.1" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";> + <groups>group</groups> + <frequency>hours(1)</frequency> + <late-arrival cut-off="hours(6)" /> + <clusters> + <cluster name="primary-cluster" type="source"> + <!--validity start="2009-01-01T00:00Z" end="2099-12-31T00:00Z" timezone="UTC" /--> + <validity start="2009-01-01T00:00Z" end="2099-12-31T00:00Z" /> + <retention limit="months(24)" action="delete" /> + </cluster> + </clusters> + + <locations> + <location type="data" path="/projects/bootcamp/data/${YEAR}-${MONTH}-${DAY}-${HOUR}/SampleInput" /> + <location type="stats" path="/projects/bootcamp/stats/SampleInput" /> + <location type="meta" path="/projects/bootcamp/meta/SampleInput" /> + </locations> + + <ACL owner="suser" group="users" permission="0755" /> + + <schema location="/none" provider="none" /> +</feed> +</verbatim> + +---+++ Result +<verbatim> +{ + "requestId": "c85b190e-e653-493a-a863-d62de9c2e3b0", + "message": "Validated successfully (FEED) SampleInput", + "status": "SUCCEEDED" +} +</verbatim> + +---+++ Rest Call +<verbatim> +POST http://localhost:15000/api/entities/validate/feed +<?xml version="1.0" encoding="UTF-8"?> +<!-- Daily sample output data --> + +<feed description="sample output data" name="SampleOutput" xmlns="uri:falcon:feed:0.1" +xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";> + <groups>group</groups> + <frequency>hours(1)</frequency> + <late-arrival cut-off="hours(6)" /> + <clusters> + <cluster name="primary-cluster" type="source"> + <!--validity start="2009-01-01T00:00Z" end="2099-12-31T00:00Z" timezone="UTC" /--> + <validity start="2009-01-01T00:00Z" end="2099-12-31T00:00Z" /> + <retention limit="months(24)" action="delete" /> + </cluster> + </clusters> + <locations> + <location type="data" path="/projects/bootcamp/output/${YEAR}-${MONTH}-${DAY}-${HOUR}/SampleOutput" /> + <location type="stats" path="/projects/bootcamp/stats/SampleOutput" /> + <location type="meta" path="/projects/bootcamp/meta/SampleOutput" /> + </locations> + <ACL owner="suser" group="users" permission="0755" /> + <schema location="/none" provider="none" /> +</feed> +</verbatim> +---+++ Result +<verbatim> +{ + "requestId": "60781732-460e-4c6c-ba86-a75fae574b05", + "message": "Validated successfully (FEED) SampleOutput", + "status": "SUCCEEDED" +} +</verbatim> + +---+++ Rest Call +<verbatim> +POST http://localhost:15000/api/entities/validate/process?skipDryRun=false&doAs=joe +<?xml version="1.0" encoding="UTF-8"?> +<!-- Daily sample process. Runs at 6th hour every day. Input - last day's hourly data. Generates output for yesterday --> +<process xmlns="uri:falcon:process:0.1" name="SampleProcess" > + <clusters> + <cluster name="primary-cluster"> + <validity start="2012-04-03T06:00Z" end="2022-12-30T00:00Z" /> + </cluster> + </clusters> + + <parallel>1</parallel> + <order>FIFO</order> + <frequency>hours(1)</frequency> + + <inputs> + <input name="input" feed="SampleInput" start="yesterday(0,0)" end="today(-1,0)" /> + </inputs> + + <outputs> + <output name="output" feed="SampleOutput" instance="yesterday(0,0)" /> + </outputs> + + <properties> + <property name="queueName" value="default" /> + <property name="ssh.host" value="localhost" /> + <property name="fileTimestamp" value="${coord:formatTime(coord:nominalTime(), 'yyyy-MM-dd')}" /> + </properties> + + <workflow engine="oozie" path="/examples/apps/aggregator" /> + <retry policy="exp-backoff" delay="minutes(5)" attempts="3" /> + + <late-process policy="exp-backoff" delay="hours(1)"> + <late-input input="input" workflow-path="/projects/bootcamp/workflow/lateinput" /> + </late-process> +</process> +</verbatim> + +---+++ Result +<verbatim> +{ + "requestId": "e4a965c6-c7a2-41d9-ba08-2e77f1c43f57", + "message": "Validated successfully (PROCESS) SampleProcess", + "status": "SUCCEEDED" +} +</verbatim> http://git-wip-us.apache.org/repos/asf/falcon/blob/4612c3f7/trunk/releases/0.10/src/site/twiki/restapi/ExtensionDefinition.twiki ---------------------------------------------------------------------- diff --git a/trunk/releases/0.10/src/site/twiki/restapi/ExtensionDefinition.twiki b/trunk/releases/0.10/src/site/twiki/restapi/ExtensionDefinition.twiki new file mode 100644 index 0000000..66f6674 --- /dev/null +++ b/trunk/releases/0.10/src/site/twiki/restapi/ExtensionDefinition.twiki @@ -0,0 +1,160 @@ +---++ GET api/extension/definition/:extensionÂname + * <a href="#Description">Description</a> + * <a href="#Parameters">Parameters</a> + * <a href="#Results">Results</a> + * <a href="#Examples">Examples</a> + +---++ Description +Get definition of the extension. + +---++ Parameters + * :extensionÂname Name of the extension. + +---++ Results +Outputs a JSON document describing the extension invocation parameters. + +---++ Examples +---+++ Rest Call +<verbatim> +GET http://localhost:15000/api/extension/definition/hdfsÂmirroring +</verbatim> +---+++ Result +<verbatim> +{ + "shortDescription": "This extension implements replicating arbitrary directories on HDFS from one Hadoop cluster to another Hadoop cluster. This piggy backs on replication solution in Falcon which uses the DistCp tool.", + "properties":[ + { + "propertyName":"jobName", + "required":true, + "description":"Unique job name", + "example":"hdfs-monthly-sales-dr" + }, + { + "propertyName":"jobClusterName", + "required":true, + "description":"Cluster where job should run", + "example":"backupCluster" + }, + { + "propertyName":"jobValidityStart", + "required":true, + "description":"Job validity start time", + "example":"2016-03-03T00:00Z" + }, + { + "propertyName":"jobValidityEnd", + "required":true, + "description":"Job validity end time", + "example":"2018-03-13T00:00Z" + }, + { + "propertyName":"jobFrequency", + "required":true, + "description":"job frequency. Valid frequency types are minutes, hours, days, months", + "example":"months(1)" + }, + { + "propertyName":"jobTimezone", + "required":false, + "description":"Time zone for the job", + "example":"GMT" + }, + { + "propertyName":"jobTags", + "required":false, + "description":"list of comma separated tags. Key Value Pairs, separated by comma", + "example":"[email protected], [email protected], _department_type=forecasting" + }, + { + "propertyName":"jobRetryPolicy", + "required":false, + "description":"Job retry policy", + "example":"periodic" + }, + { + "propertyName":"jobRetryDelay", + "required":false, + "description":"Job retry delay", + "example":"minutes(30)" + }, + { + "propertyName":"jobRetryAttempts", + "required":false, + "description":"Job retry attempts", + "example":"3" + }, + { + "propertyName":"jobRetryOnTimeout", + "required":false, + "description":"Job retry on timeout", + "example":"true" + }, + { + "propertyName":"jobAclOwner", + "required":false, + "description":"ACL owner", + "example":"ambari-qa" + }, + { + "propertyName":"jobAclGroup", + "required":false, + "description":"ACL group", + "example":"users" + }, + { + "propertyName":"jobAclPermission", + "required":false, + "description":"ACL permission", + "example":"0x755" + }, + { + "propertyName":"sourceDir", + "required":true, + "description":"Multiple hdfs comma separated source directories", + "example":"/user/ambari-qa/primaryCluster/dr/input1, /user/ambari-qa/primaryCluster/dr/input2" + }, + { + "propertyName":"sourceCluster", + "required":true, + "description":"Source cluster for hdfs mirroring", + "example":"primaryCluster" + }, + { + "propertyName":"targetDir", + "required":true, + "description":"Target hdfs directory", + "example":"/user/ambari-qa/backupCluster/dr" + }, + { + "propertyName":"targetCluster", + "required":true, + "description":"Target cluster for hdfs mirroring", + "example":"backupCluster" + }, + { + "propertyName":"distcpMaxMaps", + "required":false, + "description":"Maximum number of mappers for DistCP", + "example":"1" + }, + { + "propertyName":"distcpMapBandwidth", + "required":false, + "description":"Bandwidth in MB for each mapper in DistCP", + "example":"100" + }, + { + "propertyName":"jobNotificationType", + "required":false, + "description":"Email Notification for Falcon instance completion", + "example":"email" + }, + { + "propertyName":"jobNotificationReceivers", + "required":false, + "description":"Comma separated email Id's", + "example":"[email protected], [email protected]" + } + ] +} +</verbatim> http://git-wip-us.apache.org/repos/asf/falcon/blob/4612c3f7/trunk/releases/0.10/src/site/twiki/restapi/ExtensionDelete.twiki ---------------------------------------------------------------------- diff --git a/trunk/releases/0.10/src/site/twiki/restapi/ExtensionDelete.twiki b/trunk/releases/0.10/src/site/twiki/restapi/ExtensionDelete.twiki new file mode 100644 index 0000000..2bdd23c --- /dev/null +++ b/trunk/releases/0.10/src/site/twiki/restapi/ExtensionDelete.twiki @@ -0,0 +1,29 @@ +---++ POST api/extension/delete/:job-name + * <a href="#Description">Description</a> + * <a href="#Parameters">Parameters</a> + * <a href="#Results">Results</a> + * <a href="#Examples">Examples</a> + +---++ Description +Delete an extension job. + +---++ Parameters + * :job-name Name of the extension job. + * doAs <optional> Impersonate the user. + +---++ Results +Result of the delete operation. + +---++ Examples +---+++ Rest Call +<verbatim> +POST http://localhost:15000/api/extensions/delete/sales-monthly +</verbatim> +---+++ Result +<verbatim> +{ + "requestId": "e5cc8230-f356-4566-9b65-536abdff8aa3", + "message": "Extension job sales-monthly deleted successfully", + "status": "SUCCEEDED" +} +</verbatim> http://git-wip-us.apache.org/repos/asf/falcon/blob/4612c3f7/trunk/releases/0.10/src/site/twiki/restapi/ExtensionDescription.twiki ---------------------------------------------------------------------- diff --git a/trunk/releases/0.10/src/site/twiki/restapi/ExtensionDescription.twiki b/trunk/releases/0.10/src/site/twiki/restapi/ExtensionDescription.twiki new file mode 100644 index 0000000..5900fbb --- /dev/null +++ b/trunk/releases/0.10/src/site/twiki/restapi/ExtensionDescription.twiki @@ -0,0 +1,24 @@ +---++ GET api/extension/describe/:extensionÂname + * <a href="#Description">Description</a> + * <a href="#Parameters">Parameters</a> + * <a href="#Results">Results</a> + * <a href="#Examples">Examples</a> + +---++ Description +Description of an extension. + +---++ Parameters + * :extensionÂname Name of the extension. + +---++ Results +Outputs the README of the specified extension. + +---++ Examples +---+++ Rest Call +<verbatim> +GET http://localhost:15000/api/extension/describe/hdfsÂmirroring +</verbatim> +---+++ Result +<verbatim> +<README file of the specified extension> +</verbatim> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/falcon/blob/4612c3f7/trunk/releases/0.10/src/site/twiki/restapi/ExtensionEnumeration.twiki ---------------------------------------------------------------------- diff --git a/trunk/releases/0.10/src/site/twiki/restapi/ExtensionEnumeration.twiki b/trunk/releases/0.10/src/site/twiki/restapi/ExtensionEnumeration.twiki new file mode 100644 index 0000000..abd94c8 --- /dev/null +++ b/trunk/releases/0.10/src/site/twiki/restapi/ExtensionEnumeration.twiki @@ -0,0 +1,38 @@ +---++ GET api/extension/enumerate + * <a href="#Description">Description</a> + * <a href="#Parameters">Parameters</a> + * <a href="#Results">Results</a> + * <a href="#Examples">Examples</a> + +---++ Description +Get list of the supported extensions. + +---++ Parameters +None + +---++ Results +Total number of results and a list of extension server extensions supported. + +---++ Examples +---+++ Rest Call +<verbatim> +GET http://localhost:15000/api/extension/enumerate +</verbatim> +---+++ Result +<verbatim> +{ + "totalResults":"2â, + âextensionsâ: [ + { + ânameâ: âHdfsÂmirroringâ + âtypeâ: âTrusted/Provided extensionâ + âdescriptionâ: âThis extension implements replicating arbitrary directories on HDFS from one Hadoop cluster to another Hadoop cluster.â + }, + { + ânameâ: âHiveÂmirroringâ + âtypeâ: âTrusted/Provided extensionâ + âdescriptionâ: âThis extension implements replicating hive metadata and data from one Hadoop cluster to another Hadoop cluster.â + } + ] +} +</verbatim> http://git-wip-us.apache.org/repos/asf/falcon/blob/4612c3f7/trunk/releases/0.10/src/site/twiki/restapi/ExtensionInstances.twiki ---------------------------------------------------------------------- diff --git a/trunk/releases/0.10/src/site/twiki/restapi/ExtensionInstances.twiki b/trunk/releases/0.10/src/site/twiki/restapi/ExtensionInstances.twiki new file mode 100644 index 0000000..a11a97a --- /dev/null +++ b/trunk/releases/0.10/src/site/twiki/restapi/ExtensionInstances.twiki @@ -0,0 +1,87 @@ +---++ GET api/extension/instances/:job-name + * <a href="#Description">Description</a> + * <a href="#Parameters">Parameters</a> + * <a href="#Results">Results</a> + * <a href="#Examples">Examples</a> + +---++ Description +List instances of an extension job. + +---++ Parameters + * :job-name Name of the extension job. + * start <optional> Show instances from this nominal time. Format: yyyy-MM-dd'T'HH:mm'Z'. + * end <optional> Show instances up to this nominal time. Format: yyyy-MM-dd'T'HH:mm'Z'. + * instanceStatus <optional> Valid options are RUNNING, SUCCEEDED, FAILED, WAITING, SUSPENDED, KILLED, TIMEOUT. + * fields <optional> Entity output fields separated by commas. Valid options: STATUS, TAGS, PIPELINES, CLUSTERS. + * orderBy <optional> Column by which results should be ordered. Sorted by descending order. Valid options are nominalTime (default) and status. + * sortOrder <optional> Valid options are âascâ and âdescâ. + * offset <optional> Show results from the offset. Used for pagination. Default is 0. + * numResults <optional> Number of instances per entity to show. Default value is 10. + * doAs <optional> Impersonate the user. + +---++ Results +A list of entities of the job, each followed by a set of instances. + +---++ Examples +---+++ Rest Call +<verbatim> +GET http://localhost:15000/api/extensions/instances/daily-health-bill?start=2012-04-01T00:00 +</verbatim> +---+++ Result +<verbatim> +{ + "numEntities": "2", + "entitiesSummary": { + "entitySummary": [ + { + "entityProfile": { + "type": "FEED", + "name": "SampleUSHealthBill" + }, + "instances": [ + { + "instance": "2012-04-02T00:00Z", + "status": "RUNNING", + "cluster": "SampleCluster1", + "sourceCluster": "primaryCluster", + "details": "hftp://sandbox.hortonworks.com:50070/apps/falcon/adf-demo/pig-input-2012-04-02"; + }, + { + "instance": "2012-04-01T00:00Z", + "status": "SUCCEEDED", + "cluster": "SampleCluster1", + "sourceCluster": "primaryCluster", + "details": "hftp://sandbox.hortonworks.com:50070/apps/falcon/adf-demo/pig-input-2012-04-01"; + } + ] + }, + { + "entityProfile": { + "type": "PROCESS", + "name": "SampleBillPay" + }, + "instances": [ + { + "instance": "2012-04-02T00:00Z", + "status": "RUNNING", + "logFile": "http://sandbox.hortonworks.com:11000/oozie?job=0000085-160322203213815-oozie-oozi-W";, + "cluster": "primaryCluster", + "startTime": "2016-04-04T22:37:46Z", + "details": "", + "actions": { + "action": "dr-replication", + "status": "RUNNING", + "logFile": "http://sandbox.hortonworks.com:8088/proxy/application_1458678764964_0052/"; + } + }, + { + "instance": "2012-04-01T00:00Z", + "status": "SKIPPED", + "cluster": "primaryCluster" + } + ] + } + ] + } +} +</verbatim> http://git-wip-us.apache.org/repos/asf/falcon/blob/4612c3f7/trunk/releases/0.10/src/site/twiki/restapi/ExtensionList.twiki ---------------------------------------------------------------------- diff --git a/trunk/releases/0.10/src/site/twiki/restapi/ExtensionList.twiki b/trunk/releases/0.10/src/site/twiki/restapi/ExtensionList.twiki new file mode 100644 index 0000000..dcd4d8d --- /dev/null +++ b/trunk/releases/0.10/src/site/twiki/restapi/ExtensionList.twiki @@ -0,0 +1,71 @@ +---++ GET api/extension/list/:extension-name + * <a href="#Description">Description</a> + * <a href="#Parameters">Parameters</a> + * <a href="#Results">Results</a> + * <a href="#Examples">Examples</a> + +---++ Description +List jobs generated from an extension. + +---++ Parameters + * :extension-name Name of the extension. + * sortOrder <optional> Sort order by job name. Valid options: "asc" (default) and "desc". + * offset <optional> Show results from the offset. Default is 0. + * numResults <optional> Number of results to show per request. Default is 10. + * fields <optional> Output fields separated by commas. Valid options: STATUS, TAGS, PIPELINES, CLUSTERS. + * doAs <optional> Impersonate the user. + +---++ Results +Total number of results and a list of jobs generated from the extension, followed by the associated entities. + +---++ Examples +---+++ Rest Call +<verbatim> +GET http://localhost:15000/api/extensions/list/billCollection?fields=status,clusters,tags +</verbatim> +---+++ Result +<verbatim> +{ + "numJobs":"2", + "jobs":{ + "job": [ + { + "jobName": "daily-health-bill", + "jobEntities: { + "totalResults": "2", + "entity": [ + { + "type":"FEED", + "name":"SampleUSHealthBill", + "status":"SUBMITTEDâ, + "tags":{"tag":["related=ushealthcare","department=billingDepartment","_falcon_extension_name=billCoollection","_falcon_extension_job=daily-health-bill"]}, + "clusters": {"cluster":["SampleCluster1","primaryClusterâ]} + }, + { + "type":"PROCESSâ, + "name":"SampleBillPayâ, + "status":"RUNNINGâ, + "tags":{"tag":["related=healthcare","department=billingDepartment","_falcon_extension_name=billCoollection","_falcon_extension_job=daily-health-bill"]}, + "clusters":{"cluster":"primaryClusterâ} + } + ] + } + }, + { + "jobName": "fsa-bill", + "jobEntities": { + "totalResults": "1", + "entity": + { + "type":"PROCESSâ, + "name":"FSAPayâ, + "status":"RUNNINGâ, + "tagsâ:{"tag":["related=healthcare","department=billingDepartment","_falcon_extension_name=billCollection","_falcon_extension_job=fsa-bill"]}, + "clusters":{"cluster":"primaryClusterâ} + } + } + } + ] + } +} +</verbatim> http://git-wip-us.apache.org/repos/asf/falcon/blob/4612c3f7/trunk/releases/0.10/src/site/twiki/restapi/ExtensionResume.twiki ---------------------------------------------------------------------- diff --git a/trunk/releases/0.10/src/site/twiki/restapi/ExtensionResume.twiki b/trunk/releases/0.10/src/site/twiki/restapi/ExtensionResume.twiki new file mode 100644 index 0000000..70cbbff --- /dev/null +++ b/trunk/releases/0.10/src/site/twiki/restapi/ExtensionResume.twiki @@ -0,0 +1,29 @@ +---++ POST api/extension/resume/:job-name + * <a href="#Description">Description</a> + * <a href="#Parameters">Parameters</a> + * <a href="#Results">Results</a> + * <a href="#Examples">Examples</a> + +---++ Description +Resume an extension job. + +---++ Parameters + * :job-name Name of the extension job. + * doAs <optional> Impersonate the user. + +---++ Results +Result of the resume operation. + +---++ Examples +---+++ Rest Call +<verbatim> +POST http://localhost:15000/api/extensions/resume/sales-monthly +</verbatim> +---+++ Result +<verbatim> +{ + "requestId": "e5cc8230-f356-4566-9b65-536abdff8aa3", + "message": "Extension job sales-monthly resumed successfully", + "status": "SUCCEEDED" +} +</verbatim> http://git-wip-us.apache.org/repos/asf/falcon/blob/4612c3f7/trunk/releases/0.10/src/site/twiki/restapi/ExtensionSchedule.twiki ---------------------------------------------------------------------- diff --git a/trunk/releases/0.10/src/site/twiki/restapi/ExtensionSchedule.twiki b/trunk/releases/0.10/src/site/twiki/restapi/ExtensionSchedule.twiki new file mode 100644 index 0000000..c40ef6e --- /dev/null +++ b/trunk/releases/0.10/src/site/twiki/restapi/ExtensionSchedule.twiki @@ -0,0 +1,29 @@ +---++ POST api/extension/schedule/:job-name + * <a href="#Description">Description</a> + * <a href="#Parameters">Parameters</a> + * <a href="#Results">Results</a> + * <a href="#Examples">Examples</a> + +---++ Description +Schedule an extension job. + +---++ Parameters + * :job-name Name of the extension job. + * doAs <optional> Impersonate the user. + +---++ Results +Result of the schedule operation. + +---++ Examples +---+++ Rest Call +<verbatim> +POST http://localhost:15000/api/extensions/schedule/sales-monthly +</verbatim> +---+++ Result +<verbatim> +{ + "requestId": "e5cc8230-f356-4566-9b65-536abdff8aa3", + "message": "Extension job sales-monthly scheduled successfully", + "status": "SUCCEEDED" +} +</verbatim> http://git-wip-us.apache.org/repos/asf/falcon/blob/4612c3f7/trunk/releases/0.10/src/site/twiki/restapi/ExtensionSubmit.twiki ---------------------------------------------------------------------- diff --git a/trunk/releases/0.10/src/site/twiki/restapi/ExtensionSubmit.twiki b/trunk/releases/0.10/src/site/twiki/restapi/ExtensionSubmit.twiki new file mode 100644 index 0000000..6b4df32 --- /dev/null +++ b/trunk/releases/0.10/src/site/twiki/restapi/ExtensionSubmit.twiki @@ -0,0 +1,38 @@ +---++ POST api/extension/submit/:extension-name + * <a href="#Description">Description</a> + * <a href="#Parameters">Parameters</a> + * <a href="#Results">Results</a> + * <a href="#Examples">Examples</a> + +---++ Description +Submit an extension job. + +---++ Parameters + * :extension-name Name of the extension. + * doAs <optional> Impersonate the user. + +---++ Results +Result of submission. + +---++ Examples +---+++ Rest Call +<verbatim> +POST http://localhost:15000/api/extensions/submit/hdfs-mirroring +jobName=sales-monthly +jobClustername=primaryCluster +jobClusterValidityStart=2015-03-13T00:00Z +jobClusterValidityEnd=2016-12-30T00:00Z +jobFrequency=minutes(5) +sourceDir=/user/hrt_qa/dr/test/primaryCluster/input +sourceCluster=primaryCluster +targetDir=/user/hrt_qa/dr/test/backupCluster/input +targetCluster=backupCluster +</verbatim> +---+++ Result +<verbatim> +{ + "requestId": "e5cc8230-f356-4566-9b65-536abdff8aa3", + "message": "Extension job submitted successfully", + "status": "SUCCEEDED" +} +</verbatim> http://git-wip-us.apache.org/repos/asf/falcon/blob/4612c3f7/trunk/releases/0.10/src/site/twiki/restapi/ExtensionSubmitAndSchedule.twiki ---------------------------------------------------------------------- diff --git a/trunk/releases/0.10/src/site/twiki/restapi/ExtensionSubmitAndSchedule.twiki b/trunk/releases/0.10/src/site/twiki/restapi/ExtensionSubmitAndSchedule.twiki new file mode 100644 index 0000000..8a148ce --- /dev/null +++ b/trunk/releases/0.10/src/site/twiki/restapi/ExtensionSubmitAndSchedule.twiki @@ -0,0 +1,38 @@ +---++ POST api/extension/submitAndSchedule/:extension-name + * <a href="#Description">Description</a> + * <a href="#Parameters">Parameters</a> + * <a href="#Results">Results</a> + * <a href="#Examples">Examples</a> + +---++ Description +Submit and schedule an extension job. + +---++ Parameters + * :extension-name Name of the extension. + * doAs <optional> Impersonate the user. + +---++ Results +Result of the submit and schedule operation. + +---++ Examples +---+++ Rest Call +<verbatim> +POST http://localhost:15000/api/extensions/submitAndSchedule/hdfs-mirroring +jobName=sales-monthly +jobClustername=primaryCluster +jobClusterValidityStart=2015-03-13T00:00Z +jobClusterValidityEnd=2016-12-30T00:00Z +jobFrequency=minutes(5) +sourceDir=/user/hrt_qa/dr/test/primaryCluster/input +sourceCluster=primaryCluster +targetDir=/user/hrt_qa/dr/test/backupCluster/input +targetCluster=backupCluster +</verbatim> +---+++ Result +<verbatim> +{ + "requestId": "e5cc8230-f356-4566-9b65-536abdff8aa3", + "message": "Extension job submitted and scheduled successfully", + "status": "SUCCEEDED" +} +</verbatim> http://git-wip-us.apache.org/repos/asf/falcon/blob/4612c3f7/trunk/releases/0.10/src/site/twiki/restapi/ExtensionSuspend.twiki ---------------------------------------------------------------------- diff --git a/trunk/releases/0.10/src/site/twiki/restapi/ExtensionSuspend.twiki b/trunk/releases/0.10/src/site/twiki/restapi/ExtensionSuspend.twiki new file mode 100644 index 0000000..cc3478f --- /dev/null +++ b/trunk/releases/0.10/src/site/twiki/restapi/ExtensionSuspend.twiki @@ -0,0 +1,29 @@ +---++ POST api/extension/suspend/:job-name + * <a href="#Description">Description</a> + * <a href="#Parameters">Parameters</a> + * <a href="#Results">Results</a> + * <a href="#Examples">Examples</a> + +---++ Description +Suspend an extension job. + +---++ Parameters + * :job-name Name of the extension job. + * doAs <optional> Impersonate the user. + +---++ Results +Result of the suspend operation. + +---++ Examples +---+++ Rest Call +<verbatim> +POST http://localhost:15000/api/extensions/suspend/sales-monthly +</verbatim> +---+++ Result +<verbatim> +{ + "requestId": "e5cc8230-f356-4566-9b65-536abdff8aa3", + "message": "Extension job sales-monthly suspended successfully", + "status": "SUCCEEDED" +} +</verbatim> http://git-wip-us.apache.org/repos/asf/falcon/blob/4612c3f7/trunk/releases/0.10/src/site/twiki/restapi/ExtensionUpdate.twiki ---------------------------------------------------------------------- diff --git a/trunk/releases/0.10/src/site/twiki/restapi/ExtensionUpdate.twiki b/trunk/releases/0.10/src/site/twiki/restapi/ExtensionUpdate.twiki new file mode 100644 index 0000000..1ac626a --- /dev/null +++ b/trunk/releases/0.10/src/site/twiki/restapi/ExtensionUpdate.twiki @@ -0,0 +1,38 @@ +---++ POST api/extension/update/:extension-name + * <a href="#Description">Description</a> + * <a href="#Parameters">Parameters</a> + * <a href="#Results">Results</a> + * <a href="#Examples">Examples</a> + +---++ Description +Update an extension entity. Note that we require all the entity names to remain the same. Future work will allow different entity names. + +---++ Parameters + * :extension-name Name of the extension. + * doAs <optional> Impersonate the user. + +---++ Results +Result of update. + +---++ Examples +---+++ Rest Call +<verbatim> +POST http://localhost:15000/api/extensions/update/hdfs-mirroring +jobName=sales-monthly +jobClustername=primaryCluster +jobClusterValidityStart=2015-03-13T00:00Z +jobClusterValidityEnd=2016-12-30T00:00Z +jobFrequency=minutes(5) +sourceDir=/user/hrt_qa/dr/test/primaryCluster/input +sourceCluster=primaryCluster +targetDir=/user/hrt_qa/dr/test/backupCluster/input +targetCluster=backupCluster +</verbatim> +---+++ Result +<verbatim> +{ + "requestId": "e5cc8230-f356-4566-9b65-536abdff8aa3", + "message": "Updated successfully", + "status": "SUCCEEDED" +} +</verbatim> http://git-wip-us.apache.org/repos/asf/falcon/blob/4612c3f7/trunk/releases/0.10/src/site/twiki/restapi/ExtensionValidate.twiki ---------------------------------------------------------------------- diff --git a/trunk/releases/0.10/src/site/twiki/restapi/ExtensionValidate.twiki b/trunk/releases/0.10/src/site/twiki/restapi/ExtensionValidate.twiki new file mode 100644 index 0000000..c27f2d7 --- /dev/null +++ b/trunk/releases/0.10/src/site/twiki/restapi/ExtensionValidate.twiki @@ -0,0 +1,38 @@ +---++ POST api/extension/validate/:extension-name + * <a href="#Description">Description</a> + * <a href="#Parameters">Parameters</a> + * <a href="#Results">Results</a> + * <a href="#Examples">Examples</a> + +---++ Description +Validate the input for extension job submission. + +---++ Parameters + * :extension-name Name of the extension. + * doAs <optional> Impersonate the user. + +---++ Results +Result of validation. + +---++ Examples +---+++ Rest Call +<verbatim> +POST http://localhost:15000/api/extensions/validate/hdfs-mirroring +jobName=sales-monthly +jobClustername=primaryCluster +jobClusterValidityStart=2015-03-13T00:00Z +jobClusterValidityEnd=2016-12-30T00:00Z +jobFrequency=minutes(5) +sourceDir=/user/hrt_qa/dr/test/primaryCluster/input +sourceCluster=primaryCluster +targetDir=/user/hrt_qa/dr/test/backupCluster/input +targetCluster=backupCluster +</verbatim> +---+++ Result +<verbatim> +{ + "requestId": "e5cc8230-f356-4566-9b65-536abdff8aa3", + "message": "Validated successfully", + "status": "SUCCEEDED" +} +</verbatim> http://git-wip-us.apache.org/repos/asf/falcon/blob/4612c3f7/trunk/releases/0.10/src/site/twiki/restapi/FeedInstanceListing.twiki ---------------------------------------------------------------------- diff --git a/trunk/releases/0.10/src/site/twiki/restapi/FeedInstanceListing.twiki b/trunk/releases/0.10/src/site/twiki/restapi/FeedInstanceListing.twiki new file mode 100644 index 0000000..03f3c57 --- /dev/null +++ b/trunk/releases/0.10/src/site/twiki/restapi/FeedInstanceListing.twiki @@ -0,0 +1,46 @@ +---++ GET /api/instance/listing/feed/:entity-name + * <a href="#Description">Description</a> + * <a href="#Parameters">Parameters</a> + * <a href="#Results">Results</a> + * <a href="#Examples">Examples</a> + +---++ Description +Get falcon feed instance availability. + +---++ Parameters + * :entity-name Name of the entity. + * start <optional param> Show instances from this date. Date format is yyyy-MM-dd'T'HH:mm'Z'. + * By default, it is set to (end - (10 * entityFrequency)). + * end <optional param> Show instances up to this date. Date format is yyyy-MM-dd'T'HH:mm'Z'. + * Default is set to now. + * colo <optional param> Colo on which the query should be run. + * doAs <optional query param> allows the current user to impersonate the user passed in doAs when interacting with the Falcon system. + +---++ Results +Feed instance availability status + +---++ Examples +---+++ Rest Call +<verbatim> +GET http://localhost:15000/api/instance/listing/feed/SampleFeed?colo=*&start=2012-04-03T07:00Z&doAs=joe +</verbatim> +---+++ Result +<verbatim> +{ + "instances": [ + { + "size": "450231212222", + "creationTime": "1236679827365", + "cluster": "primary-cluster", + "uri": "/data/SampleFeed/2012-04-03", + "status": "AVAILABLE", + "instance": "2012-04-03T07:00Z" + } + ], + "requestId": "default\/3527038e-8334-4e50-8173-76c4fa430d0b\n", + "message": "default\/STATUS\n", + "status": "SUCCEEDED" +} +</verbatim> + + http://git-wip-us.apache.org/repos/asf/falcon/blob/4612c3f7/trunk/releases/0.10/src/site/twiki/restapi/FeedLookup.twiki ---------------------------------------------------------------------- diff --git a/trunk/releases/0.10/src/site/twiki/restapi/FeedLookup.twiki b/trunk/releases/0.10/src/site/twiki/restapi/FeedLookup.twiki new file mode 100644 index 0000000..053182b --- /dev/null +++ b/trunk/releases/0.10/src/site/twiki/restapi/FeedLookup.twiki @@ -0,0 +1,37 @@ +---++ GET api/entities/lookup/feed + * <a href="#Description">Description</a> + * <a href="#Parameters">Parameters</a> + * <a href="#Results">Results</a> + * <a href="#Examples">Examples</a> + +---++ Description + +---++ Parameters + * path path of the instance for which you want to determine the feed. e.g. /data/project1/2014/10/10/23/ + Path has to be the complete path and can't be a part of it. + * doAs <optional query param> allows the current user to impersonate the user passed in doAs when interacting with the Falcon system. + +---++ Results +Returns the name of the feed along with the location type(meta/data/stats) and cluster on which the given path belongs to this feed. + +---++ Examples +---+++ Rest Call +<verbatim> +GET http://localhost:15000/api/entities/lookup/feed?path=/data/project1/2014/10/10/23&doAs=joe +</verbatim> +---+++ Result +{ + "feeds": + [ + { + "feedName": "My-Feed1", + "locationType": "DATA", + "clusterName": "My-cluster1" + }, + { + "feedName": "My-Feed2", + "locationType": "DATA", + "clusterName": "My-cluster2" + } + ] +} http://git-wip-us.apache.org/repos/asf/falcon/blob/4612c3f7/trunk/releases/0.10/src/site/twiki/restapi/FeedSLA.twiki ---------------------------------------------------------------------- diff --git a/trunk/releases/0.10/src/site/twiki/restapi/FeedSLA.twiki b/trunk/releases/0.10/src/site/twiki/restapi/FeedSLA.twiki new file mode 100644 index 0000000..8760976 --- /dev/null +++ b/trunk/releases/0.10/src/site/twiki/restapi/FeedSLA.twiki @@ -0,0 +1,56 @@ +---++ GET /api/entities/sla-alert/:entity-type + * <a href="#Description">Description</a> + * <a href="#Parameters">Parameters</a> + * <a href="#Results">Results</a> + * <a href="#Examples">Examples</a> + +---++ Description +<verbatim> +Since: 0.8 +</verbatim> +This command lists all the feed instances which have missed sla and are still not available. If a feed instance missed +sla but is now available, then it will not be reported in results. The purpose of this API is alerting and hence it + doesn't return feed instances which missed SLA but are available as they don't require any action. + +---++ Parameters + * :entity-type Only valid option is feed. + * entity-name <optional param> parameter to restrict results for a particular feed using feed's name. + * start <mandatory param> start of the time window for nominal instances, inclusive. + * end <mandatory param> end of the time window for nominal instances to be considered, default is treated as current time. + * colo <optional param> name of the colo + + +---++ Results +Pending feed instances which missed SLA. + +---++ Examples +---+++ Rest Call +<verbatim> +GET http://localhost:15000/api/entities/sla-alert/feed?colo=*&start=2012-04-03T07:00Z +</verbatim> +---+++ Result +<verbatim> +{ + "status":"SUCCEEDED", + "message":"default/Success!\n", + "requestId":"default/885720178@qtp-495452957-6 - f6e82e9b-d23f-466b-82df-4fb8293ce9cf\n", + "instances":[ + {"cluster":"local","entityName":"out","entityType":"FEED","instanceTime":"2015-09-26T17:33:00+05:30","tags":"Missed SLA High"}, + {"cluster":"local","entityName":"out","entityType":"FEED","instanceTime":"2015-09-26T17:29:00+05:30","tags":"Missed SLA High"}, + {"cluster":"local","entityName":"out","entityType":"FEED","instanceTime":"2015-09-26T17:35:00+05:30","tags":"Missed SLA Low"}, + {"cluster":"local","entityName":"out","entityType":"FEED","instanceTime":"2015-09-26T17:30:00+05:30","tags":"Missed SLA High"}, + {"cluster":"local","entityName":"out","entityType":"FEED","instanceTime":"2015-09-26T17:34:00+05:30","tags":"Missed SLA High"}, + {"cluster":"local","entityName":"out","entityType":"FEED","instanceTime":"2015-09-26T17:31:00+05:30","tags":"Missed SLA High"}, + {"cluster":"local","entityName":"out","entityType":"FEED","instanceTime":"2015-09-26T17:32:00+05:30","tags":"Missed SLA High"} + ] +} +</verbatim> + +In case there are no pending instances which have missed sla the response will be like below: +<verbatim> +{ + "status":"SUCCEEDED", + "message":"default/Success!\n", + "requestId":"default/979808239@qtp-1243851750-3 - 8c7396c0-efe2-43e9-9aea-7ae6afea5fd6\n" +} +</verbatim> http://git-wip-us.apache.org/repos/asf/falcon/blob/4612c3f7/trunk/releases/0.10/src/site/twiki/restapi/Graph.twiki ---------------------------------------------------------------------- diff --git a/trunk/releases/0.10/src/site/twiki/restapi/Graph.twiki b/trunk/releases/0.10/src/site/twiki/restapi/Graph.twiki new file mode 100644 index 0000000..db58d2e --- /dev/null +++ b/trunk/releases/0.10/src/site/twiki/restapi/Graph.twiki @@ -0,0 +1,22 @@ +---++ GET api/metadata/lineage/serialize + * <a href="#Description">Description</a> + * <a href="#Parameters">Parameters</a> + * <a href="#Results">Results</a> + * <a href="#Examples">Examples</a> + +---++ Description +Dump the graph. + +---++ Parameters + * doAs <optional query param> allows the current user to impersonate the user passed in doAs when interacting with the Falcon system. + +---++ Results +Serialize graph to a file configured using *.falcon.graph.serialize.path in Custom startup.properties. + +---++ Examples +---+++ Rest Call +<verbatim> +GET http://localhost:15000/api/metadata/lineage/serialize?doAs=joe +</verbatim> +---+++ Result +None. http://git-wip-us.apache.org/repos/asf/falcon/blob/4612c3f7/trunk/releases/0.10/src/site/twiki/restapi/InstanceDependencies.twiki ---------------------------------------------------------------------- diff --git a/trunk/releases/0.10/src/site/twiki/restapi/InstanceDependencies.twiki b/trunk/releases/0.10/src/site/twiki/restapi/InstanceDependencies.twiki new file mode 100644 index 0000000..5641757 --- /dev/null +++ b/trunk/releases/0.10/src/site/twiki/restapi/InstanceDependencies.twiki @@ -0,0 +1,49 @@ +---++ GET /api/instance/dependencies/:entity-type/:entity-name + * <a href="#Description">Description</a> + * <a href="#Parameters">Parameters</a> + * <a href="#Results">Results</a> + * <a href="#Examples">Examples</a> + +---++ Description +Get dependent instances for a particular instance. + +---++ Parameters + * :entity-type Valid options are feed or process. + * :entity-name Name of the entity + * instanceTime <mandatory param> time of the given instance + * colo <optional param> name of the colo + + +---++ Results +Dependent instances for the specified instance + +---++ Examples +---+++ Rest Call +<verbatim> +GET http://localhost:15000/api/instance/dependencies/feed/myFeed?colo=*&instanceTime=2012-04-03T07:00Z +</verbatim> +---+++ Result +<verbatim> +{ + 'status': 'SUCCEEDED', + 'message': 'default/Success!\n', + 'dependencies': [ + { + 'cluster': 'local', + 'entityName': 'consumer-process', + 'entityType': 'PROCESS', + 'instanceTime': '2014-12-18T00:00Z', + 'tags': 'Input' + }, + { + 'cluster': 'local', + 'entityName': 'producer-process', + 'entityType': 'PROCESS', + 'instanceTime': '2014-12-18T00:00Z', + 'tags': 'Output' + } + ], + 'requestId': 'default/1405883107@qtp-1501726962-6-0c2e690f-546b-47b0-a5ee-0365d4522a31\n' +} +</verbatim> + http://git-wip-us.apache.org/repos/asf/falcon/blob/4612c3f7/trunk/releases/0.10/src/site/twiki/restapi/InstanceKill.twiki ---------------------------------------------------------------------- diff --git a/trunk/releases/0.10/src/site/twiki/restapi/InstanceKill.twiki b/trunk/releases/0.10/src/site/twiki/restapi/InstanceKill.twiki new file mode 100644 index 0000000..eb22945 --- /dev/null +++ b/trunk/releases/0.10/src/site/twiki/restapi/InstanceKill.twiki @@ -0,0 +1,44 @@ +---++ POST /api/instance/kill/:entity-type/:entity-name + * <a href="#Description">Description</a> + * <a href="#Parameters">Parameters</a> + * <a href="#Results">Results</a> + * <a href="#Examples">Examples</a> + +---++ Description +Kill currently running instance(s) of an entity. + +---++ Parameters + * :entity-type can either be a feed or a process. + * :entity-name is name of the entity. + * start is the start time of the instance(s) that you want to refer to + * end is the end time of the instance(s) that you want to refer to + * lifecycle <optional param> can be Eviction/Replication(default) for feed and Execution(default) for process. + * doAs <optional query param> allows the current user to impersonate the user passed in doAs when interacting with the Falcon system. + +---++ Results +Result of the kill operation. + +---++ Examples +---+++ Rest Call +<verbatim> +POST http://localhost:15000/api/instance/kill/process/SampleProcess?colo=*&start=2012-04-03T07:00Z&end=2014-04-03T07:00Z&doAs=joe +</verbatim> +---+++ Result +<verbatim> +{ + "instances": [ + { + "details": "", + "endTime": "2013-10-21T15:26:59-07:00", + "startTime": "2013-10-21T15:19:57-07:00", + "cluster": "primary-cluster", + "logFile": "http:\/\/localhost:11000\/oozie?job=0000070-131021115933395-oozie-rgau-W", + "status": "KILLED", + "instance": "2012-04-03T07:00Z" + } + ], + "requestId": "default\/23b3cfee-ee22-40c0-825d-39c322587d5f\n", + "message": "default\/KILL\n", + "status": "SUCCEEDED" +} +</verbatim> http://git-wip-us.apache.org/repos/asf/falcon/blob/4612c3f7/trunk/releases/0.10/src/site/twiki/restapi/InstanceList.twiki ---------------------------------------------------------------------- diff --git a/trunk/releases/0.10/src/site/twiki/restapi/InstanceList.twiki b/trunk/releases/0.10/src/site/twiki/restapi/InstanceList.twiki new file mode 100644 index 0000000..214c22f --- /dev/null +++ b/trunk/releases/0.10/src/site/twiki/restapi/InstanceList.twiki @@ -0,0 +1,151 @@ +---++ GET /api/instance/list/:entity-type/:entity-name + * <a href="#Description">Description</a> + * <a href="#Parameters">Parameters</a> + * <a href="#Results">Results</a> + * <a href="#Examples">Examples</a> + +---++ Description +Get list of all instances of a given entity. + +---++ Parameters + * :entity-type Valid options are cluster, feed or process. + * :entity-name Name of the entity. + * start <optional param> Show instances from this date. Date format is yyyy-MM-dd'T'HH:mm'Z'. + * By default, it is set to (end - (10 * entityFrequency)). + * end <optional param> Show instances up to this date. Date format is yyyy-MM-dd'T'HH:mm'Z'. + * Default is set to now. + * colo <optional param> Colo on which the query should be run. + * lifecycle <optional param> Valid lifecycles for feed are Eviction/Replication(default) and for process is Execution(default). + * filterBy <optional param> Filter results by list of field:value pairs. Example: filterBy=STATUS:RUNNING,CLUSTER:primary-cluster + * Supported filter fields are STATUS, CLUSTER, SOURCECLUSTER, STARTEDAFTER. + * Query will do an AND among filterBy fields. + * orderBy <optional param> Field by which results should be ordered. + * Supports ordering by "status","startTime","endTime","cluster". + * sortOrder <optional param> Valid options are "asc" and "desc" + * offset <optional param> Show results from the offset, used for pagination. Defaults to 0. + * numResults <optional param> Number of results to show per request, used for pagination. Only integers > 0 are valid, Default is 10. + * doAs <optional query param> allows the current user to impersonate the user passed in doAs when interacting with the Falcon system. + * allAttempts <optional query param> To get all the attempts for corresponding instances. + +---++ Results +List of instances of given entity. + +The possible instance status returned and its meaning are as follows: + * WAITING - The instance is waiting for the corresponding data(feed) instances to become available. + * READY - The instance is ready to be scheduled. But, is waiting for scheduling conditions to be met. For example, limitation on number of instances that can be run in parallel. + * RUNNING - The instance is running on the workflow engine. + * FAILED - The instance has failed during execution. + * KILLED - The instance has been killed either manually or by the system. + * SUCCEEDED - The instance has executed successfully. + * SKIPPED - This instance was not executed, but was skipped. For example, when the execution order is LAST_ONLY, the older instances are skipped. + * ERROR - There was error while executing this instance on the workflow engine. + * UNDEFINED - The status of the instance could not be determined. +---++ Examples +---+++ Rest Call +<verbatim> +GET http://localhost:15000/api/instance/list/process/SampleProcess?colo=*&start=2012-04-03T07:00Z +</verbatim> +---+++ Result +<verbatim> +{ + "instances": [ + { + "details": "", + "endTime": "2013-10-21T14:40:26-07:00", + "startTime": "2013-10-21T14:39:56-07:00", + "cluster": "primary-cluster", + "logFile": "http:\/\/localhost:11000\/oozie?job=0000070-131021115933395-oozie-rgau-W", + "status": "SUCCEEDED", + "instance": "2012-04-03T07:00Z" + } + ], + "requestId": "default\/e15bb378-d09f-4911-9df2-5334a45153d2\n", + "message": "default\/STATUS\n", + "status": "SUCCEEDED" +} +</verbatim> + + +---+++ Rest Call +<verbatim> +GET http://localhost:15000/api/instance/list/process/SampleProcess?colo=*&start=2012-04-03T07:00Z&filterBy=STATUS:SUCCEEDED,CLUSTER:primary-cluster&orderBy=startTime&offset=2&numResults=2&doAs=joe +</verbatim> +---+++ Result +<verbatim> +{ + "instances": [ + { + "details": "", + "endTime": "2013-10-21T14:40:26-07:00", + "startTime": "2013-10-21T14:39:56-07:00", + "cluster": "primary-cluster", + "logFile": "http:\/\/localhost:11000\/oozie?job=0000070-131021115933395-oozie-rgau-W", + "status": "SUCCEEDED", + "instance": "2012-04-03T07:00Z" + }, + { + "details": "", + "endTime": "2013-10-21T14:42:26-07:00", + "startTime": "2013-10-21T14:41:56-07:00", + "cluster": "primary-cluster", + "logFile": "http:\/\/localhost:11000\/oozie?job=0000070-131021115933397-oozie-rgau-W", + "status": "SUCCEEDED", + "instance": "2012-04-03T08:00Z" + }, + ], + + "requestId": "default\/e15bb378-d09f-4911-9df2-5334a45153d2\n", + "message": "default\/STATUS\n", + "status": "SUCCEEDED" +} +</verbatim> + + +---+++ Rest Call +<verbatim> +GET https://localhost:15443/api/instance/status/process/oozie-mr-process?user.name=narayan&start=2013-11-15T00:05Z&end=2013-11-15T01:00Z&colo=*&offset=0&allAttempts=true +</verbatim> +---+++ Result +<verbatim> +{ + "status":"SUCCEEDED", + "message":"default/STATUS\n", + "requestId":"default/942519651@qtp-1386909980-16 - 5b11a8ba-402b-4cc7-969c-256e0ed18ae2\n", + "instances":[ + { + "instance":"2013-11-15T00:05Z", + "status":"SUCCEEDED", + "logFile":"http://IM1948-X0:11000/oozie?job=0000010-160106121750678-oozie-oozi-W";, + "cluster":"local", + "startTime":"2016-01-06T12:39:22+05:30", + "endTime":"2016-01-06T12:40:05+05:30", + "runId":0, + "details":"", + "actions":[ + { + "action":"mr-node", + "status":"SUCCEEDED", + "logFile":"http://localhost:8088/proxy/application_1452062826344_0010/"; + } + ] + }, + { + "instance":"2013-11-15T00:05Z", + "status":"SUCCEEDED", + "logFile":"http://IM1948-X0:11000/oozie?job=0000011-160106121750678-oozie-oozi-W";, + "cluster":"local", + "startTime":"2016-01-06T12:40:27+05:30", + "endTime":"2016-01-06T12:41:05+05:30", + "runId":0, + "details":"", + "actions":[ + { + "action":"mr-node", + "status":"SUCCEEDED", + "logFile":"http://localhost:8088/proxy/application_1452062826344_0012/"; + } + ] + } + ] +} +</verbatim> http://git-wip-us.apache.org/repos/asf/falcon/blob/4612c3f7/trunk/releases/0.10/src/site/twiki/restapi/InstanceLogs.twiki ---------------------------------------------------------------------- diff --git a/trunk/releases/0.10/src/site/twiki/restapi/InstanceLogs.twiki b/trunk/releases/0.10/src/site/twiki/restapi/InstanceLogs.twiki new file mode 100644 index 0000000..1e1c98d --- /dev/null +++ b/trunk/releases/0.10/src/site/twiki/restapi/InstanceLogs.twiki @@ -0,0 +1,113 @@ +---++ GET /api/instance/logs/:entity-type/:entity-name + * <a href="#Description">Description</a> + * <a href="#Parameters">Parameters</a> + * <a href="#Results">Results</a> + * <a href="#Examples">Examples</a> + +---++ Description +Get log of a specific instance of an entity. + +---++ Parameters + * :entity-type Valid options are cluster, feed or process. + * :entity-name Name of the entity. + * start <optional param> Show instances from this date. Date format is yyyy-MM-dd'T'HH:mm'Z'. + * By default, it is set to (end - (10 * entityFrequency)). + * end <optional param> Show instances up to this date. Date format is yyyy-MM-dd'T'HH:mm'Z'. + * Default is set to now. + * colo <optional param> Colo on which the query should be run. + * runId <optional param> Run Id. + * lifecycle <optional param> Valid lifecycles for feed are Eviction/Replication(default) and for process is Execution(default). + * filterBy <optional param> Filter results by list of field:value pairs. Example: filterBy=STATUS:RUNNING,CLUSTER:primary-cluster + * Supported filter fields are STATUS, CLUSTER, SOURCECLUSTER, STARTEDAFTER. + * Query will do an AND among filterBy fields. + * orderBy <optional param> Field by which results should be ordered. + * Supports ordering by "status","startTime","endTime","cluster". + * sortOrder <optional param> Valid options are "asc" and "desc" + * offset <optional param> Show results from the offset, used for pagination. Defaults to 0. + * numResults <optional param> Number of results to show per request, used for pagination. Only integers > 0 are valid, Default is 10. + * doAs <optional query param> allows the current user to impersonate the user passed in doAs when interacting with the Falcon system. + +---++ Results +Log of specified instance. + +---++ Examples +---+++ Rest Call +<verbatim> +GET http://localhost:15000/api/instance/logs/process/SampleProcess?colo=*&start=2012-04-03T07:00Z +</verbatim> +---+++ Result +<verbatim> +{ + "instances": [ + { + "actions": [ + { + "logFile": "http:\/\/localhost:50070\/data\/apps\/falcon\/staging\/falcon\/workflows\/process\/SampleProcess\/logs\/job-2012-04-03-07-00\/000\/pig_SUCCEEDED.log", + "status": "SUCCEEDED", + "action": "pig" + } + ], + "details": "", + "endTime": "2013-10-21T14:40:26-07:00", + "startTime": "2013-10-21T14:39:56-07:00", + "cluster": "primary-cluster", + "logFile": "http:\/\/localhost:50070\/data\/apps\/falcon\/staging\/falcon\/workflows\/process\/SampleProcess\/logs\/job-2012-04-03-07-00\/000\/oozie.log", + "status": "SUCCEEDED", + "instance": "2012-04-03T07:00Z" + } + ], + "requestId": "default\/3527038e-8334-4e50-8173-76c4fa430d0b\n", + "message": "default\/STATUS\n", + "status": "SUCCEEDED" +} +</verbatim> + +---+++ Rest Call +<verbatim> +GET http://localhost:15000/api/instance/logs/process/SampleProcess?colo=*&start=2012-04-03T07:00Z&filterBy=STATUS:SUCCEEDED,CLUSTER:primary-cluster&orderBy=startTime&offset=2&numResults=2&doAs=joe +</verbatim> +---+++ Result +<verbatim> +{ + "instances": [ + { + "actions": [ + { + "logFile": "http:\/\/localhost:50070\/data\/apps\/falcon\/staging\/falcon\/workflows\/process\/SampleProcess\/logs\/job-2012-04-03-07-00\/000\/pig_SUCCEEDED.log", + "status": "SUCCEEDED", + "action": "pig" + } + ], + "details": "", + "endTime": "2013-10-21T14:40:26-07:00", + "startTime": "2013-10-21T14:39:56-07:00", + "cluster": "primary-cluster", + "logFile": "http:\/\/localhost:50070\/data\/apps\/falcon\/staging\/falcon\/workflows\/process\/SampleProcess\/logs\/job-2012-04-03-07-00\/000\/oozie.log", + "status": "SUCCEEDED", + "instance": "2012-04-03T07:00Z" + }, + { + "actions": [ + { + "logFile": "http:\/\/localhost:50070\/data\/apps\/falcon\/staging\/falcon\/workflows\/process\/SampleProcess\/logs\/job-2012-04-03-07-00\/001\/pig_SUCCEEDED.log", + "status": "SUCCEEDED", + "action": "pig" + } + ], + "details": "", + "endTime": "2013-10-21T14:42:27-07:00", + "startTime": "2013-10-21T14:41:57-07:00", + "cluster": "primary-cluster", + "logFile": "http:\/\/localhost:50070\/data\/apps\/falcon\/staging\/falcon\/workflows\/process\/SampleProcess\/logs\/job-2012-04-03-07-00\/001\/oozie.log", + "status": "SUCCEEDED", + "instance": "2012-04-03T08:00Z" + } + ], + "requestId": "default\/3527038e-8334-4e50-8173-76c4fa430d0b\n", + "message": "default\/STATUS\n", + "status": "SUCCEEDED" +} +</verbatim> + + + http://git-wip-us.apache.org/repos/asf/falcon/blob/4612c3f7/trunk/releases/0.10/src/site/twiki/restapi/InstanceParams.twiki ---------------------------------------------------------------------- diff --git a/trunk/releases/0.10/src/site/twiki/restapi/InstanceParams.twiki b/trunk/releases/0.10/src/site/twiki/restapi/InstanceParams.twiki new file mode 100644 index 0000000..7a340a5 --- /dev/null +++ b/trunk/releases/0.10/src/site/twiki/restapi/InstanceParams.twiki @@ -0,0 +1,83 @@ +---++ GET /api/instance/params/:entity-type/:entity-name + * <a href="#Description">Description</a> + * <a href="#Parameters">Parameters</a> + * <a href="#Results">Results</a> + * <a href="#Examples">Examples</a> + +---++ Description +Get the params passed to the workflow for an instance of feed/process. + +---++ Parameters + * :entity-type Valid options are cluster, feed or process. + * :entity-name Name of the entity. + * start should be the nominal time of the instance for which you want the params to be returned + * colo <optional param> Colo on which the query should be run. + * lifecycle <optional param> Valid lifecycles for feed are Eviction/Replication(default) and for process is Execution(default). + * doAs <optional query param> allows the current user to impersonate the user passed in doAs when interacting with the Falcon system. + + +---++ Results +List of instances currently running. + +---++ Examples +---+++ Rest Call +<verbatim> +http://userqa.user.com:16000/api/instance/params/process/Sample-Process?start=2014-10-01T11:00Z&colo=*&doAs=joe +</verbatim> +---+++ Result +{ + "status": "SUCCEEDED", + "message": "ua1/PARAMS\n", + "requestId": "ua1/807e9fe6-ba60-490e-b720-f8dc8b92063e\n", + "instances": [ + { + "instance": "2014-10-01T11:00Z", + "status": "RUNNING", + "logFile": "http://spyke.user.com:11000/oozie?job=0000211-141117203201940-oozie-oozi-W";, + "cluster": "sample-cluster", + "startTime": "2014-11-19T19:46:29+08:00", + "details": "", + "actions": [ + { + "action": "succeeded-post-processing", + "status": "RUNNING", + "logFile": "http://spyke.user.com:50030/jobdetails.jsp?jobid=job_201411071450_1052"; + } + ], + "params": { + "entry": {"key": "jobTracker", "value": "10.16.114.113:8021"}, + "entry":{"key":"falconInputNames","value":"IGNORE"}, + "entry":{"key":"shouldRecord","value":"false"}, + "entry":{"key":"timeStamp","value":"2014-11-19-11-46"}, + "entry":{"key":"falconInPaths","value":"IGNORE"}, + "entry":{"key":"broker.url","value":"tcp://localhost:61616"}, + "entry":{"key":"feedNames","value":"NONE"}, + "entry":{"key":"falcon.libpath","value":"/path/falcon/sample/lib"}, + "entry":{"key":"ENTITY_PATH","value":"/path/falcon/staging/falcon/workflows/process/Sample-Process/9506be19980e0e6fdb709e1baffff_1416397585511/DEFAULT"}, + "entry":{"key":"entityType","value":"process"}, + "entry":{"key":"nominalTime","value":"2014-10-01-11-00"}, + "entry":{"key":"feedInstancePaths","value":"IGNORE"}, + "entry":{"key":"oozie.bundle.application.path","value":"hdfs://10.16.104.13:8020/path/falcon/staging/falcon/workflows/process/Sample-Process/9506be19980e0e669709e1baffff_1416397585511"}, + "entry":{"key":"logDir","value":"hdfs://10.16.104.13:8020/path/falcon/staging/falcon/workflows/process/Sample-Process/logs"}, + "entry":{"key":"userWorkflowEngine","value":"oozie"}, + "entry":{"key":"broker.ttlInMins","value":"4320"}, + "entry":{"key":"oozie.use.system.libpath","value":"true"}, + "entry":{"key":"queueName","value":"reports"}, + "entry":{"key":"falconDataOperation","value":"GENERATE"}, + "entry":{"key":"oozie.wf.external.id","value":"Sample-Process/DEFAULT/2014-10-01T11:00Z"}, + "entry":{"key":"workflowEngineUrl","value":"http://10.11.100.10:11000/oozie/"}, + "entry":{"key":"userBrokerImplClass","value":"org.apache.activemq.ActiveMQConnectionFactory"}, + "entry":{"key":"ENTITY_NAME","value":"FALCON_PROCESS_DEFAULT_Sample-Process"}, + "entry":{"key":"broker.impl.class","value":"org.apache.activemq.ActiveMQConnectionFactory"}, + "entry":{"key":"userWorkflowName","value":"Sample-workflow"}, + "entry":{"key":"entityName","value":"Sample-Process"}, + "entry":{"key":"srcClusterName","value":"NA"}, + "entry":{"key":"userBrokerUrl","value":"tcp://localhost:61616?daemon=true"}, + "entry":{"key":"falconInputFeeds","value":"NONE"}, + "entry":{"key":"user.name","value":"sampleuser"}, + "entry":{"key":"threedaysback","value":"2014-09-28"}, + "entry":{"key":"userWorkflowVersion","value":"1.0"} + } + } + ] +}
