Repository: beam Updated Branches: refs/heads/master 07e8cd5fc -> d035a345f
Updating protobuf for dataflow api Project: http://git-wip-us.apache.org/repos/asf/beam/repo Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/34d9d913 Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/34d9d913 Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/34d9d913 Branch: refs/heads/master Commit: 34d9d913fa70033c6de0799b61250c820d4336af Parents: 07e8cd5 Author: Pablo <pabl...@google.com> Authored: Thu Aug 10 12:49:50 2017 -0700 Committer: chamik...@google.com <chamik...@google.com> Committed: Fri Aug 11 12:02:20 2017 -0700 ---------------------------------------------------------------------- .../clients/dataflow/dataflow_v1b3_client.py | 259 ++++++++++++++++++- .../clients/dataflow/dataflow_v1b3_messages.py | 90 +++++-- 2 files changed, 328 insertions(+), 21 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/beam/blob/34d9d913/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/dataflow_v1b3_client.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/dataflow_v1b3_client.py b/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/dataflow_v1b3_client.py index f280217..61d0273 100644 --- a/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/dataflow_v1b3_client.py +++ b/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/dataflow_v1b3_client.py @@ -29,7 +29,7 @@ class DataflowV1b3(base_api.BaseApiClient): BASE_URL = u'https://dataflow.googleapis.com/' _PACKAGE = u'dataflow' - _SCOPES = [u'https://www.googleapis.com/auth/cloud-platform', u'https://www.googleapis.com/auth/userinfo.email'] + _SCOPES = [u'https://www.googleapis.com/auth/cloud-platform', u'https://www.googleapis.com/auth/compute', u'https://www.googleapis.com/auth/compute.readonly', u'https://www.googleapis.com/auth/userinfo.email'] _VERSION = u'v1b3' _CLIENT_ID = '1042881264118.apps.googleusercontent.com' _CLIENT_SECRET = 'x_Tw5K8nnjoRAqULM9PFAC2b' @@ -56,9 +56,11 @@ class DataflowV1b3(base_api.BaseApiClient): self.projects_jobs_messages = self.ProjectsJobsMessagesService(self) self.projects_jobs_workItems = self.ProjectsJobsWorkItemsService(self) self.projects_jobs = self.ProjectsJobsService(self) + self.projects_locations_jobs_debug = self.ProjectsLocationsJobsDebugService(self) self.projects_locations_jobs_messages = self.ProjectsLocationsJobsMessagesService(self) self.projects_locations_jobs_workItems = self.ProjectsLocationsJobsWorkItemsService(self) self.projects_locations_jobs = self.ProjectsLocationsJobsService(self) + self.projects_locations_templates = self.ProjectsLocationsTemplatesService(self) self.projects_locations = self.ProjectsLocationsService(self) self.projects_templates = self.ProjectsTemplatesService(self) self.projects = self.ProjectsService(self) @@ -235,6 +237,18 @@ class DataflowV1b3(base_api.BaseApiClient): def __init__(self, client): super(DataflowV1b3.ProjectsJobsService, self).__init__(client) self._method_configs = { + 'Aggregated': base_api.ApiMethodInfo( + http_method=u'GET', + method_id=u'dataflow.projects.jobs.aggregated', + ordered_params=[u'projectId'], + path_params=[u'projectId'], + query_params=[u'filter', u'location', u'pageSize', u'pageToken', u'view'], + relative_path=u'v1b3/projects/{projectId}/jobs:aggregated', + request_field='', + request_type_name=u'DataflowProjectsJobsAggregatedRequest', + response_type_name=u'ListJobsResponse', + supports_download=False, + ), 'Create': base_api.ApiMethodInfo( http_method=u'POST', method_id=u'dataflow.projects.jobs.create', @@ -300,6 +314,19 @@ class DataflowV1b3(base_api.BaseApiClient): self._upload_configs = { } + def Aggregated(self, request, global_params=None): + """List the jobs of a project across all regions. + + Args: + request: (DataflowProjectsJobsAggregatedRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (ListJobsResponse) The response message. + """ + config = self.GetMethodConfig('Aggregated') + return self._RunMethod( + config, request, global_params=global_params) + def Create(self, request, global_params=None): """Creates a Cloud Dataflow job. @@ -340,7 +367,7 @@ class DataflowV1b3(base_api.BaseApiClient): config, request, global_params=global_params) def List(self, request, global_params=None): - """List the jobs of a project. + """List the jobs of a project in a given region. Args: request: (DataflowProjectsJobsListRequest) input message @@ -365,6 +392,69 @@ class DataflowV1b3(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) + class ProjectsLocationsJobsDebugService(base_api.BaseApiService): + """Service class for the projects_locations_jobs_debug resource.""" + + _NAME = u'projects_locations_jobs_debug' + + def __init__(self, client): + super(DataflowV1b3.ProjectsLocationsJobsDebugService, self).__init__(client) + self._method_configs = { + 'GetConfig': base_api.ApiMethodInfo( + http_method=u'POST', + method_id=u'dataflow.projects.locations.jobs.debug.getConfig', + ordered_params=[u'projectId', u'location', u'jobId'], + path_params=[u'jobId', u'location', u'projectId'], + query_params=[], + relative_path=u'v1b3/projects/{projectId}/locations/{location}/jobs/{jobId}/debug/getConfig', + request_field=u'getDebugConfigRequest', + request_type_name=u'DataflowProjectsLocationsJobsDebugGetConfigRequest', + response_type_name=u'GetDebugConfigResponse', + supports_download=False, + ), + 'SendCapture': base_api.ApiMethodInfo( + http_method=u'POST', + method_id=u'dataflow.projects.locations.jobs.debug.sendCapture', + ordered_params=[u'projectId', u'location', u'jobId'], + path_params=[u'jobId', u'location', u'projectId'], + query_params=[], + relative_path=u'v1b3/projects/{projectId}/locations/{location}/jobs/{jobId}/debug/sendCapture', + request_field=u'sendDebugCaptureRequest', + request_type_name=u'DataflowProjectsLocationsJobsDebugSendCaptureRequest', + response_type_name=u'SendDebugCaptureResponse', + supports_download=False, + ), + } + + self._upload_configs = { + } + + def GetConfig(self, request, global_params=None): + """Get encoded debug configuration for component. Not cacheable. + + Args: + request: (DataflowProjectsLocationsJobsDebugGetConfigRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (GetDebugConfigResponse) The response message. + """ + config = self.GetMethodConfig('GetConfig') + return self._RunMethod( + config, request, global_params=global_params) + + def SendCapture(self, request, global_params=None): + """Send encoded debug capture data for component. + + Args: + request: (DataflowProjectsLocationsJobsDebugSendCaptureRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (SendDebugCaptureResponse) The response message. + """ + config = self.GetMethodConfig('SendCapture') + return self._RunMethod( + config, request, global_params=global_params) + class ProjectsLocationsJobsMessagesService(base_api.BaseApiService): """Service class for the projects_locations_jobs_messages resource.""" @@ -579,7 +669,7 @@ class DataflowV1b3(base_api.BaseApiClient): config, request, global_params=global_params) def List(self, request, global_params=None): - """List the jobs of a project. + """List the jobs of a project in a given region. Args: request: (DataflowProjectsLocationsJobsListRequest) input message @@ -604,6 +694,94 @@ class DataflowV1b3(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) + class ProjectsLocationsTemplatesService(base_api.BaseApiService): + """Service class for the projects_locations_templates resource.""" + + _NAME = u'projects_locations_templates' + + def __init__(self, client): + super(DataflowV1b3.ProjectsLocationsTemplatesService, self).__init__(client) + self._method_configs = { + 'Create': base_api.ApiMethodInfo( + http_method=u'POST', + method_id=u'dataflow.projects.locations.templates.create', + ordered_params=[u'projectId', u'location'], + path_params=[u'location', u'projectId'], + query_params=[], + relative_path=u'v1b3/projects/{projectId}/locations/{location}/templates', + request_field=u'createJobFromTemplateRequest', + request_type_name=u'DataflowProjectsLocationsTemplatesCreateRequest', + response_type_name=u'Job', + supports_download=False, + ), + 'Get': base_api.ApiMethodInfo( + http_method=u'GET', + method_id=u'dataflow.projects.locations.templates.get', + ordered_params=[u'projectId', u'location'], + path_params=[u'location', u'projectId'], + query_params=[u'gcsPath', u'view'], + relative_path=u'v1b3/projects/{projectId}/locations/{location}/templates:get', + request_field='', + request_type_name=u'DataflowProjectsLocationsTemplatesGetRequest', + response_type_name=u'GetTemplateResponse', + supports_download=False, + ), + 'Launch': base_api.ApiMethodInfo( + http_method=u'POST', + method_id=u'dataflow.projects.locations.templates.launch', + ordered_params=[u'projectId', u'location'], + path_params=[u'location', u'projectId'], + query_params=[u'gcsPath', u'validateOnly'], + relative_path=u'v1b3/projects/{projectId}/locations/{location}/templates:launch', + request_field=u'launchTemplateParameters', + request_type_name=u'DataflowProjectsLocationsTemplatesLaunchRequest', + response_type_name=u'LaunchTemplateResponse', + supports_download=False, + ), + } + + self._upload_configs = { + } + + def Create(self, request, global_params=None): + """Creates a Cloud Dataflow job from a template. + + Args: + request: (DataflowProjectsLocationsTemplatesCreateRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (Job) The response message. + """ + config = self.GetMethodConfig('Create') + return self._RunMethod( + config, request, global_params=global_params) + + def Get(self, request, global_params=None): + """Get the template associated with a template. + + Args: + request: (DataflowProjectsLocationsTemplatesGetRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (GetTemplateResponse) The response message. + """ + config = self.GetMethodConfig('Get') + return self._RunMethod( + config, request, global_params=global_params) + + def Launch(self, request, global_params=None): + """Launch a template. + + Args: + request: (DataflowProjectsLocationsTemplatesLaunchRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (LaunchTemplateResponse) The response message. + """ + config = self.GetMethodConfig('Launch') + return self._RunMethod( + config, request, global_params=global_params) + class ProjectsLocationsService(base_api.BaseApiService): """Service class for the projects_locations resource.""" @@ -612,11 +790,36 @@ class DataflowV1b3(base_api.BaseApiClient): def __init__(self, client): super(DataflowV1b3.ProjectsLocationsService, self).__init__(client) self._method_configs = { + 'WorkerMessages': base_api.ApiMethodInfo( + http_method=u'POST', + method_id=u'dataflow.projects.locations.workerMessages', + ordered_params=[u'projectId', u'location'], + path_params=[u'location', u'projectId'], + query_params=[], + relative_path=u'v1b3/projects/{projectId}/locations/{location}/WorkerMessages', + request_field=u'sendWorkerMessagesRequest', + request_type_name=u'DataflowProjectsLocationsWorkerMessagesRequest', + response_type_name=u'SendWorkerMessagesResponse', + supports_download=False, + ), } self._upload_configs = { } + def WorkerMessages(self, request, global_params=None): + """Send a worker_message to the service. + + Args: + request: (DataflowProjectsLocationsWorkerMessagesRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (SendWorkerMessagesResponse) The response message. + """ + config = self.GetMethodConfig('WorkerMessages') + return self._RunMethod( + config, request, global_params=global_params) + class ProjectsTemplatesService(base_api.BaseApiService): """Service class for the projects_templates resource.""" @@ -637,6 +840,30 @@ class DataflowV1b3(base_api.BaseApiClient): response_type_name=u'Job', supports_download=False, ), + 'Get': base_api.ApiMethodInfo( + http_method=u'GET', + method_id=u'dataflow.projects.templates.get', + ordered_params=[u'projectId'], + path_params=[u'projectId'], + query_params=[u'gcsPath', u'location', u'view'], + relative_path=u'v1b3/projects/{projectId}/templates:get', + request_field='', + request_type_name=u'DataflowProjectsTemplatesGetRequest', + response_type_name=u'GetTemplateResponse', + supports_download=False, + ), + 'Launch': base_api.ApiMethodInfo( + http_method=u'POST', + method_id=u'dataflow.projects.templates.launch', + ordered_params=[u'projectId'], + path_params=[u'projectId'], + query_params=[u'gcsPath', u'location', u'validateOnly'], + relative_path=u'v1b3/projects/{projectId}/templates:launch', + request_field=u'launchTemplateParameters', + request_type_name=u'DataflowProjectsTemplatesLaunchRequest', + response_type_name=u'LaunchTemplateResponse', + supports_download=False, + ), } self._upload_configs = { @@ -655,6 +882,32 @@ class DataflowV1b3(base_api.BaseApiClient): return self._RunMethod( config, request, global_params=global_params) + def Get(self, request, global_params=None): + """Get the template associated with a template. + + Args: + request: (DataflowProjectsTemplatesGetRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (GetTemplateResponse) The response message. + """ + config = self.GetMethodConfig('Get') + return self._RunMethod( + config, request, global_params=global_params) + + def Launch(self, request, global_params=None): + """Launch a template. + + Args: + request: (DataflowProjectsTemplatesLaunchRequest) input message + global_params: (StandardQueryParameters, default: None) global arguments + Returns: + (LaunchTemplateResponse) The response message. + """ + config = self.GetMethodConfig('Launch') + return self._RunMethod( + config, request, global_params=global_params) + class ProjectsService(base_api.BaseApiService): """Service class for the projects resource.""" http://git-wip-us.apache.org/repos/asf/beam/blob/34d9d913/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/dataflow_v1b3_messages.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/dataflow_v1b3_messages.py b/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/dataflow_v1b3_messages.py index eb88bce..e81329f 100644 --- a/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/dataflow_v1b3_messages.py +++ b/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/dataflow_v1b3_messages.py @@ -242,7 +242,6 @@ class ComputationTopology(_messages.Message): outputs: The outputs from the computation. stateFamilies: The state family values. systemStageName: The system stage name. - userStageName: The user stage name. """ computationId = _messages.StringField(1) @@ -251,7 +250,6 @@ class ComputationTopology(_messages.Message): outputs = _messages.MessageField('StreamLocation', 4, repeated=True) stateFamilies = _messages.MessageField('StateFamilyConfig', 5, repeated=True) systemStageName = _messages.StringField(6) - userStageName = _messages.StringField(7) class ConcatPosition(_messages.Message): @@ -522,6 +520,64 @@ class DataDiskAssignment(_messages.Message): vmInstance = _messages.StringField(2) +class DataflowProjectsJobsAggregatedRequest(_messages.Message): + """A DataflowProjectsJobsAggregatedRequest object. + + Enums: + FilterValueValuesEnum: The kind of filter to use. + ViewValueValuesEnum: Level of information requested in response. Default + is `JOB_VIEW_SUMMARY`. + + Fields: + filter: The kind of filter to use. + location: The location that contains this job. + pageSize: If there are many jobs, limit response to at most this many. The + actual number of jobs returned will be the lesser of max_responses and + an unspecified server-defined limit. + pageToken: Set this to the 'next_page_token' field of a previous response + to request additional results in a long list. + projectId: The project which owns the jobs. + view: Level of information requested in response. Default is + `JOB_VIEW_SUMMARY`. + """ + + class FilterValueValuesEnum(_messages.Enum): + """The kind of filter to use. + + Values: + UNKNOWN: <no description> + ALL: <no description> + TERMINATED: <no description> + ACTIVE: <no description> + """ + UNKNOWN = 0 + ALL = 1 + TERMINATED = 2 + ACTIVE = 3 + + class ViewValueValuesEnum(_messages.Enum): + """Level of information requested in response. Default is + `JOB_VIEW_SUMMARY`. + + Values: + JOB_VIEW_UNKNOWN: <no description> + JOB_VIEW_SUMMARY: <no description> + JOB_VIEW_ALL: <no description> + JOB_VIEW_DESCRIPTION: <no description> + """ + JOB_VIEW_UNKNOWN = 0 + JOB_VIEW_SUMMARY = 1 + JOB_VIEW_ALL = 2 + JOB_VIEW_DESCRIPTION = 3 + + filter = _messages.EnumField('FilterValueValuesEnum', 1) + location = _messages.StringField(2) + pageSize = _messages.IntegerField(3, variant=_messages.Variant.INT32) + pageToken = _messages.StringField(4) + projectId = _messages.StringField(5, required=True) + view = _messages.EnumField('ViewValueValuesEnum', 6) + + class DataflowProjectsJobsCreateRequest(_messages.Message): """A DataflowProjectsJobsCreateRequest object. @@ -3020,6 +3076,8 @@ class RuntimeEnvironment(_messages.Message): Fields: bypassTempDirValidation: Whether to bypass the safety checks for the job's temporary directory. Use with caution. + machineType: The machine type to use for the job. Defaults to the value + from the template if not specified. maxWorkers: The maximum number of Google Compute Engine instances to be made available to your pipeline during execution, from 1 to 1000. serviceAccountEmail: The email address of the service account to run the @@ -3032,10 +3090,11 @@ class RuntimeEnvironment(_messages.Message): """ bypassTempDirValidation = _messages.BooleanField(1) - maxWorkers = _messages.IntegerField(2, variant=_messages.Variant.INT32) - serviceAccountEmail = _messages.StringField(3) - tempLocation = _messages.StringField(4) - zone = _messages.StringField(5) + machineType = _messages.StringField(2) + maxWorkers = _messages.IntegerField(3, variant=_messages.Variant.INT32) + serviceAccountEmail = _messages.StringField(4) + tempLocation = _messages.StringField(5) + zone = _messages.StringField(6) class SendDebugCaptureRequest(_messages.Message): @@ -3702,7 +3761,7 @@ class Status(_messages.Message): user-facing error message is needed, put the localized message in the error details or localize it in the client. The optional error details may contain arbitrary information about the error. There is a predefined set of error - detail types in the package `google.rpc` which can be used for common error + detail types in the package `google.rpc` that can be used for common error conditions. # Language mapping The `Status` message is the logical representation of the error model, but it is not necessarily the actual wire format. When the `Status` message is exposed in different client libraries @@ -3715,8 +3774,8 @@ class Status(_messages.Message): If a service needs to return partial errors to the client, it may embed the `Status` in the normal response to indicate the partial errors. - Workflow errors. A typical workflow has multiple steps. Each step may - have a `Status` message for error reporting purpose. - Batch operations. If - a client uses batch request and batch response, the `Status` message + have a `Status` message for error reporting. - Batch operations. If a + client uses batch request and batch response, the `Status` message should be used directly inside batch response, one for each error sub- response. - Asynchronous operations. If an API call embeds asynchronous operation results in its response, the status of those operations should @@ -3729,7 +3788,7 @@ class Status(_messages.Message): Fields: code: The status code, which should be an enum value of google.rpc.Code. - details: A list of messages that carry the error details. There will be a + details: A list of messages that carry the error details. There is a common set of message types for APIs to use. message: A developer-facing error message, which should be in English. Any user-facing error message should be localized and sent in the @@ -4096,19 +4155,14 @@ class TemplateMetadata(_messages.Message): """Metadata describing a template. Fields: - bypassTempDirValidation: If true, will bypass the validation that the temp - directory is writable. This should only be used with templates for - pipelines that are guaranteed not to need to write to the temp - directory, which is subject to change based on the optimizer. description: Optional. A description of the template. name: Required. The name of the template. parameters: The parameters for the template. """ - bypassTempDirValidation = _messages.BooleanField(1) - description = _messages.StringField(2) - name = _messages.StringField(3) - parameters = _messages.MessageField('ParameterMetadata', 4, repeated=True) + description = _messages.StringField(1) + name = _messages.StringField(2) + parameters = _messages.MessageField('ParameterMetadata', 3, repeated=True) class TopologyConfig(_messages.Message):
