public class DemoServlet extends HttpServlet {
/*
code for getting the data from the REST api and putting it into
HashMap Collection and then creating the pipeline from that collection and
writing the pipeline into bq table */
String[] credentials = {"--project=" + projectId, "--stagingLocation=" +
staggingLocation};
PipelineOptionsFactory.Builder builder =
PipelineOptionsFactory.fromArgs(credentials);
PipelineOptions options = builder.create();
//Setting to run the pipeline in google cloud.By Changing this
parameter we can run even in local mode
options.setRunner(DataflowPipelineRunner.class);
//Creating pipeline with above options
Pipeline youTubeChannelPipeline = Pipeline.create(options);
PCollection<String> lines = null;
lines =
youTubeChannelPipeline.apply(Create.of(allValues)).setCoder(StringUtf8Coder.of());
//Transforming PCollection<String> to PCollection<TableRow> and then
writing to bq
bqRows = lines.apply(ParDo.named("Transforming String To TableRow").of(new
DoFn<String, TableRow>() {
@Override
public void processElement(ProcessContext pc) throws
Exception {
TableRow tmpRow = new TableRow();
String[] columns =
pc.element().split(YoutubeConstants.COMA_DELIMITER);
for (int i = 0; i < no_of_col; i++) {
tmpRow.set(YoutubeConstants.CHANNEL_LEVEL_HEADERS[i], columns[i]);
}
pc.output(tmpRow);
}
}));
//Writing the rows in bq table
bqRows.apply(BigQueryIO.Write
.named("Write To BQ")
.to(destination_path)
.withSchema(schema)
.withWriteDisposition(BigQueryIO.Write.WriteDisposition.WRITE_APPEND)
.withCreateDisposition(BigQueryIO.Write.CreateDisposition.CREATE_IF_NEEDED));
youTubeChannelPipeline.run();
}
On Monday, July 20, 2015 at 5:49:07 PM UTC+5:30, Hemanth Kumar wrote:
>
> Runnig dataflow program on google app engine is giving following error
>
> W 12:36:16.124 /demo
> java.lang.RuntimeException: Failed to construct instance from factory
> method
> com.google.cloud.dataflow.sdk.runners.DataflowPipelineRunner#fromOptions
> at
> com.google.cloud.dataflow.sdk.util.InstanceBuilder.buildFromMethod(InstanceBuilder.java:228)
> at
> com.google.cloud.dataflow.sdk.util.InstanceBuilder.build(InstanceBuilder.java:161)
> at
> com.google.cloud.dataflow.sdk.runners.PipelineRunner.fromOptions(PipelineRunner.java:52)
> at
> com.google.cloud.dataflow.sdk.Pipeline.create(Pipeline.java:117)
> at myapp.DemoServlet.doGet(DemoServlet.java:91)
> at javax.servlet.http.HttpServlet.service(HttpServlet.java:617)
> at javax.servlet.http.HttpServlet.service(HttpServlet.java:717)
> at
> org.mortbay.jetty.servlet.ServletHolder.handle(ServletHolder.java:511)
> at
> org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1166)
> at
> com.google.apphosting.utils.servlet.ParseBlobUploadFilter.doFilter(ParseBlobUploadFilter.java:125)
> at
> org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1157)
> at
> com.google.apphosting.runtime.jetty.SaveSessionFilter.doFilter(SaveSessionFilter.java:35)
> at
> org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1157)
> at
> com.google.apphosting.utils.servlet.JdbcMySqlConnectionCleanupFilter.doFilter(JdbcMySqlConnectionCleanupFilter.java:60)
> at
> org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1157)
> at
> com.google.apphosting.utils.servlet.TransactionCleanupFilter.doFilter(TransactionCleanupFilter.java:43)
> at
> org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1157)
> at
> org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:388)
> at
> org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:216)
> at
> org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:182)
> at
> org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:765)
> at
> org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:418)
> at
> com.google.apphosting.runtime.jetty.AppVersionHandlerMap.handle(AppVersionHandlerMap.java:257)
> at
> org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
> at org.mortbay.jetty.Server.handle(Server.java:326)
> at
> org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:542)
> at
> org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConnection.java:923)
> at
> com.google.apphosting.runtime.jetty.RpcRequestParser.parseAvailable(RpcRequestParser.java:76)
> at
> org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:404)
> at
> com.google.apphosting.runtime.jetty.JettyServletEngineAdapter.serviceRequest(JettyServletEngineAdapter.java:146)
> at
> com.google.apphosting.runtime.JavaRuntime$RequestRunnable.run(JavaRuntime.java:482)
> at
> com.google.tracing.TraceContext$TraceContextRunnable.runInContext(TraceContext.java:437)
> at
> com.google.tracing.TraceContext$TraceContextRunnable$1.run(TraceContext.java:444)
> at
> com.google.tracing.CurrentContext.runInContext(CurrentContext.java:230)
> at
> com.google.tracing.TraceContext$AbstractTraceContextCallback.runInInheritedContextNoUnref(TraceContext.java:308)
> at
> com.google.tracing.TraceContext$AbstractTraceContextCallback.runInInheritedContext(TraceContext.java:300)
> at
> com.google.tracing.TraceContext$TraceContextRunnable.run(TraceContext.java:441)
> at
> com.google.apphosting.runtime.ThreadGroupPool$PoolEntry.run(ThreadGroupPool.java:234)
> at java.lang.Thread.run(Thread.java:745)
> Caused by: java.lang.reflect.InvocationTargetException
> at
> com.google.appengine.runtime.Request.process-8f9b045d763b7321(Request.java)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:45)
> at
> com.google.cloud.dataflow.sdk.util.InstanceBuilder.buildFromMethod(InstanceBuilder.java:221)
> at
> com.google.cloud.dataflow.sdk.util.InstanceBuilder.build(InstanceBuilder.java:161)
> at
> com.google.cloud.dataflow.sdk.runners.PipelineRunner.fromOptions(PipelineRunner.java:52)
> at
> com.google.cloud.dataflow.sdk.Pipeline.create(Pipeline.java:117)
> at myapp.DemoServlet.doGet(DemoServlet.java:91)
> at javax.servlet.http.HttpServlet.service(HttpServlet.java:617)
> at javax.servlet.http.HttpServlet.service(HttpServlet.java:717)
> at
> org.mortbay.jetty.servlet.ServletHolder.handle(ServletHolder.java:511)
> at
> org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1166)
> at
> org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1157)
> at
> org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1157)
> at
> org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1157)
> at
> org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1157)
> at
> org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:388)
> at
> org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:216)
> at
> org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:182)
> at
> org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:765)
> at
> org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:418)
> at
> org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
> at org.mortbay.jetty.Server.handle(Server.java:326)
> at
> org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:542)
> at
> org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConnection.java:923)
> at
> org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:404)
> at
> com.google.tracing.TraceContext$TraceContextRunnable.runInContext(TraceContext.java:437)
> at
> com.google.tracing.TraceContext$TraceContextRunnable$1.run(TraceContext.java:444)
> at
> com.google.tracing.CurrentContext.runInContext(CurrentContext.java:230)
> at
> com.google.tracing.TraceContext$AbstractTraceContextCallback.runInInheritedContextNoUnref(TraceContext.java:308)
> at
> com.google.tracing.TraceContext$AbstractTraceContextCallback.runInInheritedContext(TraceContext.java:300)
> at
> com.google.tracing.TraceContext$TraceContextRunnable.run(TraceContext.java:441)
> ... 1 more
> Caused by: java.lang.RuntimeException: Unable to get application
> default credentials. Please see
> https://developers.google.com/accounts/docs/application-default-credentials
> for details on how to specify credentials. This version of the SDK is
> dependent on the gcloud core component version 2015.02.05 or newer to be able
> to get credentials from the currently authorized user via gcloud auth.
> at
> com.google.cloud.dataflow.sdk.util.Credentials.getCredential(Credentials.java:122)
> at
> com.google.cloud.dataflow.sdk.util.GcpCredentialFactory.getCredential(GcpCredentialFactory.java:43)
> at
> com.google.cloud.dataflow.sdk.options.GcpOptions$GcpUserCredentialsFactory.create(GcpOptions.java:185)
> at
> com.google.cloud.dataflow.sdk.options.GcpOptions$GcpUserCredentialsFactory.create(GcpOptions.java:175)
> at
> com.google.cloud.dataflow.sdk.options.ProxyInvocationHandler.getDefault(ProxyInvocationHandler.java:288)
> at
> com.google.cloud.dataflow.sdk.options.ProxyInvocationHandler.invoke(ProxyInvocationHandler.java:127)
> at com.sun.proxy.$Proxy20.getGcpCredential(Unknown Source)
> at
> com.google.cloud.dataflow.sdk.util.Transport.newDataflowClient(Transport.java:120)
> at
> com.google.cloud.dataflow.sdk.options.DataflowPipelineDebugOptions$DataflowClientFactory.create(DataflowPipelineDebugOptions.java:157)
> at
> com.google.cloud.dataflow.sdk.options.DataflowPipelineDebugOptions$DataflowClientFactory.create(DataflowPipelineDebugOptions.java:154)
> at
> com.google.cloud.dataflow.sdk.options.ProxyInvocationHandler.getDefault(ProxyInvocationHandler.java:288)
> at
> com.google.cloud.dataflow.sdk.options.ProxyInvocationHandler.invoke(ProxyInvocationHandler.java:127)
> at com.sun.proxy.$Proxy20.getDataflowClient(Unknown Source)
> at
> com.google.cloud.dataflow.sdk.runners.DataflowPipelineRunner.<init>(DataflowPipelineRunner.java:160)
> <continued in next message>
> W 12:36:16.124 <continued from previous message>
> at
> com.google.cloud.dataflow.sdk.runners.DataflowPipelineRunner.fromOptions(DataflowPipelineRunner.java:155)
> ... 34 more
> Caused by: java.io.IOException: Application Default Credentials failed
> to create the Google App Engine service account credentials class
> com.google.api.client.googleapis.extensions.appengine.auth.oauth2.AppIdentityCredential$AppEngineCredentialWrapper.
> Check that the component 'google-api-client-appengine' is deployed.
> at
> com.google.api.client.googleapis.auth.oauth2.DefaultCredentialProvider.tryGetAppEngineCredential(DefaultCredentialProvider.java:274)
> at
> com.google.api.client.googleapis.auth.oauth2.DefaultCredentialProvider.getDefaultCredentialUnsynchronized(DefaultCredentialProvider.java:159)
> at
> com.google.api.client.googleapis.auth.oauth2.DefaultCredentialProvider.getDefaultCredential(DefaultCredentialProvider.java:86)
> at
> com.google.api.client.googleapis.auth.oauth2.GoogleCredential.getApplicationDefault(GoogleCredential.java:213)
> at
> com.google.api.client.googleapis.auth.oauth2.GoogleCredential.getApplicationDefault(GoogleCredential.java:191)
> at
> com.google.cloud.dataflow.sdk.util.Credentials.getCredential(Credentials.java:120)
> ... 48 more
> Caused by: java.lang.ClassNotFoundException:
> com.google.api.client.googleapis.extensions.appengine.auth.oauth2.AppIdentityCredential$AppEngineCredentialWrapper
> at
> com.google.appengine.runtime.Request.process-8f9b045d763b7321(Request.java)
> at java.lang.Class.forName(Class.java:136)
> at
> com.google.api.client.googleapis.auth.oauth2.DefaultCredentialProvider.forName(DefaultCredentialProvider.java:208)
> at
> com.google.api.client.googleapis.auth.oauth2.DefaultCredentialProvider.tryGetAppEngineCredential(DefaultCredentialProvider.java:259)
> ... 53 more
>
>
>
--
You received this message because you are subscribed to the Google Groups
"Google App Engine" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to [email protected].
To post to this group, send email to [email protected].
Visit this group at http://groups.google.com/group/google-appengine.
To view this discussion on the web visit
https://groups.google.com/d/msgid/google-appengine/18359040-3f65-48b5-bf18-489440039aee%40googlegroups.com.
For more options, visit https://groups.google.com/d/optout.
