[ https://issues.apache.org/jira/browse/BEAM-10416?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Arzoo Khan updated BEAM-10416: ------------------------------ Description: I have dataflow job that is written using apache beam.Here I am loading the data from one table to another table the mode of table is write truncate and the table contains sensitive data so table configured with kms key. I am following this code and trying to fetch the kms key associated with table and setting this key at the time of (BigQueryIO.write) while writing data into table. [https://github.com/apache/beam/blob/49403c00499f7f87c2bf9a4c63dec8ebd68d640d/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryKmsKeyIT.java#L91-L100] {{finalOutput.apply("Write success rows to Sensitive BigQuery", BigQueryIO.writeTableRows().withoutValidation() .to(options.getTargetTable()) .withKmsKey(NestedValueProvider.of(options.getTargetTable(),new FetchingKMSKey(options.getProject())).toString()).withoutValidation() .withCreateDisposition(BigQueryIO.Write.CreateDisposition.CREATE_NEVER) .withWriteDisposition(BigQueryIO.Write.WriteDisposition.WRITE_TRUNCATE));}} {{public class FetchingKMSKey implements SerializableFunction<String, String> \{ private static final BigqueryClient BQ_CLIENT = new BigqueryClient("dataMainPipeline"); public static final Logger LOG = LoggerFactory.getLogger(FetchingKMSKey.class); Table table ; String tableName; String project; public FetchingKMSKey(String project) { this.project=project; } public String apply(String tableName) \{ String[] name= tableName.split(Pattern.quote(".")); try { table = BQ_CLIENT.getTableResource(project,name[0],name[1]); } catch (IOException e) \{ LOG.error(String.format("exception occured: %s", e.getMessage())); } catch (InterruptedException e) \{ LOG.error(String.format("exception occured: %s", e.getMessage())); } String kmsKey =table.getEncryptionConfiguration().getKmsKeyName(); return kmsKey; } }}} {{}} After creating a template when I am running the job I am getting error. {{"location" : "US", "errors" : [ \{ "message" : "The KMS key does not contain a location.", "reason" : "invalid" } ],}} was: I have dataflow job that is written using apache beam.Here I am loading the data from one table to another table the mode of table is write truncate and the table contains sensitive data so table configured with kms key. I am following this code and trying to fetch the kms key associated with table and setting this key at the time of (BigQueryIO.write) while writing data into table. [https://github.com/apache/beam/blob/49403c00499f7f87c2bf9a4c63dec8ebd68d640d/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryKmsKeyIT.java#L91-L100] {{finalOutput.apply("Write success rows to Sensitive BigQuery", BigQueryIO.writeTableRows().withoutValidation() .to(options.getTargetTable()) .withKmsKey(NestedValueProvider.of(options.getTargetTable(),new FetchingKMSKey(options.getProject())).toString()).withoutValidation() .withCreateDisposition(BigQueryIO.Write.CreateDisposition.CREATE_NEVER) .withWriteDisposition(BigQueryIO.Write.WriteDisposition.WRITE_TRUNCATE));}} {{}} {{}} {{public class FetchingKMSKey implements SerializableFunction<String, String> \{ private static final BigqueryClient BQ_CLIENT = new BigqueryClient("dataMainPipeline"); public static final Logger LOG = LoggerFactory.getLogger(FetchingKMSKey.class); Table table ; String tableName; String project; public FetchingKMSKey(String project) { this.project=project; } public String apply(String tableName) \{ String[] name= tableName.split(Pattern.quote(".")); try { table = BQ_CLIENT.getTableResource(project,name[0],name[1]); } catch (IOException e) \{ LOG.error(String.format("exception occured: %s", e.getMessage())); } catch (InterruptedException e) \{ LOG.error(String.format("exception occured: %s", e.getMessage())); } String kmsKey =table.getEncryptionConfiguration().getKmsKeyName(); return kmsKey; } }}}{{}} After creating a template when I am running the job I am getting error. {{}} {{"location" : "US", "errors" : [ \{ "message" : "The KMS key does not contain a location.", "reason" : "invalid" } ],}}{{}} I am completely stuck. anyone could help? {{}} {{}} > Getting error "message" : "The KMS key does not contain a location." while > using withKmsKey() option at runtime in dataflow job > ------------------------------------------------------------------------------------------------------------------------------- > > Key: BEAM-10416 > URL: https://issues.apache.org/jira/browse/BEAM-10416 > Project: Beam > Issue Type: Bug > Components: beam-model, runner-dataflow, test-failures > Environment: windows 10 (dataflow runner) > Reporter: Arzoo Khan > Priority: P2 > Labels: GCP > > > I have dataflow job that is written using apache beam.Here I am loading the > data from one table to another table the mode of table is write truncate and > the table contains sensitive data so table configured with kms key. I am > following this code and trying to fetch the kms key associated with table and > setting this key at the time of (BigQueryIO.write) while writing data into > table. > [https://github.com/apache/beam/blob/49403c00499f7f87c2bf9a4c63dec8ebd68d640d/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryKmsKeyIT.java#L91-L100] > > > {{finalOutput.apply("Write success rows to Sensitive BigQuery", > BigQueryIO.writeTableRows().withoutValidation() > .to(options.getTargetTable()) > .withKmsKey(NestedValueProvider.of(options.getTargetTable(),new > FetchingKMSKey(options.getProject())).toString()).withoutValidation() > > .withCreateDisposition(BigQueryIO.Write.CreateDisposition.CREATE_NEVER) > > .withWriteDisposition(BigQueryIO.Write.WriteDisposition.WRITE_TRUNCATE));}} > > > {{public class FetchingKMSKey implements SerializableFunction<String, String> > \{ > private static final BigqueryClient BQ_CLIENT = new > BigqueryClient("dataMainPipeline"); > > public static final Logger LOG = > LoggerFactory.getLogger(FetchingKMSKey.class); > Table table ; > String tableName; > String project; > public FetchingKMSKey(String project) > { > this.project=project; > } > public String apply(String tableName) > \{ > String[] name= tableName.split(Pattern.quote(".")); > try > { > table = BQ_CLIENT.getTableResource(project,name[0],name[1]); > } > catch (IOException e) > \{ > > LOG.error(String.format("exception occured: %s", e.getMessage())); > } > catch (InterruptedException e) > \{ > > LOG.error(String.format("exception occured: %s", e.getMessage())); > } > String kmsKey =table.getEncryptionConfiguration().getKmsKeyName(); > > return kmsKey; > > } > }}} > {{}} > > After creating a template when I am running the job I am getting error. > > {{"location" : "US", > "errors" : [ \{ > "message" : "The KMS key does not contain a location.", > "reason" : "invalid" > } ],}} -- This message was sent by Atlassian Jira (v8.3.4#803005)