sapienza88 commented on code in PR #823:
URL: https://github.com/apache/incubator-xtable/pull/823#discussion_r4148935261
##########
xtable-utilities/src/main/java/org/apache/xtable/utilities/RunSync.java:
##########
@@ -155,38 +165,70 @@ static List<TargetTable> targetTableBuilder(
return targetTables;
}
- static void syncTableMetdata(
+ static void syncTableMetadata(
DatasetConfig datasetConfig,
List<String> tableFormatList,
CatalogConfig catalogConfig,
Configuration hadoopConf,
- ConversionSourceProvider conversionSourceProvider) {
+ ConversionSourceProvider conversionSourceProvider,
+ long timeoutInSeconds) {
ConversionController conversionController = new
ConversionController(hadoopConf);
- for (DatasetConfig.Table table : datasetConfig.getDatasets()) {
- log.info(
- "Running sync for basePath {} for following table formats {}",
- table.getTableBasePath(),
- tableFormatList);
- Properties sourceProperties = new Properties();
- if (table.getPartitionSpec() != null) {
- sourceProperties.put(
- HudiSourceConfig.PARTITION_FIELD_SPEC_CONFIG,
table.getPartitionSpec());
- }
+ // use a single-thread executor since tasks are processed sequentially
+ ExecutorService syncExecutor = Executors.newSingleThreadExecutor();
+ try {
- SourceTable sourceTable =
- sourceTableBuilder(table, catalogConfig, datasetConfig,
sourceProperties);
- List<TargetTable> targetTables = targetTableBuilder(table,
catalogConfig, tableFormatList);
- ConversionConfig conversionConfig =
- ConversionConfig.builder()
- .sourceTable(sourceTable)
- .targetTables(targetTables)
- .syncMode(SyncMode.INCREMENTAL)
- .build();
- try {
- conversionController.sync(conversionConfig, conversionSourceProvider);
- } catch (Exception e) {
- log.error("Error running sync for {}", table.getTableBasePath(), e);
+ for (DatasetConfig.Table table : datasetConfig.getDatasets()) {
+ log.info(
+ "Running sync for basePath {} for following table formats {}",
+ table.getTableBasePath(),
+ tableFormatList);
+ Properties sourceProperties = new Properties();
+ if (table.getPartitionSpec() != null) {
+ sourceProperties.put(
+ HudiSourceConfig.PARTITION_FIELD_SPEC_CONFIG,
table.getPartitionSpec());
+ }
+
+ SourceTable sourceTable =
+ sourceTableBuilder(table, catalogConfig, datasetConfig,
sourceProperties);
+ List<TargetTable> targetTables = targetTableBuilder(table,
catalogConfig, tableFormatList);
+ ConversionConfig conversionConfig =
+ ConversionConfig.builder()
+ .sourceTable(sourceTable)
+ .targetTables(targetTables)
+ .syncMode(SyncMode.INCREMENTAL)
+ .build();
+ if (timeoutInSeconds > 0) {
+ Future<?> standardFuture =
+ syncExecutor.submit(
+ () -> {
+ conversionController.sync(conversionConfig,
conversionSourceProvider);
+ });
+ try {
+ standardFuture.get(timeoutInSeconds, TimeUnit.SECONDS);
+ } catch (TimeoutException e) {
+ log.error(
+ "Sync timed out for {} after {} seconds. Triggering thread
interrupt.",
+ table.getTableBasePath(),
+ timeoutInSeconds);
+ standardFuture.cancel(true);
Review Comment:
can you be more specific and brief, what should be changed? (applies for all
your reviews). thanks
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
