bamaer commented on code in PR #9607:
URL: https://github.com/apache/hop/pull/9607#discussion_r4652986257


##########
ui/src/main/java/org/apache/hop/ui/datacatalog/CatalogFieldsDialog.java:
##########
@@ -0,0 +1,196 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.ui.datacatalog;
+
+import java.util.List;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.row.IRowMeta;
+import org.apache.hop.core.variables.IVariables;
+import org.apache.hop.datacatalog.CatalogDataset;
+import org.apache.hop.datacatalog.CatalogFieldTypes;
+import org.apache.hop.datacatalog.CatalogSchemas;
+import org.apache.hop.datacatalog.DataCatalog;
+import org.apache.hop.datacatalog.ISchemaCapability.CatalogField;
+import org.apache.hop.i18n.BaseMessages;
+import org.apache.hop.metadata.api.IHopMetadataProvider;
+import org.apache.hop.ui.core.PropsUi;
+import org.apache.hop.ui.core.dialog.BaseDialog;
+import org.apache.hop.ui.core.dialog.ErrorDialog;
+import org.apache.hop.ui.core.gui.GuiCompositeWidgets;
+import org.apache.hop.ui.core.widget.TableView;
+import org.apache.hop.ui.pipeline.transform.BaseTransformDialog;
+import org.eclipse.swt.SWT;
+import org.eclipse.swt.layout.FormLayout;
+import org.eclipse.swt.widgets.Button;
+import org.eclipse.swt.widgets.Shell;
+
+/**
+ * Asks for a data catalog and a dataset in it, and reads the fields of that 
dataset, for a
+ * transform dialog to fill its field table with. The transform keeps its own 
field definitions:
+ * nothing is read from the catalog when the pipeline runs.
+ */
+public class CatalogFieldsDialog {
+
+  private static final Class<?> PKG = CatalogFieldsDialog.class;
+
+  /** The catalog picked last in this session, to start from the next time. */
+  private static String lastCatalogName;
+
+  private final Shell parent;
+  private final IVariables variables;
+  private final IHopMetadataProvider metadataProvider;
+  private final CatalogFieldsSource source;
+  private IRowMeta fields;
+
+  /**
+   * @param dataset the dataset to start from, typically the file or table of 
the transform
+   */
+  public CatalogFieldsDialog(
+      Shell parent,
+      IVariables variables,
+      IHopMetadataProvider metadataProvider,
+      CatalogDataset dataset) {
+    this.parent = parent;
+    this.variables = variables;
+    this.metadataProvider = metadataProvider;
+    this.source = new CatalogFieldsSource();
+    source.setCatalogName(defaultCatalogName(metadataProvider));
+    if (dataset != null) {
+      source.setDatasetType(dataset.getType());
+      source.setConnection(dataset.getConnection());
+      source.setSchema(dataset.getSchema());
+      source.setName(dataset.getName());
+    }
+  }
+
+  /** True when the project has a data catalog to get fields from. */
+  public static boolean isAvailable(IHopMetadataProvider metadataProvider) {
+    return !catalogNames(metadataProvider).isEmpty();
+  }
+
+  /**
+   * Fills a field table with the fields of a dataset in a data catalog. The 
person picks the
+   * catalog and dataset, and then whether to add the new fields, add all of 
them or replace the
+   * table.
+   *
+   * @param formatColumn the column of the format, or -1 when the table has 
none
+   */
+  public static void getFields(
+      Shell parent,
+      IVariables variables,
+      IHopMetadataProvider metadataProvider,
+      CatalogDataset dataset,
+      TableView table,
+      int nameColumn,
+      int typeColumn,
+      int formatColumn,
+      int lengthColumn,
+      int precisionColumn) {
+    IRowMeta rowMeta = new CatalogFieldsDialog(parent, variables, 
metadataProvider, dataset).open();
+    if (rowMeta == null) {
+      return;
+    }
+    BaseTransformDialog.getFieldsFromPrevious(
+        rowMeta,
+        table,
+        nameColumn,
+        new int[] {nameColumn},
+        new int[] {typeColumn},
+        lengthColumn,
+        precisionColumn,
+        true,
+        (item, valueMeta) -> {
+          if (formatColumn > 0 && valueMeta.isDate()) {

Review Comment:
   Dropped the parameter in 978692970b.



##########
plugins/transforms/textfile/src/test/java/org/apache/hop/pipeline/transforms/csvinput/GetFieldsFromCatalogDialogTest.java:
##########
@@ -0,0 +1,462 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hop.pipeline.transforms.csvinput;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.lang.reflect.Field;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+import java.util.function.Function;
+import org.apache.hop.core.fileinput.InputFile;
+import org.apache.hop.core.gui.plugin.GuiRegistry;
+import org.apache.hop.core.gui.plugin.GuiWidgetElement;
+import org.apache.hop.core.plugins.PluginRegistry;
+import org.apache.hop.core.plugins.TransformPluginType;
+import org.apache.hop.core.variables.IVariables;
+import org.apache.hop.core.variables.Variables;
+import org.apache.hop.datacatalog.CatalogAnswer;
+import org.apache.hop.datacatalog.CatalogEntityId;
+import org.apache.hop.datacatalog.DataCatalog;
+import org.apache.hop.datacatalog.DatasetRef;
+import org.apache.hop.datacatalog.IDataCatalog;
+import org.apache.hop.datacatalog.ISchemaCapability;
+import org.apache.hop.i18n.BaseMessages;
+import org.apache.hop.metadata.api.IHopMetadataProvider;
+import org.apache.hop.metadata.serializer.memory.MemoryMetadataProvider;
+import org.apache.hop.metadata.serializer.multi.MultiMetadataProvider;
+import org.apache.hop.pipeline.PipelineMeta;
+import org.apache.hop.pipeline.transform.ITransformMeta;
+import org.apache.hop.pipeline.transform.TransformMeta;
+import org.apache.hop.pipeline.transforms.fileinput.text.TextFileInputDialog;
+import org.apache.hop.pipeline.transforms.fileinput.text.TextFileInputMeta;
+import org.apache.hop.ui.core.widget.TableView;
+import org.apache.hop.ui.datacatalog.CatalogFieldsDialog;
+import org.apache.hop.ui.datacatalog.CatalogFieldsSource;
+import org.apache.hop.ui.hopgui.HopGui;
+import org.apache.hop.ui.hopgui.HopGuiImpl;
+import org.apache.hop.ui.testing.SwtBotTestBase;
+import org.eclipse.swt.custom.CTabFolder;
+import org.eclipse.swt.custom.CTabItem;
+import org.eclipse.swt.widgets.Composite;
+import org.eclipse.swt.widgets.Control;
+import org.eclipse.swt.widgets.Shell;
+import org.eclipse.swt.widgets.TableItem;
+import org.eclipse.swtbot.swt.finder.SWTBot;
+import org.eclipse.swtbot.swt.finder.utils.SWTUtils;
+import org.eclipse.swtbot.swt.finder.waits.DefaultCondition;
+import org.eclipse.swtbot.swt.finder.widgets.SWTBotShell;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Tag;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+/**
+ * The CSV file input and Text file input dialogs fill their field table from 
the schema a data
+ * catalog has for their file.
+ */
+@Tag("uitest")
+class GetFieldsFromCatalogDialogTest extends SwtBotTestBase {
+
+  private static final String FILE = "/data/in/customers.csv";
+
+  private static final String CATALOG_TITLE =
+      BaseMessages.getString(CatalogFieldsDialog.class, 
"CatalogFieldsDialog.Title");
+
+  /** Name, type, format, length and precision of the three fields of the 
dataset. */
+  private static final List<List<String>> EXPECTED =
+      List.of(
+          List.of("id", "Integer", "", "", "0"),
+          List.of("email", "String", "", "255", ""),
+          List.of("signed_up", "Date", "", "", ""));
+
+  private static MemoryMetadataProvider metadataProvider;
+
+  /** The annotated widgets are looked up by class name; register them as the 
plugin scan does. */
+  @BeforeAll
+  static void registerWidgets() {
+    GuiRegistry registry = GuiRegistry.getInstance();
+    if (registry.findGuiElements(CatalogFieldsSource.class.getName(), 
CatalogFieldsSource.PARENT_ID)
+        != null) {
+      return;
+    }
+    for (Field field : CatalogFieldsSource.class.getDeclaredFields()) {
+      GuiWidgetElement element = field.getAnnotation(GuiWidgetElement.class);
+      if (element != null) {
+        registry.addGuiWidgetElement(CatalogFieldsSource.class.getName(), 
element, field);
+      }
+    }
+  }
+
+  /** The metadata selection lines read their provider from the running Hop 
GUI. */
+  @BeforeAll
+  static void installHopGui() throws Exception {
+    metadataProvider = new MemoryMetadataProvider();
+    metadataProvider
+        .getSerializer(DataCatalog.class)
+        .save(new DataCatalog("catalog", null, new SchemaCatalog()));
+    Variables variables = new Variables();
+    HopGui hopGui = mock(HopGui.class);
+    when(hopGui.getMetadataProvider())
+        .thenReturn(new MultiMetadataProvider(variables, metadataProvider));
+    when(hopGui.getVariables()).thenReturn(variables);
+    setHopGui(hopGui);
+  }
+
+  @AfterAll
+  static void removeHopGui() throws Exception {
+    setHopGui(null);
+  }
+
+  private static void setHopGui(HopGui hopGui) throws Exception {
+    Field instance = HopGuiImpl.class.getDeclaredField("instance");
+    instance.setAccessible(true);
+    instance.set(null, hopGui);
+  }
+
+  @BeforeEach
+  void forget() {
+    SchemaCatalog.asked = null;
+  }
+
+  @Test
+  void csvFileInputFillsItsFieldsFromTheCatalog() {
+    CsvInputMeta meta = new CsvInputMeta();
+    meta.setDefault();
+    meta.setFilename(FILE);
+    PipelineMeta pipelineMeta = pipelineWith(meta, "CSV file input");
+    List<List<String>> rows = new ArrayList<>();
+
+    withDialog(
+        parent -> new CsvInputDialog(parent, new Variables(), meta, 
pipelineMeta).open(),
+        bot -> {
+          SWTBotShell dialog =
+              bot.shell(BaseMessages.getString(CsvInput.class, 
"CsvInputDialog.Shell.Title"));
+          dialog.activate();
+          dialog
+              .bot()
+              .button(
+                  BaseMessages.getString(
+                      CsvInput.class, 
"CsvInputDialog.GetFieldsFromCatalog.Button"))
+              .click();
+          getFieldsFromCatalog(bot, "csv");
+          rows.addAll(readFields(bot, dialog, shell -> onlyTable(shell)));
+          dialog.bot().button(buttonLabel("System.Button.Cancel")).click();
+        });
+
+    assertEquals(EXPECTED, rows);
+    assertNotNull(SchemaCatalog.asked, "the catalog was not asked");
+    assertTrue(
+        SchemaCatalog.asked.name().endsWith(FILE),
+        "the dataset starts as the file: " + SchemaCatalog.asked);
+  }
+
+  @Test
+  void textFileInputFillsItsFieldsFromTheCatalog() {
+    TextFileInputMeta meta = new TextFileInputMeta();
+    meta.setDefault();
+    InputFile file = new InputFile();
+    file.setFileName(FILE);
+    meta.getFileInput().getInputFiles().add(file);
+    PipelineMeta pipelineMeta = pipelineWith(meta, "Text file input");
+    String fieldsTab =
+        BaseMessages.getString(TextFileInputMeta.class, 
"TextFileInputDialog.FieldsTab.TabTitle");
+    List<List<String>> rows = new ArrayList<>();
+
+    withDialog(
+        parent -> new TextFileInputDialog(parent, new Variables(), meta, 
pipelineMeta).open(),
+        bot -> {
+          SWTBotShell dialog =
+              bot.shell(
+                  BaseMessages.getString(
+                      TextFileInputMeta.class, 
"TextFileInputDialog.DialogTitle"));
+          dialog.activate();
+          display.syncExec(() -> selectTab(dialog.widget, fieldsTab));
+          dialog
+              .bot()
+              .button(
+                  BaseMessages.getString(
+                      TextFileInputMeta.class, 
"TextFileInputDialog.GetFieldsFromCatalog.Button"))
+              .click();
+          getFieldsFromCatalog(bot, "text");
+          rows.addAll(readFields(bot, dialog, shell -> tableOnTab(shell, 
fieldsTab)));
+          dialog.bot().button(buttonLabel("System.Button.Cancel")).click();
+        });
+
+    // Text file input has a position column between the format and the length
+    assertEquals(EXPECTED, rows);
+    assertTrue(SchemaCatalog.asked.name().endsWith(FILE), 
String.valueOf(SchemaCatalog.asked));
+  }
+
+  @Test
+  void csvFileInputDisablesTheCatalogButtonWhenItIgnoresTheFields() {
+    CsvInputMeta meta = new CsvInputMeta();
+    meta.setDefault();
+    meta.setFilename(FILE);
+    meta.setIgnoreFields(true);
+    PipelineMeta pipelineMeta = pipelineWith(meta, "CSV file input");
+    boolean[] enabled = new boolean[1];
+
+    withDialog(
+        parent -> new CsvInputDialog(parent, new Variables(), meta, 
pipelineMeta).open(),
+        bot -> {
+          SWTBotShell dialog =
+              bot.shell(BaseMessages.getString(CsvInput.class, 
"CsvInputDialog.Shell.Title"));
+          dialog.activate();
+          enabled[0] =
+              dialog
+                  .bot()
+                  .button(
+                      BaseMessages.getString(
+                          CsvInput.class, 
"CsvInputDialog.GetFieldsFromCatalog.Button"))
+                  .isEnabled();
+          dialog.bot().button(buttonLabel("System.Button.Cancel")).click();
+        });
+
+    assertFalse(enabled[0], "the button does nothing while the fields are 
ignored");
+  }
+
+  @Test
+  void textFileInputLooksUpTheFirstFileAFolderWildcardMatches(@TempDir Path 
folder)
+      throws Exception {
+    Files.writeString(folder.resolve("customers.csv"), "id;email;signed_up\n");

Review Comment:
   Done in 978692970b.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to