bamaer commented on code in PR #9607: URL: https://github.com/apache/hop/pull/9607#discussion_r4652983135
########## engine/src/main/java/org/apache/hop/datacatalog/CatalogFieldTypes.java: ########## @@ -0,0 +1,161 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.datacatalog; + +import java.util.List; +import java.util.Locale; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.apache.commons.lang3.StringUtils; +import org.apache.hop.core.row.IRowMeta; +import org.apache.hop.core.row.IValueMeta; +import org.apache.hop.core.row.RowMeta; +import org.apache.hop.core.row.value.ValueMetaBigNumber; +import org.apache.hop.core.row.value.ValueMetaBinary; +import org.apache.hop.core.row.value.ValueMetaBoolean; +import org.apache.hop.core.row.value.ValueMetaDate; +import org.apache.hop.core.row.value.ValueMetaInteger; +import org.apache.hop.core.row.value.ValueMetaNumber; +import org.apache.hop.core.row.value.ValueMetaString; +import org.apache.hop.core.row.value.ValueMetaTimestamp; +import org.apache.hop.datacatalog.ISchemaCapability.CatalogField; + +/** + * Turns the fields of a dataset in a data catalog into Hop field definitions, for a transform + * dialog to start from. + * + * <p>The native type comes first: {@code VARCHAR(255)} is a String of length 255, {@code + * DECIMAL(10,2)} a BigNumber with length 10 and precision 2, and the names the Hop lineage sinks + * write ({@code string}, {@code int}, {@code double}, {@code decimal}, ...) map back to the Hop + * type they came from. A native type it doesn't know falls back to the catalog's own type ({@code + * STRING}, {@code NUMBER}, ...), and anything else becomes a String. A catalog has no formats, so + * no field gets one: a date then uses Hop's default format, which is what Hop itself writes to a + * file when no format is set. + */ +public final class CatalogFieldTypes { + + /** Longest integer that still fits a Hop Integer. */ + private static final int MAX_INTEGER_DIGITS = 18; + + /** A type name with an optional length and scale: {@code DECIMAL(10, 2)}, {@code varchar(20)}. */ + private static final Pattern NATIVE_TYPE = + Pattern.compile("^\\s*([a-z][a-z0-9_ ]*)(?:\\(\\s*(\\d+)\\s*(?:,\\s*(-?\\d+)\\s*)?\\))?"); + + private CatalogFieldTypes() {} + + /** The Hop fields of a catalog schema, in its order. */ + public static IRowMeta toRowMeta(List<CatalogField> fields) { + IRowMeta rowMeta = new RowMeta(); + for (CatalogField field : fields) { + rowMeta.addValueMeta(toValueMeta(field)); + } + return rowMeta; + } + + /** The Hop field for a catalog field. */ + public static IValueMeta toValueMeta(CatalogField field) { + IValueMeta valueMeta = fromNativeType(field.name(), field.nativeType()); + if (valueMeta == null) { + valueMeta = fromCatalogType(field.name(), field.type()); + } + // A number comes with a default mask of its own; a catalog says nothing about formats + valueMeta.setConversionMask(null); + return valueMeta; + } + + private static IValueMeta fromNativeType(String name, String nativeType) { + if (StringUtils.isBlank(nativeType)) { + return null; + } + Matcher matcher = NATIVE_TYPE.matcher(nativeType.toLowerCase(Locale.ROOT)); + if (!matcher.lookingAt()) { + return null; + } + String typeName = matcher.group(1).trim(); + int length = matcher.group(2) == null ? -1 : Integer.parseInt(matcher.group(2)); Review Comment: Fixed in 978692970b: a length or scale too large for an int now means no length, and `accept()` also catches `RuntimeException`. ########## plugins/transforms/textfile/src/main/java/org/apache/hop/pipeline/transforms/fileinput/text/TextFileInputDialog.java: ########## @@ -3220,6 +3241,38 @@ private Vector<ITextFileInputField> getFields(TextFileInputMeta info, List<Strin return fields; } + /** Fills the fields from the schema a data catalog has for the first file. */ + private void getFieldsFromCatalog() { + CatalogFieldsDialog.getFields( + shell, + variables, + metadataProvider, + new CatalogDataset(CatalogDatasetType.FILE, null, null, firstFileForCatalog()), + wFields, + 1, + 2, + 3, + 5, + 6); + } + + /** + * The first file the transform reads, resolved the way lineage names it, so a folder with a + * wildcard gives the first matching file. Falls back to the first file row as entered when + * nothing matches yet. + */ + private String firstFileForCatalog() { + if (wFilenameList.getItemCount() <= 0) { + return null; + } + String[] fileRecord = wFilenameList.getItem(0); + String entered = fileRecord == null || fileRecord.length == 0 ? null : fileRecord[0]; + TextFileInputMeta meta = new TextFileInputMeta(); + getInfo(meta, true); + String[] files = meta.getFilePaths(variables); Review Comment: Fixed in 978692970b: only the first row is used. Without a wildcard there is no VFS call; with one, only that folder is listed (no subfolders), with the typed name as fallback. ########## ui/src/main/java/org/apache/hop/ui/datacatalog/CatalogFieldsDialog.java: ########## @@ -0,0 +1,196 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.ui.datacatalog; + +import java.util.List; +import org.apache.commons.lang3.StringUtils; +import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.row.IRowMeta; +import org.apache.hop.core.variables.IVariables; +import org.apache.hop.datacatalog.CatalogDataset; +import org.apache.hop.datacatalog.CatalogFieldTypes; +import org.apache.hop.datacatalog.CatalogSchemas; +import org.apache.hop.datacatalog.DataCatalog; +import org.apache.hop.datacatalog.ISchemaCapability.CatalogField; +import org.apache.hop.i18n.BaseMessages; +import org.apache.hop.metadata.api.IHopMetadataProvider; +import org.apache.hop.ui.core.PropsUi; +import org.apache.hop.ui.core.dialog.BaseDialog; +import org.apache.hop.ui.core.dialog.ErrorDialog; +import org.apache.hop.ui.core.gui.GuiCompositeWidgets; +import org.apache.hop.ui.core.widget.TableView; +import org.apache.hop.ui.pipeline.transform.BaseTransformDialog; +import org.eclipse.swt.SWT; +import org.eclipse.swt.layout.FormLayout; +import org.eclipse.swt.widgets.Button; +import org.eclipse.swt.widgets.Shell; + +/** + * Asks for a data catalog and a dataset in it, and reads the fields of that dataset, for a + * transform dialog to fill its field table with. The transform keeps its own field definitions: + * nothing is read from the catalog when the pipeline runs. + */ +public class CatalogFieldsDialog { + + private static final Class<?> PKG = CatalogFieldsDialog.class; + + /** The catalog picked last in this session, to start from the next time. */ + private static String lastCatalogName; Review Comment: Fixed in 978692970b: kept per session through `HopGui.getSessionSingleton`. ########## engine/src/main/java/org/apache/hop/datacatalog/CatalogSchemas.java: ########## @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.datacatalog; + +import java.util.List; +import org.apache.commons.lang3.StringUtils; +import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.variables.IVariables; +import org.apache.hop.datacatalog.ISchemaCapability.CatalogField; +import org.apache.hop.i18n.BaseMessages; +import org.apache.hop.metadata.api.IHopMetadataProvider; + +/** Reads the fields a data catalog has for a dataset. */ +public final class CatalogSchemas { + private static final Class<?> PKG = CatalogSchemas.class; + + private CatalogSchemas() {} + + /** + * The fields a data catalog has for a dataset. + * + * @throws HopException when there is no such catalog or dataset, or the catalog doesn't answer + */ + public static List<CatalogField> read( + IVariables variables, + IHopMetadataProvider metadataProvider, + String catalogName, + CatalogDataset dataset) + throws HopException { + String resolvedName = variables.resolve(catalogName); + if (StringUtils.isBlank(resolvedName)) { + throw new HopException(BaseMessages.getString(PKG, "CatalogSchemas.Error.NoCatalog")); + } + DataCatalog dataCatalog = metadataProvider.getSerializer(DataCatalog.class).load(resolvedName); + if (dataCatalog == null) { + throw new HopException( + BaseMessages.getString(PKG, "CatalogSchemas.Error.MissingCatalog", resolvedName)); + } + DatasetRef ref = dataset.toRef(variables, metadataProvider); + try (IDataCatalog catalog = dataCatalog.connect(variables, metadataProvider)) { + ISchemaCapability schemaCapability = + catalog + .capability(ISchemaCapability.class) + .orElseThrow( + () -> + new HopException( + BaseMessages.getString( + PKG, "CatalogSchemas.Error.NoSchemaSupport", resolvedName))); + CatalogEntityId id = known(catalog.resolve(ref), resolvedName, ref); + return known(schemaCapability.schema(id), resolvedName, ref); + } + } + + private static <T> T known(CatalogAnswer<T> answer, String catalogName, DatasetRef ref) + throws HopException { + return switch (answer) { + case CatalogAnswer.Known<T> known -> known.value(); Review Comment: Fixed in 978692970b: an empty field list is now reported as "no fields" and the dialog stays open. ########## engine/src/main/java/org/apache/hop/datacatalog/CatalogFieldTypes.java: ########## @@ -0,0 +1,161 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.datacatalog; + +import java.util.List; +import java.util.Locale; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.apache.commons.lang3.StringUtils; +import org.apache.hop.core.row.IRowMeta; +import org.apache.hop.core.row.IValueMeta; +import org.apache.hop.core.row.RowMeta; +import org.apache.hop.core.row.value.ValueMetaBigNumber; +import org.apache.hop.core.row.value.ValueMetaBinary; +import org.apache.hop.core.row.value.ValueMetaBoolean; +import org.apache.hop.core.row.value.ValueMetaDate; +import org.apache.hop.core.row.value.ValueMetaInteger; +import org.apache.hop.core.row.value.ValueMetaNumber; +import org.apache.hop.core.row.value.ValueMetaString; +import org.apache.hop.core.row.value.ValueMetaTimestamp; +import org.apache.hop.datacatalog.ISchemaCapability.CatalogField; + +/** + * Turns the fields of a dataset in a data catalog into Hop field definitions, for a transform + * dialog to start from. + * + * <p>The native type comes first: {@code VARCHAR(255)} is a String of length 255, {@code + * DECIMAL(10,2)} a BigNumber with length 10 and precision 2, and the names the Hop lineage sinks + * write ({@code string}, {@code int}, {@code double}, {@code decimal}, ...) map back to the Hop + * type they came from. A native type it doesn't know falls back to the catalog's own type ({@code + * STRING}, {@code NUMBER}, ...), and anything else becomes a String. A catalog has no formats, so + * no field gets one: a date then uses Hop's default format, which is what Hop itself writes to a + * file when no format is set. + */ +public final class CatalogFieldTypes { + + /** Longest integer that still fits a Hop Integer. */ + private static final int MAX_INTEGER_DIGITS = 18; + + /** A type name with an optional length and scale: {@code DECIMAL(10, 2)}, {@code varchar(20)}. */ + private static final Pattern NATIVE_TYPE = + Pattern.compile("^\\s*([a-z][a-z0-9_ ]*)(?:\\(\\s*(\\d+)\\s*(?:,\\s*(-?\\d+)\\s*)?\\))?"); + + private CatalogFieldTypes() {} + + /** The Hop fields of a catalog schema, in its order. */ + public static IRowMeta toRowMeta(List<CatalogField> fields) { + IRowMeta rowMeta = new RowMeta(); + for (CatalogField field : fields) { + rowMeta.addValueMeta(toValueMeta(field)); + } + return rowMeta; + } + + /** The Hop field for a catalog field. */ + public static IValueMeta toValueMeta(CatalogField field) { + IValueMeta valueMeta = fromNativeType(field.name(), field.nativeType()); + if (valueMeta == null) { + valueMeta = fromCatalogType(field.name(), field.type()); + } + // A number comes with a default mask of its own; a catalog says nothing about formats + valueMeta.setConversionMask(null); + return valueMeta; + } + + private static IValueMeta fromNativeType(String name, String nativeType) { + if (StringUtils.isBlank(nativeType)) { + return null; + } + Matcher matcher = NATIVE_TYPE.matcher(nativeType.toLowerCase(Locale.ROOT)); + if (!matcher.lookingAt()) { + return null; + } + String typeName = matcher.group(1).trim(); + int length = matcher.group(2) == null ? -1 : Integer.parseInt(matcher.group(2)); + int scale = matcher.group(3) == null ? -1 : Integer.parseInt(matcher.group(3)); + return switch (typeName) { + case "varchar", + "varchar2", + "nvarchar", + "nvarchar2", + "char", + "nchar", + "character", + "character varying", + "bpchar", + "text", + "string", + "clob", + "nclob", + "uuid" -> + withLength(new ValueMetaString(name), length, -1); + case "int", + "integer", + "bigint", + "smallint", + "tinyint", + "mediumint", + "long", + "int2", + "int4", + "int8", + "serial", + "bigserial" -> + new ValueMetaInteger(name); + case "decimal", "numeric", "number", "bignumber" -> + length > 0 && length <= MAX_INTEGER_DIGITS && scale <= 0 + ? withLength(new ValueMetaInteger(name), length, 0) + : withLength(new ValueMetaBigNumber(name), length, scale); + case "double", "double precision", "float", "float4", "float8", "real", "binary_double" -> + new ValueMetaNumber(name); + case "boolean", "bool", "bit" -> new ValueMetaBoolean(name); + case "date" -> new ValueMetaDate(name); + case "datetime", "datetime2", "smalldatetime" -> new ValueMetaDate(name); + case "time" -> new ValueMetaDate(name); + case "timestamp", "timestamptz", "timestamp with time zone", "timestamp without time zone" -> + new ValueMetaTimestamp(name); + case "binary", "varbinary", "blob", "longblob", "bytea", "bytes", "raw" -> + new ValueMetaBinary(name); + default -> null; + }; + } + + /** DataHub's generic field types, the names of SchemaFieldDataType. */ + private static IValueMeta fromCatalogType(String name, String type) { + String typeName = StringUtils.defaultString(type).trim().toUpperCase(Locale.ROOT); + return switch (typeName) { + case "NUMBER" -> new ValueMetaNumber(name); + case "BOOLEAN" -> new ValueMetaBoolean(name); + case "DATE" -> new ValueMetaDate(name); + case "TIME" -> new ValueMetaTimestamp(name); Review Comment: Fixed in 978692970b: both map to Date. ########## engine/src/main/java/org/apache/hop/datacatalog/CatalogFieldTypes.java: ########## @@ -0,0 +1,161 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.datacatalog; + +import java.util.List; +import java.util.Locale; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.apache.commons.lang3.StringUtils; +import org.apache.hop.core.row.IRowMeta; +import org.apache.hop.core.row.IValueMeta; +import org.apache.hop.core.row.RowMeta; +import org.apache.hop.core.row.value.ValueMetaBigNumber; +import org.apache.hop.core.row.value.ValueMetaBinary; +import org.apache.hop.core.row.value.ValueMetaBoolean; +import org.apache.hop.core.row.value.ValueMetaDate; +import org.apache.hop.core.row.value.ValueMetaInteger; +import org.apache.hop.core.row.value.ValueMetaNumber; +import org.apache.hop.core.row.value.ValueMetaString; +import org.apache.hop.core.row.value.ValueMetaTimestamp; +import org.apache.hop.datacatalog.ISchemaCapability.CatalogField; + +/** + * Turns the fields of a dataset in a data catalog into Hop field definitions, for a transform + * dialog to start from. + * + * <p>The native type comes first: {@code VARCHAR(255)} is a String of length 255, {@code + * DECIMAL(10,2)} a BigNumber with length 10 and precision 2, and the names the Hop lineage sinks + * write ({@code string}, {@code int}, {@code double}, {@code decimal}, ...) map back to the Hop + * type they came from. A native type it doesn't know falls back to the catalog's own type ({@code + * STRING}, {@code NUMBER}, ...), and anything else becomes a String. A catalog has no formats, so + * no field gets one: a date then uses Hop's default format, which is what Hop itself writes to a + * file when no format is set. + */ +public final class CatalogFieldTypes { + + /** Longest integer that still fits a Hop Integer. */ + private static final int MAX_INTEGER_DIGITS = 18; + + /** A type name with an optional length and scale: {@code DECIMAL(10, 2)}, {@code varchar(20)}. */ + private static final Pattern NATIVE_TYPE = + Pattern.compile("^\\s*([a-z][a-z0-9_ ]*)(?:\\(\\s*(\\d+)\\s*(?:,\\s*(-?\\d+)\\s*)?\\))?"); + + private CatalogFieldTypes() {} + + /** The Hop fields of a catalog schema, in its order. */ + public static IRowMeta toRowMeta(List<CatalogField> fields) { + IRowMeta rowMeta = new RowMeta(); + for (CatalogField field : fields) { + rowMeta.addValueMeta(toValueMeta(field)); + } + return rowMeta; + } + + /** The Hop field for a catalog field. */ + public static IValueMeta toValueMeta(CatalogField field) { + IValueMeta valueMeta = fromNativeType(field.name(), field.nativeType()); + if (valueMeta == null) { + valueMeta = fromCatalogType(field.name(), field.type()); + } + // A number comes with a default mask of its own; a catalog says nothing about formats + valueMeta.setConversionMask(null); + return valueMeta; + } + + private static IValueMeta fromNativeType(String name, String nativeType) { + if (StringUtils.isBlank(nativeType)) { + return null; + } + Matcher matcher = NATIVE_TYPE.matcher(nativeType.toLowerCase(Locale.ROOT)); + if (!matcher.lookingAt()) { + return null; + } + String typeName = matcher.group(1).trim(); + int length = matcher.group(2) == null ? -1 : Integer.parseInt(matcher.group(2)); + int scale = matcher.group(3) == null ? -1 : Integer.parseInt(matcher.group(3)); + return switch (typeName) { + case "varchar", + "varchar2", + "nvarchar", + "nvarchar2", + "char", + "nchar", + "character", + "character varying", + "bpchar", + "text", + "string", + "clob", + "nclob", + "uuid" -> + withLength(new ValueMetaString(name), length, -1); + case "int", + "integer", + "bigint", + "smallint", + "tinyint", + "mediumint", + "long", + "int2", + "int4", + "int8", + "serial", + "bigserial" -> + new ValueMetaInteger(name); + case "decimal", "numeric", "number", "bignumber" -> + length > 0 && length <= MAX_INTEGER_DIGITS && scale <= 0 + ? withLength(new ValueMetaInteger(name), length, 0) + : withLength(new ValueMetaBigNumber(name), length, scale); + case "double", "double precision", "float", "float4", "float8", "real", "binary_double" -> + new ValueMetaNumber(name); + case "boolean", "bool", "bit" -> new ValueMetaBoolean(name); Review Comment: Fixed in 978692970b: Boolean for `bit` / `bit(1)`, String of length n otherwise. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
