This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/master by this push:
new a9450cf99d Dimension lookup/update does not load/store "type of return
field" in read only mode #2221
new bdecfb89f5 Merge pull request #2246 from nadment/2221
a9450cf99d is described below
commit a9450cf99dd9fe11c710da7e81a03f292d17fd2e
Author: Nicolas Adment <[email protected]>
AuthorDate: Mon Feb 6 21:35:25 2023 +0100
Dimension lookup/update does not load/store "type of return field" in
read only mode #2221
Before v2.2 ReturnType is stored in update property but are never used,
for now I only fix the error. In futur we could remove return type or
use it to change the ValueMeta.
---
.../dimensionlookup/DimensionLookup.java | 32 ++++++++-----
.../dimensionlookup/DimensionLookupDialog.java | 14 ++++--
.../dimensionlookup/DimensionLookupMeta.java | 53 ++++++++++++++--------
.../messages/messages_en_US.properties | 1 +
.../dimensionlookup/DimensionLookupMetaTest.java | 14 +++++-
.../test/resources/dimension-lookup-transform.xml | 14 +++---
...ransform.xml => dimension-update-transform.xml} | 0
7 files changed, 82 insertions(+), 46 deletions(-)
diff --git
a/plugins/transforms/dimensionlookup/src/main/java/org/apache/hop/pipeline/transforms/dimensionlookup/DimensionLookup.java
b/plugins/transforms/dimensionlookup/src/main/java/org/apache/hop/pipeline/transforms/dimensionlookup/DimensionLookup.java
index b9f807885d..36231f8c8f 100644
---
a/plugins/transforms/dimensionlookup/src/main/java/org/apache/hop/pipeline/transforms/dimensionlookup/DimensionLookup.java
+++
b/plugins/transforms/dimensionlookup/src/main/java/org/apache/hop/pipeline/transforms/dimensionlookup/DimensionLookup.java
@@ -142,17 +142,11 @@ public class DimensionLookup extends
BaseTransform<DimensionLookupMeta, Dimensio
data.fieldnrs = new int[f.getFields().size()];
for (int i = 0; i < data.fieldnrs.length; i++) {
DLField field = f.getFields().get(i);
- DimensionUpdateType updateType = field.getUpdateType();
- if (updateType == null) {
- throw new HopTransformException(
- "Please specify an update type for field nr " + (i + 1) + " : "
+ field.getName());
- }
- if (meta.isUpdate() && updateType.isWithArgument()) {
+ if (isLookupOrUpdateTypeWithArgument(meta.isUpdate(), field)) {
data.fieldnrs[i] = data.outputRowMeta.indexOfValue(field.getName());
if (data.fieldnrs[i] < 0) {
- throw new HopTransformException(
- BaseMessages.getString(
- PKG, "DimensionLookup.Exception.KeyFieldNotFound",
field.getName()));
+ throw new HopTransformException(BaseMessages.getString(PKG,
+ "DimensionLookup.Exception.KeyFieldNotFound",
field.getName()));
}
} else {
data.fieldnrs[i] = -1;
@@ -227,6 +221,21 @@ public class DimensionLookup extends
BaseTransform<DimensionLookupMeta, Dimensio
return true;
}
+ public boolean isLookupOrUpdateTypeWithArgument(final boolean update, final
DLField field) throws HopTransformException {
+ // Lookup
+ if (!update) {
+ return true;
+ }
+
+ // Update type field
+ DimensionUpdateType updateType = field.getUpdateType();
+ if (updateType == null) {
+ throw new HopTransformException(BaseMessages.getString(
+ PKG, "DimensionLookup.Exception.MissingUpdateTypeField ",
field.getName()));
+ }
+ return updateType.isWithArgument();
+ }
+
private Date determineDimensionUpdatedDate(Object[] row) throws HopException
{
if (data.datefieldnr < 0) {
return getPipeline().getExecutionStartDate(); // start of pipeline...
@@ -822,7 +831,7 @@ public class DimensionLookup extends
BaseTransform<DimensionLookupMeta, Dimensio
* table: dimension table keys[]: which dim-fields do we use to look up key?
retval: name of the
* key to return datefield: do we have a datefield? datefrom, dateto:
date-range, if any.
*/
- private void setDimLookup(IRowMeta rowMeta) throws HopDatabaseException {
+ private void setDimLookup(IRowMeta rowMeta) throws HopException {
DLFields f = meta.getFields();
DatabaseMeta databaseMeta = meta.getDatabaseMeta();
@@ -847,8 +856,7 @@ public class DimensionLookup extends
BaseTransform<DimensionLookupMeta, Dimensio
for (DLField field : f.getFields()) {
// Don't retrieve the fields without input
- if (StringUtils.isNotEmpty(field.getLookup())
- && meta.isUpdate() && field.getUpdateType().isWithArgument()) {
+ if (StringUtils.isNotEmpty(field.getLookup()) &&
isLookupOrUpdateTypeWithArgument(meta.isUpdate(), field)) {
sql += ", " + databaseMeta.quoteField(field.getLookup());
if (StringUtils.isNotEmpty(field.getName()) &&
!field.getLookup().equals(field.getName())) {
diff --git
a/plugins/transforms/dimensionlookup/src/main/java/org/apache/hop/pipeline/transforms/dimensionlookup/DimensionLookupDialog.java
b/plugins/transforms/dimensionlookup/src/main/java/org/apache/hop/pipeline/transforms/dimensionlookup/DimensionLookupDialog.java
index e207c2a3fa..4f4f27743a 100644
---
a/plugins/transforms/dimensionlookup/src/main/java/org/apache/hop/pipeline/transforms/dimensionlookup/DimensionLookupDialog.java
+++
b/plugins/transforms/dimensionlookup/src/main/java/org/apache/hop/pipeline/transforms/dimensionlookup/DimensionLookupDialog.java
@@ -1108,8 +1108,7 @@ public class DimensionLookupDialog extends
BaseTransformDialog implements ITrans
DimensionUpdateType updateType = field.getUpdateType();
item.setText(3, updateType == null ? "" : updateType.getDescription());
} else {
- // String -> int -> String
- item.setText(3,
ValueMetaFactory.getValueMetaName(field.getReturnType()));
+ item.setText(3, Const.NVL(field.getReturnType(),""));
}
}
@@ -1227,10 +1226,15 @@ public class DimensionLookupDialog extends
BaseTransformDialog implements ITrans
DLField field = new DLField();
field.setLookup(item.getText(1));
field.setName(item.getText(2));
- DimensionUpdateType updateType =
DimensionUpdateType.lookupDescription(item.getText(3));
- if ( updateType!=null) {
- field.setUpdate(updateType.getCode());
+ if (in.isUpdate()) {
+ DimensionUpdateType updateType =
DimensionUpdateType.lookupDescription(item.getText(3));
+ if (updateType != null) {
+ field.setUpdate(updateType.getCode());
+ }
+ } else {
+ field.setReturnType(item.getText(3));
}
+
f.getFields().add(field);
}
if (log.isDebug()) {
diff --git
a/plugins/transforms/dimensionlookup/src/main/java/org/apache/hop/pipeline/transforms/dimensionlookup/DimensionLookupMeta.java
b/plugins/transforms/dimensionlookup/src/main/java/org/apache/hop/pipeline/transforms/dimensionlookup/DimensionLookupMeta.java
index 4dbbb17510..30087dfad5 100644
---
a/plugins/transforms/dimensionlookup/src/main/java/org/apache/hop/pipeline/transforms/dimensionlookup/DimensionLookupMeta.java
+++
b/plugins/transforms/dimensionlookup/src/main/java/org/apache/hop/pipeline/transforms/dimensionlookup/DimensionLookupMeta.java
@@ -37,7 +37,6 @@ import org.apache.hop.core.row.value.ValueMetaBoolean;
import org.apache.hop.core.row.value.ValueMetaDate;
import org.apache.hop.core.row.value.ValueMetaFactory;
import org.apache.hop.core.row.value.ValueMetaInteger;
-import org.apache.hop.core.util.Utils;
import org.apache.hop.core.variables.IVariables;
import org.apache.hop.i18n.BaseMessages;
import org.apache.hop.metadata.api.HopMetadataProperty;
@@ -643,7 +642,7 @@ public class DimensionLookupMeta extends
BaseTransformMeta<DimensionLookup, Dime
new CheckResult(
ICheckResult.TYPE_RESULT_ERROR,
"The update type specified is not valid for field '"
- + field.getName()
+ + Const.NVL(field.getName(), field.getLookup())
+ "' : '"
+ field.getUpdate()
+ "'",
@@ -653,14 +652,15 @@ public class DimensionLookupMeta extends
BaseTransformMeta<DimensionLookup, Dime
} else {
// Check the type of the dimension field to look up
//
- if (field.getReturnType() <= 0) {
+ int type = ValueMetaFactory.getIdForValueMeta(field.getReturnType());
+ if (type == IValueMeta.TYPE_NONE) {
remarks.add(
new CheckResult(
ICheckResult.TYPE_RESULT_ERROR,
"The return type specified is not valid for field '"
- + field.getName()
+ + Const.NVL(field.getName(), field.getLookup())
+ "' : '"
- + field.getUpdate()
+ + field.getReturnType()
+ "'",
transformMeta));
allOk = false;
@@ -1510,14 +1510,18 @@ public class DimensionLookupMeta extends
BaseTransformMeta<DimensionLookup, Dime
injectionKeyDescription = "DimensionLookup.Injection.UPDATE_TYPE")
private String update;
+ @HopMetadataProperty(
+ key = "type",
+ injectionKey = "TYPE_OF_RETURN_FIELD",
+ injectionKeyDescription =
"DimensionLookup.Injection.TYPE_OF_RETURN_FIELD")
+ private String returnType;
+
/** Not serialized. This is used to cache the lookup of the dimension type
*/
private DimensionUpdateType updateType;
- /** Not serialized. This is used to cache the lookup of the Hop value
return type */
- private int returnType;
public DLField() {
this.updateType = null;
- this.returnType = -1;
+ this.returnType = null;
}
public DLField(DLField f) {
@@ -1525,7 +1529,7 @@ public class DimensionLookupMeta extends
BaseTransformMeta<DimensionLookup, Dime
this.lookup = f.lookup;
this.update = f.update;
this.updateType = null;
- this.returnType = -1;
+ this.returnType = f.returnType;
}
public DimensionUpdateType getUpdateType() {
@@ -1541,14 +1545,6 @@ public class DimensionLookupMeta extends
BaseTransformMeta<DimensionLookup, Dime
return null;
}
- public int getReturnType() {
- if (returnType >= 0) {
- return returnType;
- }
- returnType = ValueMetaFactory.getIdForValueMeta(update);
- return returnType;
- }
-
/**
* Gets name
*
@@ -1586,7 +1582,7 @@ public class DimensionLookupMeta extends
BaseTransformMeta<DimensionLookup, Dime
}
/**
- * Gets update
+ * Gets update type code
*
* @return value of update
*/
@@ -1595,14 +1591,31 @@ public class DimensionLookupMeta extends
BaseTransformMeta<DimensionLookup, Dime
}
/**
- * Sets update
+ * Sets update type code
*
* @param update value of update
*/
public void setUpdate(String update) {
this.update = update;
this.updateType = null;
- this.returnType = -1;
+ }
+
+ /**
+ * Gets return type for read only lookup
+ *
+ * @return type of
+ */
+ public String getReturnType() {
+ return returnType;
+ }
+
+ /**
+ * Sets return type for read only lookup
+ *
+ * @param type the return type
+ */
+ public void setReturnType(String type) {
+ this.returnType = type;
}
}
diff --git
a/plugins/transforms/dimensionlookup/src/main/resources/org/apache/hop/pipeline/transforms/dimensionlookup/messages/messages_en_US.properties
b/plugins/transforms/dimensionlookup/src/main/resources/org/apache/hop/pipeline/transforms/dimensionlookup/messages/messages_en_US.properties
index 01af25051f..df08d5dbb6 100644
---
a/plugins/transforms/dimensionlookup/src/main/resources/org/apache/hop/pipeline/transforms/dimensionlookup/messages/messages_en_US.properties
+++
b/plugins/transforms/dimensionlookup/src/main/resources/org/apache/hop/pipeline/transforms/dimensionlookup/messages/messages_en_US.properties
@@ -188,6 +188,7 @@ DimensionLookupDialog.DateField.Label=Stream Datefield
DimensionLookupDialog.UseCache.Label=Enable the cache?
DimensionLookup.Exception.NullDimensionUpdatedDate=Invalid data - dimension
updated date cannot be null - {0}
DimensionLookup.Exception.ErrorDetectedInComparingFields=Error comparing
fields - cannot find lookup field [{0}]
+DimensionLookup.Exception.MissingUpdateTypeField=Please specify an update type
for field [{0}]
DimensionLookup.Injection.TARGET_SCHEMA=The name of the database schema to use.
DimensionLookup.Injection.TARGET_TABLE=The name of the target table to write
data to.
DimensionLookup.Injection.CONNECTION_NAME=The name of the database connection.
diff --git
a/plugins/transforms/dimensionlookup/src/test/java/org/apache/hop/pipeline/transforms/dimensionlookup/DimensionLookupMetaTest.java
b/plugins/transforms/dimensionlookup/src/test/java/org/apache/hop/pipeline/transforms/dimensionlookup/DimensionLookupMetaTest.java
index f26af94a9b..7adb82a7ad 100644
---
a/plugins/transforms/dimensionlookup/src/test/java/org/apache/hop/pipeline/transforms/dimensionlookup/DimensionLookupMetaTest.java
+++
b/plugins/transforms/dimensionlookup/src/test/java/org/apache/hop/pipeline/transforms/dimensionlookup/DimensionLookupMetaTest.java
@@ -18,7 +18,6 @@
package org.apache.hop.pipeline.transforms.dimensionlookup;
import org.apache.hop.core.HopClientEnvironment;
-import org.apache.hop.core.HopEnvironment;
import org.apache.hop.core.database.DatabaseMeta;
import org.apache.hop.metadata.serializer.memory.MemoryMetadataProvider;
import org.apache.hop.pipeline.transform.TransformSerializationTestUtil;
@@ -26,7 +25,9 @@ import org.junit.Before;
import org.junit.Test;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
public class DimensionLookupMetaTest {
@@ -44,11 +45,12 @@ public class DimensionLookupMetaTest {
DimensionLookupMeta meta =
TransformSerializationTestUtil.testSerialization(
- "/dimension-lookup-transform.xml", DimensionLookupMeta.class,
metadataProvider);
+ "/dimension-update-transform.xml", DimensionLookupMeta.class,
metadataProvider);
assertNotNull(meta.getDatabaseMeta());
assertNotNull(meta.getTableName());
assertNotNull(meta.getSchemaName());
+ assertTrue(meta.isUpdate());
assertEquals(100, meta.getCommitSize());
assertEquals(5000, meta.getCacheSize());
assertEquals(1, meta.getFields().getKeys().size());
@@ -63,5 +65,13 @@ public class DimensionLookupMetaTest {
assertEquals("lastVersionLookup",
meta.getFields().getFields().get(1).getLookup());
assertEquals("LastVersion",
meta.getFields().getFields().get(1).getUpdate());
assertEquals(DimensionLookupMeta.DimensionUpdateType.LAST_VERSION,
meta.getFields().getFields().get(1).getUpdateType());
+
+ meta =
+ TransformSerializationTestUtil.testSerialization(
+ "/dimension-lookup-transform.xml", DimensionLookupMeta.class,
metadataProvider);
+
+ assertFalse(meta.isUpdate());
+ assertEquals("Number",
meta.getFields().getFields().get(0).getReturnType());
+ assertEquals("String",
meta.getFields().getFields().get(1).getReturnType());
}
}
diff --git
a/plugins/transforms/dimensionlookup/src/test/resources/dimension-lookup-transform.xml
b/plugins/transforms/dimensionlookup/src/test/resources/dimension-lookup-transform.xml
index 6bf1a30b66..be294db0ff 100644
---
a/plugins/transforms/dimensionlookup/src/test/resources/dimension-lookup-transform.xml
+++
b/plugins/transforms/dimensionlookup/src/test/resources/dimension-lookup-transform.xml
@@ -31,7 +31,7 @@
<table>dimension</table>
<connection>unit-test-db</connection>
<commit>100</commit>
- <update>Y</update>
+ <update>N</update>
<fields>
<key>
<name>key</name>
@@ -43,14 +43,14 @@
<to>date_to</to>
</date>
<field>
- <name>value</name>
- <lookup>valueLookup</lookup>
- <update>Insert</update>
+ <name>valueNumber</name>
+ <lookup>fieldNumber</lookup>
+ <type>Number</type>
</field>
<field>
- <name>lastVersion</name>
- <lookup>lastVersionLookup</lookup>
- <update>LastVersion</update>
+ <name>valueString</name>
+ <lookup>fieldString</lookup>
+ <type>String</type>
</field>
<return>
<name>dimension_id</name>
diff --git
a/plugins/transforms/dimensionlookup/src/test/resources/dimension-lookup-transform.xml
b/plugins/transforms/dimensionlookup/src/test/resources/dimension-update-transform.xml
similarity index 100%
copy from
plugins/transforms/dimensionlookup/src/test/resources/dimension-lookup-transform.xml
copy to
plugins/transforms/dimensionlookup/src/test/resources/dimension-update-transform.xml
