This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-hop.git
The following commit(s) were added to refs/heads/master by this push:
new a0a374b HOP-3456 - ETL metadata injection transform create parent
folder
new d2f74be Merge pull request #1186 from sramazzina/HOP-3456
a0a374b is described below
commit a0a374b3852a6ebf8adb9b72ee36c782988e60f8
Author: sergio.ramazzina <[email protected]>
AuthorDate: Mon Nov 1 22:30:41 2021 +0100
HOP-3456 - ETL metadata injection transform create parent folder
---
.../pipeline/transforms/metainject/MetaInject.java | 62 +++++++++++++++++++++-
.../transforms/metainject/MetaInjectDialog.java | 25 ++++++++-
.../transforms/metainject/MetaInjectMeta.java | 19 +++++++
.../metainject/messages/messages_en_US.properties | 9 +++-
.../metainject/messages/messages_it_IT.properties | 6 +++
5 files changed, 118 insertions(+), 3 deletions(-)
diff --git
a/plugins/transforms/metainject/src/main/java/org/apache/hop/pipeline/transforms/metainject/MetaInject.java
b/plugins/transforms/metainject/src/main/java/org/apache/hop/pipeline/transforms/metainject/MetaInject.java
index 32ad3ea..285389f 100644
---
a/plugins/transforms/metainject/src/main/java/org/apache/hop/pipeline/transforms/metainject/MetaInject.java
+++
b/plugins/transforms/metainject/src/main/java/org/apache/hop/pipeline/transforms/metainject/MetaInject.java
@@ -18,6 +18,7 @@
package org.apache.hop.pipeline.transforms.metainject;
import org.apache.commons.lang.StringUtils;
+import org.apache.commons.vfs2.FileObject;
import org.apache.hop.core.Const;
import org.apache.hop.core.IRowSet;
import org.apache.hop.core.Result;
@@ -269,10 +270,15 @@ public class MetaInject extends
BaseTransform<MetaInjectMeta, MetaInjectData>
OutputStream os = null;
try {
+
+ if (meta.isCreateParentFolder()) {
+ createParentFolder(targetFilePath);
+ }
+
os = HopVfs.getOutputStream(targetFilePath, false);
os.write(XmlHandler.getXmlHeader().getBytes(Const.XML_ENCODING));
os.write(data.pipelineMeta.getXml(this).getBytes(Const.XML_ENCODING));
- } catch (IOException e) {
+ } catch (Exception e) {
throw new HopException(
"Unable to write target file (hpl after injection) to file '" +
targetFilePath + "'", e);
} finally {
@@ -286,6 +292,60 @@ public class MetaInject extends
BaseTransform<MetaInjectMeta, MetaInjectData>
}
}
+
+ private void createParentFolder(String filename) throws Exception {
+ // Check for parent folder
+ FileObject parentfolder = null;
+
+ try {
+ // Get parent folder
+ parentfolder = HopVfs.getFileObject(filename).getParent();
+
+ if (parentfolder.exists()) {
+ if (isDetailed()) {
+ logDetailed(
+ BaseMessages.getString(
+ PKG,
+ "MetaInject.Log.ParentFolderExist",
+ HopVfs.getFriendlyURI(parentfolder)));
+ }
+ } else {
+ if (isDetailed()) {
+ logDetailed(
+ BaseMessages.getString(
+ PKG,
+ "MetaInject.Log.ParentFolderNotExist",
+ HopVfs.getFriendlyURI(parentfolder)));
+ }
+ if (meta.isCreateParentFolder()) {
+ parentfolder.createFolder();
+ if (isDetailed()) {
+ logDetailed(
+ BaseMessages.getString(
+ PKG,
+ "MetaInject.Log.ParentFolderCreated",
+ HopVfs.getFriendlyURI(parentfolder)));
+ }
+ } else {
+ throw new HopException(
+ BaseMessages.getString(
+ PKG,
+ "MetaInject.Log.ParentFolderNotExistCreateIt",
+ HopVfs.getFriendlyURI(parentfolder),
+ HopVfs.getFriendlyURI(filename)));
+ }
+ }
+ } finally {
+ if (parentfolder != null) {
+ try {
+ parentfolder.close();
+ } catch (Exception ex) {
+ // Ignore
+ }
+ }
+ }
+ }
+
/** Inject values from transforms. */
private void newInjection(String targetTransform, ITransformMeta
targetTransformMeta)
throws HopException {
diff --git
a/plugins/transforms/metainject/src/main/java/org/apache/hop/pipeline/transforms/metainject/MetaInjectDialog.java
b/plugins/transforms/metainject/src/main/java/org/apache/hop/pipeline/transforms/metainject/MetaInjectDialog.java
index dcaf434..872bc75 100644
---
a/plugins/transforms/metainject/src/main/java/org/apache/hop/pipeline/transforms/metainject/MetaInjectDialog.java
+++
b/plugins/transforms/metainject/src/main/java/org/apache/hop/pipeline/transforms/metainject/MetaInjectDialog.java
@@ -88,6 +88,9 @@ public class MetaInjectDialog extends BaseTransformDialog
implements ITransformD
protected Label wlRunConfiguration;
protected ComboVar wRunConfiguration;
+ // Create parent folder
+ protected Button wCreateParentFolder;
+
// the source transform
//
private CCombo wSourceTransform;
@@ -480,6 +483,24 @@ public class MetaInjectDialog extends BaseTransformDialog
implements ITransformD
fdTargetFile.top = new FormAttachment(wlTargetFile, margin);
wTargetFile.setLayoutData(fdTargetFile);
+ wCreateParentFolder = new Button(wOptionsComp, SWT.CHECK);
+ wCreateParentFolder.setText(
+ BaseMessages.getString(PKG,
"MetaInjectDialog.CreateParentFolder.Label"));
+ wCreateParentFolder.setToolTipText(
+ BaseMessages.getString(PKG,
"MetaInjectDialog.CreateParentFolder.Tooltip"));
+ props.setLook(wCreateParentFolder);
+ FormData fdCreateParentFolder = new FormData();
+ fdCreateParentFolder.left = new FormAttachment(0, 0);
+ fdCreateParentFolder.top = new FormAttachment(wTargetFile, margin);
+ wCreateParentFolder.setLayoutData(fdCreateParentFolder);
+ wCreateParentFolder.addSelectionListener(
+ new SelectionAdapter() {
+ @Override
+ public void widgetSelected(SelectionEvent e) {
+ metaInjectMeta.setChanged();
+ }
+ });
+
// the streaming source transform
//
Label wlStreamingSourceTransform = new Label(wOptionsComp, SWT.RIGHT);
@@ -488,7 +509,7 @@ public class MetaInjectDialog extends BaseTransformDialog
implements ITransformD
props.setLook(wlStreamingSourceTransform);
FormData fdlStreamingSourceTransform = new FormData();
fdlStreamingSourceTransform.left = new FormAttachment(0, 0);
- fdlStreamingSourceTransform.top = new FormAttachment(wTargetFile, 10);
+ fdlStreamingSourceTransform.top = new FormAttachment(wCreateParentFolder,
10);
wlStreamingSourceTransform.setLayoutData(fdlStreamingSourceTransform);
wStreamingSourceTransform = new CCombo(wOptionsComp, SWT.SINGLE | SWT.LEFT
| SWT.BORDER);
@@ -866,6 +887,7 @@ public class MetaInjectDialog extends BaseTransformDialog
implements ITransformD
}
wTargetFile.setText(Const.NVL(metaInjectMeta.getTargetFile(), ""));
+ wCreateParentFolder.setSelection(metaInjectMeta.isCreateParentFolder());
wNoExecution.setSelection(!metaInjectMeta.isNoExecution());
wStreamingSourceTransform.setText(
@@ -1068,6 +1090,7 @@ public class MetaInjectDialog extends BaseTransformDialog
implements ITransformD
}
meta.setTargetFile(wTargetFile.getText());
+ meta.setCreateParentFolder(wCreateParentFolder.getSelection());
meta.setNoExecution(!wNoExecution.getSelection());
final TransformMeta streamSourceTransform =
diff --git
a/plugins/transforms/metainject/src/main/java/org/apache/hop/pipeline/transforms/metainject/MetaInjectMeta.java
b/plugins/transforms/metainject/src/main/java/org/apache/hop/pipeline/transforms/metainject/MetaInjectMeta.java
index 7495214..b4797b4 100644
---
a/plugins/transforms/metainject/src/main/java/org/apache/hop/pipeline/transforms/metainject/MetaInjectMeta.java
+++
b/plugins/transforms/metainject/src/main/java/org/apache/hop/pipeline/transforms/metainject/MetaInjectMeta.java
@@ -77,6 +77,7 @@ public class MetaInjectMeta extends BaseTransformMeta
private static final String FILENAME = "filename";
private static final String RUN_CONFIG = "run_configuration";
private static final String TARGET_FILE = "target_file";
+ private static final String CREATE_PARENT_FOLDER = "create_parent_folder";
private static final String NO_EXECUTION = "no_execution";
private static final String SOURCE_TRANSFORM = "source_transform";
@@ -125,6 +126,8 @@ public class MetaInjectMeta extends BaseTransformMeta
private String runConfigurationName;
+ private boolean createParentFolder;
+
public MetaInjectMeta() {
super(); // allocate BaseTransformMeta
targetSourceMapping = new HashMap<>();
@@ -167,6 +170,7 @@ public class MetaInjectMeta extends BaseTransformMeta
retval.append(" ").append(XmlHandler.closeTag(SOURCE_OUTPUT_FIELDS));
retval.append(" ").append(XmlHandler.addTagValue(TARGET_FILE,
targetFile));
+ retval.append(" ").append(XmlHandler.addTagValue(CREATE_PARENT_FOLDER,
createParentFolder));
retval.append(" ").append(XmlHandler.addTagValue(NO_EXECUTION,
noExecution));
if ((streamSourceTransformName == null) && (streamSourceTransform !=
null)) {
@@ -224,6 +228,7 @@ public class MetaInjectMeta extends BaseTransformMeta
}
targetFile = XmlHandler.getTagValue(transformNode, TARGET_FILE);
+ createParentFolder =
"Y".equalsIgnoreCase(XmlHandler.getTagValue(transformNode,
CREATE_PARENT_FOLDER));
noExecution = "Y".equalsIgnoreCase(XmlHandler.getTagValue(transformNode,
NO_EXECUTION));
streamSourceTransformName = XmlHandler.getTagValue(transformNode,
STREAM_SOURCE_TRANSFORM);
@@ -464,6 +469,14 @@ public class MetaInjectMeta extends BaseTransformMeta
this.runConfigurationName = runConfigurationName;
}
+ public boolean isCreateParentFolder() {
+ return createParentFolder;
+ }
+
+ public void setCreateParentFolder(boolean createParentFolder) {
+ this.createParentFolder = createParentFolder;
+ }
+
/**
* @return The objects referenced in the transform, like a mapping, a
pipeline, a workflow, ...
*/
@@ -603,6 +616,12 @@ public class MetaInjectMeta extends BaseTransformMeta
}
@Override
+ public void setDefault() {
+ super.setDefault();
+ createParentFolder = true;
+ }
+
+ @Override
public void onTransformChange(
PipelineMeta pipelineMeta, TransformMeta oldMeta, TransformMeta newMeta)
{
for (int i = 0; i < pipelineMeta.nrPipelineHops(); i++) {
diff --git
a/plugins/transforms/metainject/src/main/resources/org/apache/hop/pipeline/transforms/metainject/messages/messages_en_US.properties
b/plugins/transforms/metainject/src/main/resources/org/apache/hop/pipeline/transforms/metainject/messages/messages_en_US.properties
index 1619a41..47974d4 100644
---
a/plugins/transforms/metainject/src/main/resources/org/apache/hop/pipeline/transforms/metainject/messages/messages_en_US.properties
+++
b/plugins/transforms/metainject/src/main/resources/org/apache/hop/pipeline/transforms/metainject/messages/messages_en_US.properties
@@ -80,6 +80,8 @@
MappingDialog.Exception.UnableToReferenceObjectId.Message=Unable to reference ob
MetaInjectDialog.SourceTransform.Label=Transform to read from:
MetaInjectDialog.StreamingSourceTransform.Label=Streaming source transform:
MetaInjectDialog.StreamingTargetTransform.Label=Streaming target transform:
+MetaInjectDialog.CreateParentFolder.Label=Create Parent folder
+MetaInjectDialog.CreateParentFolder.Tooltip=Check this if you want to create
parent folder\n when necessary. Otherwise, Apache Hop will throw an exception
when parent folder doesn''t exist.
MetaInjectDialog.FileTab.TabTitle=File
MetaInjectDialog.OptionsTab.TabTitle=Options
MetaInjectDialog.InjectTab.TabTitle=Inject Metadata
@@ -123,4 +125,9 @@ MetaInject.Injection.MAPPING_TARGET_FIELD=The name of the
target field from the
PipelineGraph.AfterInjection=After injection
MetaInjectDialog.ErrorLoadingPipeline.DialogMessage=There was an error loading
the specified pipeline\:
MetaInjectDialog.Button.EnterMapping=Enter Mapping
-MetaInjectMeta.keyword=meta,inject
\ No newline at end of file
+MetaInjectMeta.keyword=meta,inject
+MetaInject.Log.ParentFolderExist=Parent folder exists [{0}]!
+MetaInject.Log.ParentFolderNotExist=We can not find parent folder [{0}]!
+MetaInject.Log.ParentFolderCreated=Folder [{0}] was created.
+MetaInject.Log.ParentFolderNotExistCreateIt=We can not find folder [{0}]! You
need to create it before generating file [{1}].
+
diff --git
a/plugins/transforms/metainject/src/main/resources/org/apache/hop/pipeline/transforms/metainject/messages/messages_it_IT.properties
b/plugins/transforms/metainject/src/main/resources/org/apache/hop/pipeline/transforms/metainject/messages/messages_it_IT.properties
index a6cf7b4..10664b2 100644
---
a/plugins/transforms/metainject/src/main/resources/org/apache/hop/pipeline/transforms/metainject/messages/messages_it_IT.properties
+++
b/plugins/transforms/metainject/src/main/resources/org/apache/hop/pipeline/transforms/metainject/messages/messages_it_IT.properties
@@ -35,6 +35,8 @@ MetaInjectDialog.Validate.Tooltip=Valida la posizione della
pipeline ed aggiorna
MetaInjectDialog.Exception.UnableToReferenceObjectId.Title=Errore
MappingDialog.Exception.NotConnectedToRepository.Message=Non si \u00E8
connessi ad un repository\!
MetaInject.BadEncoding.Message=Codifica errata\!
+MetaInjectDialog.CreateParentFolder.Label=Crea cartella padre
+MetaInjectDialog.CreateParentFolder.Tooltip=Selezionare qua per creare la
cartella padre quando necessario.\nAltrimenti Apache Hop lancer\u00E0
un''eccezione quando non esiste la cartella padre.
MetaInjectDialog.ErrorShowingPipeline.Message=Errore durante il caricamento e
la visualizzazione della pipeline selezionata.
MetaInjectDialog.TransGroup.Label=Modello di pipeline
MetaInjectDialog.NoExecution.Label=Non eseguire la pipeline risultante
@@ -70,3 +72,7 @@ MetaInjectDialog.InjectTab.FilterString.ExpandAll=Espandi
tutti i transforms
MetaInjectDialog.InjectTab.FilterString.CollapseAll=Collassa tutti i transforms
MetaInjectDialog.SourceFieldDialog.Title=Campi sorgente
MetaInjectDialog.SourceFieldDialog.Label=Campo:
+MetaInject.Log.ParentFolderExist=Parent folder exists [{0}]!
+MetaInject.Log.ParentFolderNotExist=We can not find parent folder [{0}]!
+MetaInject.Log.ParentFolderCreated=Folder [{0}] was created.
+MetaInject.Log.ParentFolderNotExistCreateIt=We can not find folder [{0}]! You
need to create it before generating file [{1}].
