Added: oodt/trunk/tools/pdi_plugin/src/org/apache/oodt/filemgringest/FilemgrIngestStep.java URL: http://svn.apache.org/viewvc/oodt/trunk/tools/pdi_plugin/src/org/apache/oodt/filemgringest/FilemgrIngestStep.java?rev=1601222&view=auto ============================================================================== --- oodt/trunk/tools/pdi_plugin/src/org/apache/oodt/filemgringest/FilemgrIngestStep.java (added) +++ oodt/trunk/tools/pdi_plugin/src/org/apache/oodt/filemgringest/FilemgrIngestStep.java Sun Jun 8 14:04:07 2014 @@ -0,0 +1,212 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.oodt.filemgringest; + +import org.apache.commons.lang.exception.ExceptionUtils; +import org.apache.oodt.cas.metadata.Metadata; +import org.pentaho.di.core.exception.KettleException; +import org.pentaho.di.core.row.RowDataUtil; +import org.pentaho.di.core.row.RowMetaInterface; +import org.pentaho.di.trans.Trans; +import org.pentaho.di.trans.TransMeta; +import org.pentaho.di.trans.step.*; + +import java.io.File; +import java.util.Arrays; + +/** + * This class is part of the demo step plug-in implementation. + * It demonstrates the basics of developing a plug-in step for PDI. + * + * The demo step adds a new string field to the row stream and sets its + * value to "Hello World!". The user may select the name of the new field. + * + * This class is the implementation of StepInterface. + * Classes implementing this interface need to: + * + * - initialize the step + * - execute the row processing logic + * - dispose of the step + * + * Please do not create any local fields in a StepInterface class. Store any + * information related to the processing logic in the supplied step data interface + * instead. + * + */ + +public class FilemgrIngestStep extends BaseStep implements StepInterface { + + private OODTConfig oodt = new OODTConfig(); + private OODTProcesses oodtproc = new OODTProcesses(); + /** + * The constructor should simply pass on its arguments to the parent class. + * + * @param s step description + * @param stepDataInterface step data class + * @param c step copy + * @param t transformation description + * @param t transformation description + * @param dis transformation executing + */ + public FilemgrIngestStep(StepMeta s, StepDataInterface stepDataInterface, int c, TransMeta t, Trans dis) { + super(s, stepDataInterface, c, t, dis); + } + + /** + * This method is called by PDI during transformation startup. + * + * It should initialize required for step execution. + * + * The meta and data implementations passed in can safely be cast + * to the step's respective implementations. + * + * It is mandatory that super.init() is called to ensure correct behavior. + * + * Typical tasks executed here are establishing the connection to a database, + * as wall as obtaining resources, like file handles. + * + * @param smi step meta interface implementation, containing the step settings + * @param sdi step data interface implementation, used to store runtime information + * + * @return true if initialization completed successfully, false if there was an error preventing the step from working. + * + */ + public boolean init(StepMetaInterface smi, StepDataInterface sdi) { + // Casting to step-specific implementation classes is safe + FilemgrIngestStepMeta meta = (FilemgrIngestStepMeta) smi; + FilemgrIngestStepData data = (FilemgrIngestStepData) sdi; + + try { + oodt.loadIngester(meta.getServerURLField()); + } catch (InstantiationException e) { + logError(e.getMessage()); + e.printStackTrace(); + } + + return super.init(meta, data); + } + + /** + * Once the transformation starts executing, the processRow() method is called repeatedly + * by PDI for as long as it returns true. To indicate that a step has finished processing rows + * this method must call setOutputDone() and return false; + * + * Steps which process incoming rows typically call getRow() to read a single row from the + * input stream, change or add row content, call putRow() to pass the changed row on + * and return true. If getRow() returns null, no more rows are expected to come in, + * and the processRow() implementation calls setOutputDone() and returns false to + * indicate that it is done too. + * + * Steps which generate rows typically construct a new row Object[] using a call to + * RowDataUtil.allocateRowData(numberOfFields), add row content, and call putRow() to + * pass the new row on. Above process may happen in a loop to generate multiple rows, + * at the end of which processRow() would call setOutputDone() and return false; + * + * @param smi the step meta interface containing the step settings + * @param sdi the step data interface that should be used to store + * + * @return true to indicate that the function should be called again, false if the step is done + */ + public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException { + + // safely cast the step settings (meta) and runtime info (data) to specific implementations + FilemgrIngestStepMeta meta = (FilemgrIngestStepMeta) smi; + FilemgrIngestStepData data = (FilemgrIngestStepData) sdi; + + // get incoming row, getRow() potentially blocks waiting for more rows, returns null if no more rows expected + Object[] r = getRow(); + + // if no more rows are expected, indicate step is finished and processRow() should not be called again + if (r == null){ + setOutputDone(); + return false; + } + + // the "first" flag is inherited from the base step implementation + // it is used to guard some processing tasks, like figuring out field indexes + // in the row structure that only need to be done once + if (first) { + first = false; + // clone the input row structure and place it in our data object + data.outputRowMeta = (RowMetaInterface) getInputRowMeta().clone(); + // use meta.getFields() to change it, so it reflects the output row structure + meta.getFields(data.outputRowMeta, getStepname(), null, null, this, null, null); + } + + String[] names = getInputRowMeta().getFieldNames(); + int idx = Arrays.asList(names).indexOf(meta.getFilenameField()); + int idx2 = Arrays.asList(names).indexOf(meta.getMetadataField()); + + + + /*try { + logError("does file exist?"+oodtproc.isAlreadyInDatabase(oodt, (String)r[idx])); + } catch (Exception e) { + e.printStackTrace(); + }*/ + + try { + Metadata m = oodtproc.getMetadata((String)r[idx2]); + oodtproc.ingest(oodt, new File((String)r[idx]), m); + + } catch (Exception e) { + logError(ExceptionUtils.getStackTrace(e)); + e.printStackTrace(); + } + + // safely add the string "Hello World!" at the end of the output row + // the row array will be resized if necessary + Object[] outputRow = RowDataUtil.addValueData(r, data.outputRowMeta.size() - 1, "Hello World!"); + + // put the row to the output row stream + putRow(data.outputRowMeta, outputRow); + + // log progress if it is time to to so + if (checkFeedback(getLinesRead())) { + logBasic("Linenr " + getLinesRead()); // Some basic logging + } + + // indicate that processRow() should be called again + return true; + } + + /** + * This method is called by PDI once the step is done processing. + * + * The dispose() method is the counterpart to init() and should release any resources + * acquired for step execution like file handles or database connections. + * + * The meta and data implementations passed in can safely be cast + * to the step's respective implementations. + * + * It is mandatory that super.dispose() is called to ensure correct behavior. + * + * @param smi step meta interface implementation, containing the step settings + * @param sdi step data interface implementation, used to store runtime information + */ + public void dispose(StepMetaInterface smi, StepDataInterface sdi) { + + // Casting to step-specific implementation classes is safe + FilemgrIngestStepMeta meta = (FilemgrIngestStepMeta) smi; + FilemgrIngestStepData data = (FilemgrIngestStepData) sdi; + + super.dispose(meta, data); + } + +}
Added: oodt/trunk/tools/pdi_plugin/src/org/apache/oodt/filemgringest/FilemgrIngestStepData.java URL: http://svn.apache.org/viewvc/oodt/trunk/tools/pdi_plugin/src/org/apache/oodt/filemgringest/FilemgrIngestStepData.java?rev=1601222&view=auto ============================================================================== --- oodt/trunk/tools/pdi_plugin/src/org/apache/oodt/filemgringest/FilemgrIngestStepData.java (added) +++ oodt/trunk/tools/pdi_plugin/src/org/apache/oodt/filemgringest/FilemgrIngestStepData.java Sun Jun 8 14:04:07 2014 @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.oodt.filemgringest; + +import org.pentaho.di.core.row.RowMetaInterface; +import org.pentaho.di.trans.step.BaseStepData; +import org.pentaho.di.trans.step.StepDataInterface; + +/** + * This class is part of the demo step plug-in implementation. + * It demonstrates the basics of developing a plug-in step for PDI. + * + * The demo step adds a new string field to the row stream and sets its + * value to "Hello World!". The user may select the name of the new field. + * + * This class is the implementation of StepDataInterface. + * + * Implementing classes inherit from BaseStepData, which implements the entire + * interface completely. + * + * In addition classes implementing this interface usually keep track of + * per-thread resources during step execution. Typical examples are: + * result sets, temporary data, caching indexes, etc. + * + * The implementation for the demo step stores the output row structure in + * the data class. + * + */ +public class FilemgrIngestStepData extends BaseStepData implements StepDataInterface { + + public RowMetaInterface outputRowMeta; + + public String fieldname; + + public FilemgrIngestStepData() + { + super(); + } +} + Added: oodt/trunk/tools/pdi_plugin/src/org/apache/oodt/filemgringest/FilemgrIngestStepDialog.java URL: http://svn.apache.org/viewvc/oodt/trunk/tools/pdi_plugin/src/org/apache/oodt/filemgringest/FilemgrIngestStepDialog.java?rev=1601222&view=auto ============================================================================== --- oodt/trunk/tools/pdi_plugin/src/org/apache/oodt/filemgringest/FilemgrIngestStepDialog.java (added) +++ oodt/trunk/tools/pdi_plugin/src/org/apache/oodt/filemgringest/FilemgrIngestStepDialog.java Sun Jun 8 14:04:07 2014 @@ -0,0 +1,333 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.oodt.filemgringest; + +import org.eclipse.swt.SWT; +import org.eclipse.swt.events.*; +import org.eclipse.swt.layout.FormAttachment; +import org.eclipse.swt.layout.FormData; +import org.eclipse.swt.layout.FormLayout; +import org.eclipse.swt.widgets.*; +import org.pentaho.di.core.Const; +import org.pentaho.di.i18n.BaseMessages; +import org.pentaho.di.trans.TransMeta; +import org.pentaho.di.trans.step.BaseStepMeta; +import org.pentaho.di.trans.step.StepDialogInterface; +import org.pentaho.di.ui.trans.step.BaseStepDialog; + +/** + * This class is part of the demo step plug-in implementation. + * It demonstrates the basics of developing a plug-in step for PDI. + * + * The demo step adds a new string field to the row stream and sets its + * value to "Hello World!". The user may select the name of the new field. + * + * This class is the implementation of StepDialogInterface. + * Classes implementing this interface need to: + * + * - build and open a SWT dialog displaying the step's settings (stored in the step's meta object) + * - write back any changes the user makes to the step's meta object + * - report whether the user changed any settings when confirming the dialog + * + */ +public class FilemgrIngestStepDialog extends BaseStepDialog implements StepDialogInterface { + + /** + * The PKG member is used when looking up internationalized strings. + * The properties file with localized keys is expected to reside in + * {the package of the class specified}/messages/messages_{locale}.properties + */ + private static Class<?> PKG = FilemgrIngestStepMeta.class; // for i18n purposes + + // this is the object the stores the step's settings + // the dialog reads the settings from it when opening + // the dialog writes the settings to it when confirmed + private FilemgrIngestStepMeta meta; + + // text field holding the name of the field to add to the row stream + //private Text wHelloFieldName; + + // text field holding the name of the field to check the filename against + private Text wFilenameField; + private Text wServerURLField; + private Text wResultField; + private Text wMetadataField; + + /** + * The constructor should simply invoke super() and save the incoming meta + * object to a local variable, so it can conveniently read and write settings + * from/to it. + * + * @param parent the SWT shell to open the dialog in + * @param in the meta object holding the step's settings + * @param transMeta transformation description + * @param sname the step name + */ + public FilemgrIngestStepDialog(Shell parent, Object in, TransMeta transMeta, String sname) { + super(parent, (BaseStepMeta) in, transMeta, sname); + meta = (FilemgrIngestStepMeta) in; + } + + /** + * This method is called by Spoon when the user opens the settings dialog of the step. + * It should open the dialog and return only once the dialog has been closed by the user. + * + * If the user confirms the dialog, the meta object (passed in the constructor) must + * be updated to reflect the new step settings. The changed flag of the meta object must + * reflect whether the step configuration was changed by the dialog. + * + * If the user cancels the dialog, the meta object must not be updated, and its changed flag + * must remain unaltered. + * + * The open() method must return the name of the step after the user has confirmed the dialog, + * or null if the user cancelled the dialog. + */ + public String open() { + + // store some convenient SWT variables + Shell parent = getParent(); + Display display = parent.getDisplay(); + + // SWT code for preparing the dialog + shell = new Shell(parent, SWT.DIALOG_TRIM | SWT.RESIZE | SWT.MIN | SWT.MAX); + props.setLook(shell); + setShellImage(shell, meta); + + // Save the value of the changed flag on the meta object. If the user cancels + // the dialog, it will be restored to this saved value. + // The "changed" variable is inherited from BaseStepDialog + changed = meta.hasChanged(); + + // The ModifyListener used on all controls. It will update the meta object to + // indicate that changes are being made. + ModifyListener lsMod = new ModifyListener() { + public void modifyText(ModifyEvent e) { + meta.setChanged(); + } + }; + + // ------------------------------------------------------- // + // SWT code for building the actual settings dialog // + // ------------------------------------------------------- // + FormLayout formLayout = new FormLayout(); + formLayout.marginWidth = Const.FORM_MARGIN; + formLayout.marginHeight = Const.FORM_MARGIN; + + shell.setLayout(formLayout); + shell.setText(BaseMessages.getString(PKG, "FilemgrIngestStep.Name")); + + int middle = props.getMiddlePct(); + int margin = Const.MARGIN; + + // Stepname line + wlStepname = new Label(shell, SWT.RIGHT); + wlStepname.setText(BaseMessages.getString(PKG, "System.Label.StepName")); + props.setLook(wlStepname); + fdlStepname = new FormData(); + fdlStepname.left = new FormAttachment(0, 0); + fdlStepname.right = new FormAttachment(middle, -margin); + fdlStepname.top = new FormAttachment(0, margin); + wlStepname.setLayoutData(fdlStepname); + + wStepname = new Text(shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + wStepname.setText(stepname); + props.setLook(wStepname); + wStepname.addModifyListener(lsMod); + fdStepname = new FormData(); + fdStepname.left = new FormAttachment(middle, 0); + fdStepname.top = new FormAttachment(0, margin); + fdStepname.right = new FormAttachment(100, 0); + wStepname.setLayoutData(fdStepname); + + // output field value + Label wlValName = new Label(shell, SWT.RIGHT); + wlValName.setText(BaseMessages.getString(PKG, "FilemgrIngest.FieldName.Label")); + props.setLook(wlValName); + FormData fdlValName = new FormData(); + fdlValName.left = new FormAttachment(0, 0); + fdlValName.right = new FormAttachment(middle, -margin); + fdlValName.top = new FormAttachment(wStepname, margin); + wlValName.setLayoutData(fdlValName); + + wFilenameField = new Text(shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + props.setLook(wFilenameField); + wFilenameField.addModifyListener(lsMod); + FormData fdValName = new FormData(); + fdValName.left = new FormAttachment(middle, 0); + fdValName.right = new FormAttachment(100, 0); + fdValName.top = new FormAttachment(wStepname, margin); + wFilenameField.setLayoutData(fdValName); + + + // output field value + Label wlMetadataName = new Label(shell, SWT.RIGHT); + wlMetadataName.setText(BaseMessages.getString(PKG, "FilemgrIngest.MetadataFieldName.Label")); + props.setLook(wlMetadataName); + FormData fdlMetadataName = new FormData(); + fdlMetadataName.left = new FormAttachment(0, 0); + fdlMetadataName.right = new FormAttachment(middle, -margin); + fdlMetadataName.top = new FormAttachment(wFilenameField, margin); + wlMetadataName.setLayoutData(fdlMetadataName); + + wMetadataField = new Text(shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + props.setLook(wMetadataField); + wMetadataField.addModifyListener(lsMod); + FormData fdMetadataName = new FormData(); + fdMetadataName.left = new FormAttachment(middle, 0); + fdMetadataName.right = new FormAttachment(100, 0); + fdMetadataName.top = new FormAttachment(wFilenameField, margin); + wMetadataField.setLayoutData(fdMetadataName); + + + // servername field value + Label wlServerName = new Label(shell, SWT.RIGHT); + wlServerName.setText(BaseMessages.getString(PKG, "FilemgrIngest.ServerURL.Label")); + props.setLook(wlServerName); + FormData fdlServerName = new FormData(); + fdlServerName.left = new FormAttachment(0, 0); + fdlServerName.right = new FormAttachment(middle, -margin); + fdlServerName.top = new FormAttachment(wMetadataField, margin); + wlServerName.setLayoutData(fdlServerName); + + wServerURLField = new Text(shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + props.setLook(wServerURLField); + wServerURLField.addModifyListener(lsMod); + FormData fdServerName = new FormData(); + fdServerName.left = new FormAttachment(middle, 0); + fdServerName.right = new FormAttachment(100, 0); + fdServerName.top = new FormAttachment(wMetadataField, margin); + wServerURLField.setLayoutData(fdServerName); + + // servername field value + Label wlResultName = new Label(shell, SWT.RIGHT); + wlResultName.setText(BaseMessages.getString(PKG, "FilemgrIngest.Result.Label")); + props.setLook(wlResultName); + FormData fdlResultName = new FormData(); + fdlResultName.left = new FormAttachment(0, 0); + fdlResultName.right = new FormAttachment(middle, -margin); + fdlResultName.top = new FormAttachment(wServerURLField, margin); + wlResultName.setLayoutData(fdlResultName); + + wResultField = new Text(shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + props.setLook(wResultField); + wResultField.addModifyListener(lsMod); + FormData fdResultName = new FormData(); + fdResultName.left = new FormAttachment(middle, 0); + fdResultName.right = new FormAttachment(100, 0); + fdResultName.top = new FormAttachment(wServerURLField, margin); + wResultField.setLayoutData(fdResultName); + + + // OK and cancel buttons + wOK = new Button(shell, SWT.PUSH); + wOK.setText(BaseMessages.getString(PKG, "System.Button.OK")); + wCancel = new Button(shell, SWT.PUSH); + wCancel.setText(BaseMessages.getString(PKG, "System.Button.Cancel")); + + BaseStepDialog.positionBottomButtons(shell, new Button[] { wOK, wCancel }, margin, wResultField); + + // Add listeners for cancel and OK + lsCancel = new Listener() { + public void handleEvent(Event e) {cancel();} + }; + lsOK = new Listener() { + public void handleEvent(Event e) {ok();} + }; + + wCancel.addListener(SWT.Selection, lsCancel); + wOK.addListener(SWT.Selection, lsOK); + + // default listener (for hitting "enter") + lsDef = new SelectionAdapter() { + public void widgetDefaultSelected(SelectionEvent e) {ok();} + }; + wStepname.addSelectionListener(lsDef); + wFilenameField.addSelectionListener(lsDef); + wServerURLField.addSelectionListener(lsDef); + wResultField.addSelectionListener(lsDef); + // Detect X or ALT-F4 or something that kills this window and cancel the dialog properly + shell.addShellListener(new ShellAdapter() { + public void shellClosed(ShellEvent e) {cancel();} + }); + + // Set/Restore the dialog size based on last position on screen + // The setSize() method is inherited from BaseStepDialog + setSize(); + + // populate the dialog with the values from the meta object + populateDialog(); + + // restore the changed flag to original value, as the modify listeners fire during dialog population + meta.setChanged(changed); + + // open dialog and enter event loop + shell.open(); + while (!shell.isDisposed()) { + if (!display.readAndDispatch()) + display.sleep(); + } + + // at this point the dialog has closed, so either ok() or cancel() have been executed + // The "stepname" variable is inherited from BaseStepDialog + return stepname; + } + + /** + * This helper method puts the step configuration stored in the meta object + * and puts it into the dialog controls. + */ + private void populateDialog() { + wStepname.selectAll(); + wFilenameField.setText(meta.getFilenameField()); + wServerURLField.setText(meta.getServerURLField()); + wResultField.setText(meta.getResultField()); + wMetadataField.setText(meta.getMetadataField()); + + } + + /** + * Called when the user cancels the dialog. + */ + private void cancel() { + // The "stepname" variable will be the return value for the open() method. + // Setting to null to indicate that dialog was cancelled. + stepname = null; + // Restoring original "changed" flag on the met aobject + meta.setChanged(changed); + // close the SWT dialog window + dispose(); + } + + /** + * Called when the user confirms the dialog + */ + private void ok() { + // The "stepname" variable will be the return value for the open() method. + // Setting to step name from the dialog control + stepname = wStepname.getText(); + // Setting the settings to the meta object + //meta.setOutputField(wFilenameField.getText()); + meta.setFilenameField(wFilenameField.getText()); + meta.setServerURLField(wServerURLField.getText()); + meta.setResultField(wResultField.getText()); + meta.setMetadataField(wMetadataField.getText()); + // close the SWT dialog window + dispose(); + } +} Added: oodt/trunk/tools/pdi_plugin/src/org/apache/oodt/filemgringest/FilemgrIngestStepMeta.java URL: http://svn.apache.org/viewvc/oodt/trunk/tools/pdi_plugin/src/org/apache/oodt/filemgringest/FilemgrIngestStepMeta.java?rev=1601222&view=auto ============================================================================== --- oodt/trunk/tools/pdi_plugin/src/org/apache/oodt/filemgringest/FilemgrIngestStepMeta.java (added) +++ oodt/trunk/tools/pdi_plugin/src/org/apache/oodt/filemgringest/FilemgrIngestStepMeta.java Sun Jun 8 14:04:07 2014 @@ -0,0 +1,358 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.oodt.filemgringest; + +import org.eclipse.swt.widgets.Shell; +import org.pentaho.di.core.CheckResult; +import org.pentaho.di.core.CheckResultInterface; +import org.pentaho.di.core.annotations.Step; +import org.pentaho.di.core.database.DatabaseMeta; +import org.pentaho.di.core.exception.KettleException; +import org.pentaho.di.core.exception.KettleStepException; +import org.pentaho.di.core.exception.KettleValueException; +import org.pentaho.di.core.exception.KettleXMLException; +import org.pentaho.di.core.row.RowMetaInterface; +import org.pentaho.di.core.row.ValueMeta; +import org.pentaho.di.core.row.ValueMetaInterface; +import org.pentaho.di.core.variables.VariableSpace; +import org.pentaho.di.core.xml.XMLHandler; +import org.pentaho.di.i18n.BaseMessages; +import org.pentaho.di.repository.ObjectId; +import org.pentaho.di.repository.Repository; +import org.pentaho.di.trans.Trans; +import org.pentaho.di.trans.TransMeta; +import org.pentaho.di.trans.step.*; +import org.pentaho.metastore.api.IMetaStore; +import org.w3c.dom.Node; + +import java.util.List; + +/** + * This class is part of the demo step plug-in implementation. + * It demonstrates the basics of developing a plug-in step for PDI. + * + * The demo step adds a new string field to the row stream and sets its + * value to "Hello World!". The user may select the name of the new field. + * + * This class is the implementation of StepMetaInterface. + * Classes implementing this interface need to: + * + * - keep track of the step settings + * - serialize step settings both to xml and a repository + * - provide new instances of objects implementing StepDialogInterface, StepInterface and StepDataInterface + * - report on how the step modifies the meta-data of the row-stream (row structure and field types) + * - perform a sanity-check on the settings provided by the user + * + */ + +@Step( + id = "FilemgrIngest", + image = "bi/meteorite/filemgringest/resources/oodt.jpg", + i18nPackageName="bi.meteorite.filemgringest", + name="FilemgrIngestStep.Name", + description = "FilemgrIngestStep.TooltipDesc", + categoryDescription="i18n:org.pentaho.di.trans.step:BaseStep.Category.BigData" +) +public class FilemgrIngestStepMeta extends BaseStepMeta implements StepMetaInterface { + + /** + * The PKG member is used when looking up internationalized strings. + * The properties file with localized keys is expected to reside in + * {the package of the class specified}/messages/messages_{locale}.properties + */ + private static Class<?> PKG = FilemgrIngestStepMeta.class; // for i18n purposes + + /** + * Stores the name of the field added to the row-stream. + */ + private String outputField; + private String filenameField; + private String serverURLField; + private String resultField; + private String metadataField; + + /** + * Constructor should call super() to make sure the base class has a chance to initialize properly. + */ + public FilemgrIngestStepMeta() { + super(); + } + + /** + * Called by Spoon to get a new instance of the SWT dialog for the step. + * A standard implementation passing the arguments to the constructor of the step dialog is recommended. + * + * @param shell an SWT Shell + * @param meta description of the step + * @param transMeta description of the the transformation + * @param name the name of the step + * @return new instance of a dialog for this step + */ + public StepDialogInterface getDialog(Shell shell, StepMetaInterface meta, TransMeta transMeta, String name) { + return new FilemgrIngestStepDialog(shell, meta, transMeta, name); + } + + /** + * Called by PDI to get a new instance of the step implementation. + * A standard implementation passing the arguments to the constructor of the step class is recommended. + * + * @param stepMeta description of the step + * @param stepDataInterface instance of a step data class + * @param cnr copy number + * @param transMeta description of the transformation + * @param disp runtime implementation of the transformation + * @return the new instance of a step implementation + */ + public StepInterface getStep(StepMeta stepMeta, StepDataInterface stepDataInterface, int cnr, TransMeta transMeta, Trans disp) { + return new FilemgrIngestStep(stepMeta, stepDataInterface, cnr, transMeta, disp); + } + + /** + * Called by PDI to get a new instance of the step data class. + */ + public StepDataInterface getStepData() { + return new FilemgrIngestStepData(); + } + + /** + * This method is called every time a new step is created and should allocate/set the step configuration + * to sensible defaults. The values set here will be used by Spoon when a new step is created. + */ + public void setDefault() { + outputField = "demo_field"; + filenameField = "fieldname"; + serverURLField = "http://localhost:9000";; + resultField = "result"; + metadataField = "metadata"; + } + + /** + * Getter for the name of the field added by this step + * @return the name of the field added + */ + public String getOutputField() { + return outputField; + } + + /** + * Setter for the name of the field added by this step + * @param outputField the name of the field added + */ + public void setOutputField(String outputField) { + this.outputField = outputField; + } + + + public String getFilenameField(){ + return filenameField; + } + + public void setFilenameField(String filenameField){ + this.filenameField = filenameField; + } + /** + * This method is used when a step is duplicated in Spoon. It needs to return a deep copy of this + * step meta object. Be sure to create proper deep copies if the step configuration is stored in + * modifiable objects. + * + * See org.pentaho.di.trans.steps.rowgenerator.RowGeneratorMeta.clone() for an example on creating + * a deep copy. + * + * @return a deep copy of this + */ + public Object clone() { + Object retval = super.clone(); + return retval; + } + + /** + * This method is called by Spoon when a step needs to serialize its configuration to XML. The expected + * return value is an XML fragment consisting of one or more XML tags. + * + * Please use org.pentaho.di.core.xml.XMLHandler to conveniently generate the XML. + * + * @return a string containing the XML serialization of this step + */ + public String getXML() throws KettleValueException { + + // only one field to serialize + String xml = " "; + xml += " " +XMLHandler.addTagValue("outputfield", outputField); + xml += " " +XMLHandler.addTagValue("filenamefield", filenameField); + xml += " " +XMLHandler.addTagValue("serverurlfield", serverURLField); + xml += " " +XMLHandler.addTagValue("resultfield", resultField); + xml += " " +XMLHandler.addTagValue("metadatafield", metadataField); + return xml; + } + + /** + * This method is called by PDI when a step needs to load its configuration from XML. + * + * Please use org.pentaho.di.core.xml.XMLHandler to conveniently read from the + * XML node passed in. + * + * @param stepnode the XML node containing the configuration + * @param databases the databases available in the transformation + * @param metaStore the metaStore to optionally read from + */ + public void loadXML(Node stepnode, List<DatabaseMeta> databases, IMetaStore metaStore) throws KettleXMLException { + + try { + setOutputField(XMLHandler.getNodeValue(XMLHandler.getSubNode(stepnode, "outputfield"))); + setFilenameField(XMLHandler.getNodeValue(XMLHandler.getSubNode(stepnode, "filenamefield"))); + setServerURLField(XMLHandler.getNodeValue(XMLHandler.getSubNode(stepnode, "serverurlfield"))); + setResultField(XMLHandler.getNodeValue(XMLHandler.getSubNode(stepnode, "resultfield"))); + setMetadataField(XMLHandler.getNodeValue(XMLHandler.getSubNode(stepnode,"metadatafield"))); + } catch (Exception e) { + throw new KettleXMLException("Demo plugin unable to read step info from XML node", e); + } + + } + /** + * This method is called by Spoon when a step needs to serialize its configuration to a repository. + * The repository implementation provides the necessary methods to save the step attributes. + * + * @param rep the repository to save to + * @param metaStore the metaStore to optionally write to + * @param id_transformation the id to use for the transformation when saving + * @param id_step the id to use for the step when saving + */ + public void saveRep(Repository rep, IMetaStore metaStore, ObjectId id_transformation, ObjectId id_step) throws KettleException + { + try{ + rep.saveStepAttribute(id_transformation, id_step, "outputfield", outputField); //$NON-NLS-1$ + } + catch(Exception e){ + throw new KettleException("Unable to save step into repository: "+id_step, e); + } + } + + /** + * This method is called by PDI when a step needs to read its configuration from a repository. + * The repository implementation provides the necessary methods to read the step attributes. + * + * @param rep the repository to read from + * @param metaStore the metaStore to optionally read from + * @param id_step the id of the step being read + * @param databases the databases available in the transformation + */ + public void readRep(Repository rep, IMetaStore metaStore, ObjectId id_step, List<DatabaseMeta> databases) throws KettleException { + try{ + outputField = rep.getStepAttributeString(id_step, "outputfield"); //$NON-NLS-1$ + } + catch(Exception e){ + throw new KettleException("Unable to load step from repository", e); + } + } + + /** + * This method is called to determine the changes the step is making to the row-stream. + * To that end a RowMetaInterface object is passed in, containing the row-stream structure as it is when entering + * the step. This method must apply any changes the step makes to the row stream. Usually a step adds fields to the + * row-stream. + * + * @param inputRowMeta the row structure coming in to the step + * @param name the name of the step making the changes + * @param info row structures of any info steps coming in + * @param nextStep the description of a step this step is passing rows to + * @param space the variable space for resolving variables + * @param repository the repository instance optionally read from + * @param metaStore the metaStore to optionally read from + */ + public void getFields(RowMetaInterface inputRowMeta, String name, RowMetaInterface[] info, StepMeta nextStep, VariableSpace space, Repository repository, IMetaStore metaStore) throws KettleStepException{ + + /* + * This implementation appends the outputField to the row-stream + */ + + // a value meta object contains the meta data for a field + ValueMetaInterface v = new ValueMeta(outputField, ValueMeta.TYPE_STRING); + + // setting trim type to "both" + v.setTrimType(ValueMeta.TRIM_TYPE_BOTH); + + // the name of the step that adds this field + v.setOrigin(name); + + // modify the row structure and add the field this step generates + inputRowMeta.addValueMeta(v); + + } + + /** + * This method is called when the user selects the "Verify Transformation" option in Spoon. + * A list of remarks is passed in that this method should add to. Each remark is a comment, warning, error, or ok. + * The method should perform as many checks as necessary to catch design-time errors. + * + * Typical checks include: + * - verify that all mandatory configuration is given + * - verify that the step receives any input, unless it's a row generating step + * - verify that the step does not receive any input if it does not take them into account + * - verify that the step finds fields it relies on in the row-stream + * + * @param remarks the list of remarks to append to + * @param transMeta the description of the transformation + * @param stepMeta the description of the step + * @param prev the structure of the incoming row-stream + * @param input names of steps sending input to the step + * @param output names of steps this step is sending output to + * @param info fields coming in from info steps + * @param metaStore metaStore to optionally read from + */ + public void check(List<CheckResultInterface> remarks, TransMeta transMeta, StepMeta stepMeta, RowMetaInterface prev, String input[], String output[], RowMetaInterface info, VariableSpace space, Repository repository, IMetaStore metaStore) { + + CheckResult cr; + + // See if there are input streams leading to this step! + if (input.length > 0) { + cr = new CheckResult(CheckResult.TYPE_RESULT_OK, BaseMessages.getString(PKG, "Demo.CheckResult.ReceivingRows.OK"), stepMeta); + remarks.add(cr); + } else { + cr = new CheckResult(CheckResult.TYPE_RESULT_ERROR, BaseMessages.getString(PKG, "Demo.CheckResult.ReceivingRows.ERROR"), stepMeta); + remarks.add(cr); + } + + } + + + public void setServerURLField(String serverURLField) { + this.serverURLField = serverURLField; + } + + public String getServerURLField() { + return serverURLField; + } + + + public void setResultField(String resultField) { + this.resultField = resultField; + } + + public String getResultField() { + return resultField; + } + + public void setMetadataField(String metadataField) { + this.metadataField = metadataField; + } + + public String getMetadataField() { + return metadataField; + } +} Added: oodt/trunk/tools/pdi_plugin/src/org/apache/oodt/filemgringest/OODTConfig.java URL: http://svn.apache.org/viewvc/oodt/trunk/tools/pdi_plugin/src/org/apache/oodt/filemgringest/OODTConfig.java?rev=1601222&view=auto ============================================================================== --- oodt/trunk/tools/pdi_plugin/src/org/apache/oodt/filemgringest/OODTConfig.java (added) +++ oodt/trunk/tools/pdi_plugin/src/org/apache/oodt/filemgringest/OODTConfig.java Sun Jun 8 14:04:07 2014 @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.oodt.filemgringest; + +import org.apache.oodt.cas.filemgr.ingest.Ingester; + +import java.lang.reflect.InvocationTargetException; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Created by bugg on 07/03/14. + */ +public class OODTConfig { + + private static final Logger LOG = Logger.getLogger(OODTConfig.class.getName()); + + private URL fmUrl; + + private Ingester ingester; + + public Ingester getIngester() { + return this.ingester; + } + + public URL getFmUrl(){ + return this.fmUrl; + } + + + void loadIngester(String fmUrlStr) throws InstantiationException { + try { + + String ingesterClass = "org.apache.oodt.cas.filemgr.ingest.StdIngester"; + + String dataTransferClass ="org.apache.oodt.cas.filemgr.datatransfer.LocalDataTransferFactory"; + + String cacheFactoryClass = null; + //"org.apache.oodt.cas.filemgr.ingest.cache.factory"; + + LOG.log(Level.INFO, "Configuring and building ingester: [" + + ingesterClass + "]: data transfer: [" + + dataTransferClass + "]: to ingest to file manager: [" + + fmUrlStr + "]"); + + if (cacheFactoryClass != null) { + LOG.log(Level.INFO, "Configuring Ingester cache: [" + + cacheFactoryClass + "]"); + } + + this.ingester = PushPullObjectFactory.createIngester( + ingesterClass, cacheFactoryClass); + + this.fmUrl = safeGetUrlFromString(fmUrlStr); + + } catch (ClassNotFoundException e1) { + e1.printStackTrace(); + } catch (InvocationTargetException e1) { + e1.printStackTrace(); + } catch (NoSuchMethodException e1) { + e1.printStackTrace(); + } catch (IllegalAccessException e1) { + e1.printStackTrace(); + } + + } + + + private static URL safeGetUrlFromString(String urlStr) { + URL url = null; + + try { + url = new URL(urlStr); + } catch (MalformedURLException e) { + LOG.log(Level.WARNING, "Unable to generate url from url string: [" + + urlStr + "]: Message: " + e.getMessage()); + } + + return url; + } +} Added: oodt/trunk/tools/pdi_plugin/src/org/apache/oodt/filemgringest/OODTProcesses.java URL: http://svn.apache.org/viewvc/oodt/trunk/tools/pdi_plugin/src/org/apache/oodt/filemgringest/OODTProcesses.java?rev=1601222&view=auto ============================================================================== --- oodt/trunk/tools/pdi_plugin/src/org/apache/oodt/filemgringest/OODTProcesses.java (added) +++ oodt/trunk/tools/pdi_plugin/src/org/apache/oodt/filemgringest/OODTProcesses.java Sun Jun 8 14:04:07 2014 @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.oodt.filemgringest; + +import com.google.gson.Gson; +import com.google.gson.internal.LinkedTreeMap; +import org.apache.oodt.cas.filemgr.structs.exceptions.CatalogException; +import org.apache.oodt.cas.filemgr.structs.exceptions.IngestException; +import org.apache.oodt.cas.metadata.Metadata; + +import java.io.File; +import java.util.*; +import java.util.logging.Logger; + +/** + * Created by bugg on 07/03/14. + */ +public class OODTProcesses { + + private static final Logger LOG = Logger.getLogger(OODTProcesses.class.getName()); + + private String filemgrUrl; + + public boolean isAlreadyInDatabase(OODTConfig config, String filename) throws CatalogException { + return config.getIngester() != null && config.getIngester().hasProduct( + config.getFmUrl(), filename); + } + + Metadata getMetadata(String json) { + Gson gson=new Gson(); + Hashtable table = new Hashtable<String, String>(); + table = (Hashtable<String, String>) gson.fromJson(json, table.getClass()); + + ArrayList l = (ArrayList) table.get("data"); + LinkedTreeMap d = (LinkedTreeMap) l.get(0); + Hashtable<String,Object> ht = new Hashtable<String,Object>(); + + + + ht.putAll(d); + Metadata m = new Metadata(); + + m.addMetadata(ht); + return m; + + } + + boolean ingest(OODTConfig config, File product, Metadata productMetdata) throws IngestException { + // try { + String productId = config.getIngester().ingest(config.getFmUrl(), + product, productMetdata); +/* LOG.log(Level.INFO, "Successfully ingested product: [" + product + + "]: product id: " + productId); + } catch (Exception e) { + LOG.log(Level.WARNING, + "ProductCrawler: Exception ingesting product: [" + product + + "]: Message: " + e.getMessage() + + ": attempting to continue crawling", e); + return false; + }*/ + return true; + } + + +} Added: oodt/trunk/tools/pdi_plugin/src/org/apache/oodt/filemgringest/PushPullObjectFactory.java URL: http://svn.apache.org/viewvc/oodt/trunk/tools/pdi_plugin/src/org/apache/oodt/filemgringest/PushPullObjectFactory.java?rev=1601222&view=auto ============================================================================== --- oodt/trunk/tools/pdi_plugin/src/org/apache/oodt/filemgringest/PushPullObjectFactory.java (added) +++ oodt/trunk/tools/pdi_plugin/src/org/apache/oodt/filemgringest/PushPullObjectFactory.java Sun Jun 8 14:04:07 2014 @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.oodt.filemgringest; + +//OODT imports +import org.apache.oodt.cas.filemgr.ingest.Cache; +import org.apache.oodt.cas.filemgr.ingest.CacheFactory; +import org.apache.oodt.cas.filemgr.ingest.Ingester; + +//JDK imports +import java.lang.reflect.InvocationTargetException; + +/** + * + * @author bfoster + * @version $Revision$ + * + * <p> + * Describe your class here + * </p>. + */ +public class PushPullObjectFactory { + + private PushPullObjectFactory() throws InstantiationException { + throw new InstantiationException("Don't construct factory classes!"); + } + + public static <T> T createNewInstance(Class<T> clazz) throws InstantiationException { + try { + return clazz.newInstance(); + } catch (Exception e) { + throw new InstantiationException( + "Failed to create new object : " + + e.getMessage()); + } + } + + public static Ingester createIngester(String ingesterClass, + String cacheFactoryClass) throws InstantiationException, + IllegalAccessException, ClassNotFoundException, + IllegalArgumentException, SecurityException, + InvocationTargetException, NoSuchMethodException { + String dataTransferFactory = "org.apache.oodt.cas.filemgr.datatransfer.LocalDataTransferFactory"; + System.out.println("TRANSFER: " + dataTransferFactory); + if (cacheFactoryClass == null || cacheFactoryClass.equals("")) { + return (Ingester) Class.forName(ingesterClass).getConstructor( + dataTransferFactory.getClass()).newInstance( + dataTransferFactory); + } else { + Class<CacheFactory> cacheFactory = (Class<CacheFactory>) Class + .forName(cacheFactoryClass); + Cache cache = cacheFactory.newInstance().createCache(); + return (Ingester) Class.forName(ingesterClass).getConstructor( + dataTransferFactory.getClass(), cache.getClass()) + .newInstance(dataTransferFactory, cache); + } + } + +} Added: oodt/trunk/tools/pdi_plugin/src/org/apache/oodt/filemgringest/messages/messages_en_US.properties URL: http://svn.apache.org/viewvc/oodt/trunk/tools/pdi_plugin/src/org/apache/oodt/filemgringest/messages/messages_en_US.properties?rev=1601222&view=auto ============================================================================== --- oodt/trunk/tools/pdi_plugin/src/org/apache/oodt/filemgringest/messages/messages_en_US.properties (added) +++ oodt/trunk/tools/pdi_plugin/src/org/apache/oodt/filemgringest/messages/messages_en_US.properties Sun Jun 8 14:04:07 2014 @@ -0,0 +1,9 @@ +FilemgrIngestStep.Name=OODT Filemgr File Ingest +FilemgrIngestStep.TooltipDesc=Check if the OODT Filemgr has a file in its repository + +FilemgrIngest.FieldName.Label=Filename Field + +FilemgrIngest.MetadataFieldName.Label=Metadata Field + +FilemgrIngest.ServerURL.Label=Filemgr URL +FilemgrIngest.Result.Label=Result Field Added: oodt/trunk/tools/pdi_plugin/src/org/apache/oodt/filemgringest/resources/oodt.jpg URL: http://svn.apache.org/viewvc/oodt/trunk/tools/pdi_plugin/src/org/apache/oodt/filemgringest/resources/oodt.jpg?rev=1601222&view=auto ============================================================================== Binary file - no diff available. Propchange: oodt/trunk/tools/pdi_plugin/src/org/apache/oodt/filemgringest/resources/oodt.jpg ------------------------------------------------------------------------------ svn:mime-type = image/jpeg Added: oodt/trunk/tools/pdi_plugin/src/org/apache/oodt/filemgringest/resources/oodt.png URL: http://svn.apache.org/viewvc/oodt/trunk/tools/pdi_plugin/src/org/apache/oodt/filemgringest/resources/oodt.png?rev=1601222&view=auto ============================================================================== (empty) Added: oodt/trunk/tools/pdi_plugin/test-lib/junit-3.8.1.jar URL: http://svn.apache.org/viewvc/oodt/trunk/tools/pdi_plugin/test-lib/junit-3.8.1.jar?rev=1601222&view=auto ============================================================================== --- oodt/trunk/tools/pdi_plugin/test-lib/junit-3.8.1.jar (added) +++ oodt/trunk/tools/pdi_plugin/test-lib/junit-3.8.1.jar Sun Jun 8 14:04:07 2014 @@ -0,0 +1 @@ +link /home/bugg/.ivy2/cache/junit/junit/jars/junit-3.8.1.jar \ No newline at end of file Propchange: oodt/trunk/tools/pdi_plugin/test-lib/junit-3.8.1.jar ------------------------------------------------------------------------------ svn:special = * Added: oodt/trunk/tools/pdi_plugin/test-lib/log4j-1.2.16.jar URL: http://svn.apache.org/viewvc/oodt/trunk/tools/pdi_plugin/test-lib/log4j-1.2.16.jar?rev=1601222&view=auto ============================================================================== --- oodt/trunk/tools/pdi_plugin/test-lib/log4j-1.2.16.jar (added) +++ oodt/trunk/tools/pdi_plugin/test-lib/log4j-1.2.16.jar Sun Jun 8 14:04:07 2014 @@ -0,0 +1 @@ +link /home/bugg/.ivy2/cache/log4j/log4j/jars/log4j-1.2.16.jar \ No newline at end of file Propchange: oodt/trunk/tools/pdi_plugin/test-lib/log4j-1.2.16.jar ------------------------------------------------------------------------------ svn:special = *
