Author: mcantelon
Date: Tue Jan 17 17:01:28 2012
New Revision: 10708
Log:
Grouped related methods in QubitFlatfiles to make source easier to understand.
Modified:
trunk/lib/QubitFlatfileImport.class.php
Modified: trunk/lib/QubitFlatfileImport.class.php
==============================================================================
--- trunk/lib/QubitFlatfileImport.class.php Tue Jan 17 16:33:49 2012
(r10707)
+++ trunk/lib/QubitFlatfileImport.class.php Tue Jan 17 17:01:28 2012
(r10708)
@@ -66,6 +66,13 @@
$this->rowStatusVars = array();
}
+
+ /*
+ *
+ * Helper methods
+ * --------------
+ */
+
/**
* Use an array of properties and their respective values to set an object's
* properties (restricting to a set of allowed properties and allowing the
@@ -122,19 +129,6 @@
}
/**
- * Add an ad-hoc column handler
- *
- * @param string $column name of column
- * @param closure $handler column handling logic
- *
- * @return void
- */
- public function addColumnHandler($column, $handler)
- {
- $this->handlers[$column] = $handler;
- }
-
- /**
* Test whether a property is set and, if so, execute it
*
* @param string $property name of property
@@ -152,179 +146,110 @@
}
/**
- * Determine whether columns isn't handled by import logic
+ * Start import timer
*
- * @return boolean TRUE if column not handled by import logic
+ * @return void
*/
- public function unhandledColumn($column)
+ protected function startTimer()
{
- return !in_array($column, $this->ignoreColumns)
- && !in_array($column, $this->standardColumns)
- && !isset($this->columnMap[$column])
- && !isset($this->propertyMap[$column])
- && !isset($this->noteMap[$column])
- && !isset($this->handlers[$column])
- && !in_array($column, $this->variableColumns)
- && !isset($this->arrayColumns[$column]);
+ $this->timer = new QubitTimer;
+ $this->timer->start();
}
/**
- * Combine two or more arrays, eliminating any duplicates
+ * Stop import timer
*
- * @return array combined array
+ * @return void
*/
- protected function combineArraysWithoutDuplicates()
+ protected function stopTimer()
{
- $args = func_get_args();
- $combined = array();
-
- // go through each array providesd
- for($index = 0; $index < count($args); $index++)
- {
- // for each element of array, add to combined array if element isn't a
dupe
- foreach($args[$index] as $element)
- {
- if (!in_array($element, $combined)) array_push($combined, $element);
- }
- }
-
- return $combined;
+ $this->timer->stop();
}
/**
- * Return an array of import columns that aren't handled by import logic
+ * Get time elapsed during import
*
- * @return array array of column names
+ * @return int microseconds since import began
*/
- public function determineUnmatchedHandledColumns()
+ public function getTimeElapsed()
{
- $unmatchedColumns = array();
-
- foreach($this->handledColumns() as $handledColumn)
- {
- if (!in_array($handledColumn, $this->columnNames))
array_push($unmatchedColumns, $handledColumn);
- }
-
- return $unmatchedColumns;
+ return $this->timer->elapsed();
}
/**
- * Return an array of columns that are handled by import logic
+ * Log error message if an error log has been defined
*
- * @return array array of column names
+ * @param string $message error message
+ *
+ * @return string message prefixed with current row number
*/
- public function handledColumns()
+ public function logError($message)
{
- return $this->combineArraysWithoutDuplicates(
- $this->standardColumns,
- $this->variableColumns,
- array_keys($this->columnMap),
- array_keys($this->handlers),
- array_keys($this->propertyMap),
- array_keys($this->noteMap),
- array_keys($this->arrayColumns)
- );
+ $message = 'Row '. $this->getStatus('rows') .': '. $message ."\n";
+ if ($this->errorLog) file_put_contents($this->errorLog, $message,
FILE_APPEND);
+ return $message;
}
/**
- * Return an array of columns that are included in the import but not
- * handled by logic.
+ * Append content to existing content, prepending a line break to new content
+ * if necessary
*
- * @return array array of column names
+ * @param string $oldContent existing content
+ * @param string $newContent new content to add to existing content
+ *
+ * @return string both strings appended
*/
- public function determineUnhandledColumns()
+ public function appendWithLineBreakIfNeeded($oldContent, $newContent)
{
- $ignored = array();
- foreach($this->columnNames as $column)
- {
- if ($this->unhandledColumn($column))
- {
- array_push($ignored, $column);
- }
- }
-
- return $ignored;
+ return ($oldContent) ? $oldContent ."\n". $newContent : $newContent;
}
/**
- * Render a string description of problem columns from an array
+ * Convert human readable (e.g. 'This string') strings to camelCase
+ * representation (e.g. 'thisString')
*
- * @return string description
+ * @param string $str input string
+ *
+ * @return string camelCase string
*/
- public function renderProblemColumns($columns, $problemDescription)
+ public static function camelize($str)
{
- $output = '';
-
- if (count($columns))
- {
- $output .= count($columns) . " columns ". $problemDescription .":\n";
- $output .= ' '. implode("\n ", $columns) ."\n";
-
- $output .= "\n";
- }
+ $str = str_replace(' ', '_', $str);
+ $str = sfInflector::camelize($str);
+ $str = lcfirst($str);
- return $output;
+ return $str;
}
/**
- * Render a string description of import columns that aren't handled by
- * the import logic
+ * Combine two or more arrays, eliminating any duplicates
*
- * @return string description
+ * @return array combined array
*/
- public function renderUnhandledColumns()
+ protected function combineArraysWithoutDuplicates()
{
- return $this->renderProblemColumns(
- $this->determineUnhandledColumns(),
- "aren't handled or ignored"
- );
- }
+ $args = func_get_args();
+ $combined = array();
- /**
- * Render a string description of columns that are handled by the import
- * logic but don't actually exist in the import itself
- *
- * @return string description
- */
- public function renderUnmatchedColumns()
- {
- return $this->renderProblemColumns(
- $this->determineUnmatchedHandledColumns(),
- 'are being handled but do not have an import column to work with'
- );
- return $output;
- }
+ // go through each array providesd
+ for($index = 0; $index < count($args); $index++)
+ {
+ // for each element of array, add to combined array if element isn't a
dupe
+ foreach($args[$index] as $element)
+ {
+ if (!in_array($element, $combined)) array_push($combined, $element);
+ }
+ }
- /**
- * Start import timer
- *
- * @return void
- */
- protected function startTimer()
- {
- $this->timer = new QubitTimer;
- $this->timer->start();
+ return $combined;
}
- /**
- * Stop import timer
- *
- * @return void
- */
- protected function stopTimer()
- {
- $this->timer->stop();
- }
- /**
- * Get time elapsed during import
+ /*
*
- * @return int microseconds since import began
+ * Row processing methods
+ * ----------------------
*/
- public function getTimeElapsed()
- {
- return $this->timer->elapsed();
- }
/**
* Pull data from a csv file and process each row
@@ -378,6 +303,157 @@
}
/**
+ * Process a row of imported data
+ *
+ * @param array $row array of column data
+ *
+ * @return void
+ */
+ public function row($row = array())
+ {
+ // stash raw row data so it's accessible to closure logic
+ $this->status['row'] = $row;
+
+ // set row status variables that are based on column values
+ $this->rowProcessingBeforeObjectCreation($row);
+
+ if (isset($this->className))
+ {
+ // create new object
+ $this->object = new $this->className;
+ } else {
+ // execute ad-hoc row initialization logic (which can make objects, load
+ // them, etc.)
+ $this->executeClosurePropertyIfSet('rowInitLogic');
+ }
+
+ // set fields in information object and execute custom column handlers
+ $this->rowProcessingBeforeSave($row);
+
+ // execute pre-save ad-hoc import logic
+ $this->executeClosurePropertyIfSet('preSaveLogic');
+
+ if (isset($this->className))
+ {
+ $this->object->save();
+
+ // execute row completion logic
+ $this->executeClosurePropertyIfSet('postSaveLogic');
+ } else {
+ // execute row completion logic
+ $this->executeClosurePropertyIfSet('saveLogic');
+ }
+
+ // execute post-save ad-hoc import logic
+
+ // process import columns that produce child data (properties and notes)
+ $this->rowProcessingAfterSave($row);
+
+ // reset row-specific status variables
+ $this->rowStatusVars = array();
+ }
+
+ /**
+ * Log error message if an error log has been defined
+ *
+ * @param string $message error message
+ *
+ * @return void
+ */
+ protected function rowProcessingBeforeObjectCreation($row)
+ {
+ // process import columns that don't produce child data
+ $this->forEachRowColumn($row, function(&$self, $index, $columnName, $value)
+ {
+ if (
+ isset($self->columnNames[$index])
+ && in_array($self->columnNames[$index], $self->variableColumns)
+ )
+ {
+ $self->rowStatusVars[$self->columnNames[$index]] = $value;
+ }
+ else if (
+ isset($self->columnNames[$index])
+ && isset($self->arrayColumns[($self->columnNames[$index])])
+ )
+ {
+ $self->arrayColumnHandler($columnName,
$self->arrayColumns[$columnName], $value);
+ }
+ });
+ }
+
+ /**
+ * Perform row processing for before an object is saved such as setting
+ * object properties and executing ad-hoc column handlers
+ *
+ * @param array $row array of column data
+ *
+ * @return void
+ */
+ protected function rowProcessingBeforeSave($row)
+ {
+ // process import columns that don't produce child data
+ $this->forEachRowColumn($row, function(&$self, $index, $columnName, $value)
+ {
+ // if column maps to an attribute, set the attribute
+ if (isset($self->columnMap) && isset($self->columnMap[$columnName]))
+ {
+ $self->mappedColumnHandler($self->columnMap[$columnName], $value);
+ }
+ else if (
+ isset($self->columnNames[$index])
+ && isset($self->handlers[($self->columnNames[$index])])
+ )
+ {
+ // otherwise, if column is data and a handler for it is set, use it
+ call_user_func_array(
+ $self->handlers[$columnName],
+ array($self, $value)
+ );
+ }
+ else if (
+ isset($self->columnNames[$index])
+ && in_array($self->columnNames[$index], $self->standardColumns)
+ )
+ {
+ // otherwise, if column is data and it's a standard column, use it
+ $self->object->{$self->columnNames[$index]} = $value;
+ }
+ });
+ }
+
+ /**
+ * Perform row processing for after an object is saved and has an ID such
+ * as creating child properties and notes
+ *
+ * @param array $row array of column data
+ *
+ * @return void
+ */
+ protected function rowProcessingAfterSave($row)
+ {
+ $this->forEachRowColumn($row, function(&$self, $index, $columnName, $value)
+ {
+ // if column maps to a property, set the property
+ if (isset($self->propertyMap) && isset($self->propertyMap[$columnName])
&& trim($value))
+ {
+ $self->object->addProperty(
+ $self->propertyMap[$columnName],
+ $value
+ );
+ }
+ else if (isset($self->noteMap) && isset($self->noteMap[$columnName]) &&
trim($value))
+ {
+ // otherwise, if maps to a note, create it
+ $transformationLogic =
(isset($self->noteMap[$columnName]['transformationLogic']))
+ ? $self->noteMap[$columnName]['transformationLogic']
+ : false;
+ $self->createNote($self->noteMap[$columnName]['typeId'], $value,
$transformationLogic);
+ }
+ });
+ }
+
+ /**
* Execute logic, defined by a closure, on each column of a row
*
* @param array $row array of column data
@@ -402,6 +478,58 @@
}
/**
+ * Output import progress, time elapsed, and memory usage
+ *
+ * @return string description of import progress
+ */
+ public function renderProgressDescription()
+ {
+ $output = '.';
+
+ // return empty string if no intermittant progress display
+ if (!isset($this->rowsUntilProgressDisplay)
+ || !$this->rowsUntilProgressDisplay
+ ) return $output;
+
+ // row count isn't incremented until after this is displayed, so add one
to reflect reality
+ $rowsProcessed = $this->getStatus('rows') -
$this->getStatus('skippedRows');
+ $memoryUsageMB = round(memory_get_usage() / (1024 * 1024), 2);
+
+ // if this show should be displayed, display it
+ if (!($rowsProcessed % $this->rowsUntilProgressDisplay))
+ {
+ $elapsed = $this->getTimeElapsed();
+ $elapsedMinutes = round($elapsed / 60, 2);
+ $averageTime = round($elapsed / $rowsProcessed, 2);
+
+ $output .= "\n". $rowsProcessed ." rows processed in ". $elapsedMinutes
+ . " minutes (". $averageTime ." second/row average, ". $memoryUsageMB ."
MB used).\n";
+ }
+
+ return $output;
+ }
+
+
+ /*
+ *
+ * Column handlers
+ * ---------------
+ */
+
+ /**
+ * Add an ad-hoc column handler
+ *
+ * @param string $column name of column
+ * @param closure $handler column handling logic
+ *
+ * @return void
+ */
+ public function addColumnHandler($column, $handler)
+ {
+ $this->handlers[$column] = $handler;
+ }
+
+ /**
* Handle mapping of column to object property
*
* @param array $mapDefinition array defining which property to map to and
@@ -448,201 +576,159 @@
}
}
- /**
- * Log error message if an error log has been defined
+
+ /*
*
- * @param string $message error message
+ * Import auditing methods
+ * -----------------------
+ */
+
+ /**
+ * Determine whether columns isn't handled by import logic
*
- * @return void
+ * @return boolean TRUE if column not handled by import logic
*/
- protected function rowProcessingBeforeObjectCreation($row)
- {
- // process import columns that don't produce child data
- $this->forEachRowColumn($row, function(&$self, $index, $columnName, $value)
- {
- if (
- isset($self->columnNames[$index])
- && in_array($self->columnNames[$index], $self->variableColumns)
- )
- {
- $self->rowStatusVars[$self->columnNames[$index]] = $value;
- }
- else if (
- isset($self->columnNames[$index])
- && isset($self->arrayColumns[($self->columnNames[$index])])
- )
- {
- $self->arrayColumnHandler($columnName,
$self->arrayColumns[$columnName], $value);
- }
- });
+ public function unhandledColumn($column)
+ {
+ return !in_array($column, $this->ignoreColumns)
+ && !in_array($column, $this->standardColumns)
+ && !isset($this->columnMap[$column])
+ && !isset($this->propertyMap[$column])
+ && !isset($this->noteMap[$column])
+ && !isset($this->handlers[$column])
+ && !in_array($column, $this->variableColumns)
+ && !isset($this->arrayColumns[$column]);
+ }
+
+ /**
+ * Return an array of import columns that aren't handled by import logic
+ *
+ * @return array array of column names
+ */
+ public function determineUnmatchedHandledColumns()
+ {
+ $unmatchedColumns = array();
+
+ foreach($this->handledColumns() as $handledColumn)
+ {
+ if (!in_array($handledColumn, $this->columnNames))
array_push($unmatchedColumns, $handledColumn);
+ }
+
+ return $unmatchedColumns;
}
/**
- * Perform row processing for before an object is saved such as setting
- * object properties and executing ad-hoc column handlers
- *
- * @param array $row array of column data
+ * Return an array of columns that are handled by import logic
*
- * @return void
+ * @return array array of column names
*/
- protected function rowProcessingBeforeSave($row)
+ public function handledColumns()
{
- // process import columns that don't produce child data
- $this->forEachRowColumn($row, function(&$self, $index, $columnName, $value)
- {
- // if column maps to an attribute, set the attribute
- if (isset($self->columnMap) && isset($self->columnMap[$columnName]))
- {
- $self->mappedColumnHandler($self->columnMap[$columnName], $value);
- }
- else if (
- isset($self->columnNames[$index])
- && isset($self->handlers[($self->columnNames[$index])])
- )
- {
- // otherwise, if column is data and a handler for it is set, use it
- call_user_func_array(
- $self->handlers[$columnName],
- array($self, $value)
- );
- }
- else if (
- isset($self->columnNames[$index])
- && in_array($self->columnNames[$index], $self->standardColumns)
- )
- {
- // otherwise, if column is data and it's a standard column, use it
- $self->object->{$self->columnNames[$index]} = $value;
- }
- });
+ return $this->combineArraysWithoutDuplicates(
+ $this->standardColumns,
+ $this->variableColumns,
+ array_keys($this->columnMap),
+ array_keys($this->handlers),
+ array_keys($this->propertyMap),
+ array_keys($this->noteMap),
+ array_keys($this->arrayColumns)
+ );
}
/**
- * Perform row processing for after an object is saved and has an ID such
- * as creating child properties and notes
- *
- * @param array $row array of column data
+ * Return an array of columns that are included in the import but not
+ * handled by logic.
*
- * @return void
+ * @return array array of column names
*/
- protected function rowProcessingAfterSave($row)
+ public function determineUnhandledColumns()
{
- $this->forEachRowColumn($row, function(&$self, $index, $columnName, $value)
+ $ignored = array();
+ foreach($this->columnNames as $column)
{
- // if column maps to a property, set the property
- if (isset($self->propertyMap) && isset($self->propertyMap[$columnName])
&& trim($value))
- {
- $self->object->addProperty(
- $self->propertyMap[$columnName],
- $value
- );
- }
- else if (isset($self->noteMap) && isset($self->noteMap[$columnName]) &&
trim($value))
+ if ($this->unhandledColumn($column))
{
- // otherwise, if maps to a note, create it
- $transformationLogic =
(isset($self->noteMap[$columnName]['transformationLogic']))
- ? $self->noteMap[$columnName]['transformationLogic']
- : false;
- $self->createNote($self->noteMap[$columnName]['typeId'], $value,
$transformationLogic);
+ array_push($ignored, $column);
}
- });
+ }
+
+ return $ignored;
}
/**
- * Process a row of imported data
- *
- * @param array $row array of column data
+ * Render a string description of problem columns from an array
*
- * @return void
+ * @return string description
*/
- public function row($row = array())
+ public function renderProblemColumns($columns, $problemDescription)
{
- // stash raw row data so it's accessible to closure logic
- $this->status['row'] = $row;
-
- // set row status variables that are based on column values
- $this->rowProcessingBeforeObjectCreation($row);
-
- if (isset($this->className))
- {
- // create new object
- $this->object = new $this->className;
- } else {
- // execute ad-hoc row initialization logic (which can make objects, load
- // them, etc.)
- $this->executeClosurePropertyIfSet('rowInitLogic');
- }
-
- // set fields in information object and execute custom column handlers
- $this->rowProcessingBeforeSave($row);
-
- // execute pre-save ad-hoc import logic
- $this->executeClosurePropertyIfSet('preSaveLogic');
+ $output = '';
- if (isset($this->className))
+ if (count($columns))
{
- $this->object->save();
+ $output .= count($columns) . " columns ". $problemDescription .":\n";
+ $output .= ' '. implode("\n ", $columns) ."\n";
- // execute row completion logic
- $this->executeClosurePropertyIfSet('postSaveLogic');
- } else {
- // execute row completion logic
- $this->executeClosurePropertyIfSet('saveLogic');
+ $output .= "\n";
}
- // execute post-save ad-hoc import logic
-
- // process import columns that produce child data (properties and notes)
- $this->rowProcessingAfterSave($row);
-
- // reset row-specific status variables
- $this->rowStatusVars = array();
+ return $output;
}
/**
- * Log error message if an error log has been defined
- *
- * @param string $message error message
+ * Render a string description of import columns that aren't handled by
+ * the import logic
*
- * @return string message prefixed with current row number
+ * @return string description
*/
- public function logError($message)
+ public function renderUnhandledColumns()
{
- $message = 'Row '. $this->getStatus('rows') .': '. $message ."\n";
- if ($this->errorLog) file_put_contents($this->errorLog, $message,
FILE_APPEND);
- return $message;
+ return $this->renderProblemColumns(
+ $this->determineUnhandledColumns(),
+ "aren't handled or ignored"
+ );
}
/**
- * Output import progress, time elapsed, and memory usage
+ * Render a string description of columns that are handled by the import
+ * logic but don't actually exist in the import itself
*
- * @return string description of import progress
+ * @return string description
*/
- public function renderProgressDescription()
+ public function renderUnmatchedColumns()
{
- $output = '.';
+ return $this->renderProblemColumns(
+ $this->determineUnmatchedHandledColumns(),
+ 'are being handled but do not have an import column to work with'
+ );
+ return $output;
+ }
- // return empty string if no intermittant progress display
- if (!isset($this->rowsUntilProgressDisplay)
- || !$this->rowsUntilProgressDisplay
- ) return $output;
- // row count isn't incremented until after this is displayed, so add one
to reflect reality
- $rowsProcessed = $this->getStatus('rows') -
$this->getStatus('skippedRows');
- $memoryUsageMB = round(memory_get_usage() / (1024 * 1024), 2);
+ /*
+ *
+ * Qubit data helpers
+ * ------------------
+ */
- // if this show should be displayed, display it
- if (!($rowsProcessed % $this->rowsUntilProgressDisplay))
+ /**
+ * Issue an SQL query
+ *
+ * @param string $query SQL query
+ * @param string $params values to map to placeholders (optional)
+ *
+ * @return object database statement object
+ */
+ public function sqlQuery($query, $params = array())
+ {
+ $connection = Propel::getConnection();
+ $statement = $connection->prepare($query);
+ for($index = 0; $index < count($params); $index++)
{
- $elapsed = $this->getTimeElapsed();
- $elapsedMinutes = round($elapsed / 60, 2);
- $averageTime = round($elapsed / $rowsProcessed, 2);
-
- $output .= "\n". $rowsProcessed ." rows processed in ". $elapsedMinutes
- . " minutes (". $averageTime ." second/row average, ". $memoryUsageMB ."
MB used).\n";
+ $statement->bindValue($index + 1, $params[$index]);
}
-
- return $output;
+ $statement->execute();
+ return $statement;
}
/**
@@ -702,26 +788,6 @@
}
/**
- * Issue an SQL query
- *
- * @param string $query SQL query
- * @param string $params values to map to placeholders (optional)
- *
- * @return object database statement object
- */
- public function sqlQuery($query, $params = array())
- {
- $connection = Propel::getConnection();
- $statement = $connection->prepare($query);
- for($index = 0; $index < count($params); $index++)
- {
- $statement->bindValue($index + 1, $params[$index]);
- }
- $statement->execute();
- return $statement;
- }
-
- /**
* Create a Qubit actor or, if one already exists, fetch it
*
* @param string $name name of actor
@@ -964,37 +1030,6 @@
}
/**
- * Append content to existing content, prepending a line break to new content
- * if necessary
- *
- * @param string $oldContent existing content
- * @param string $newContent new content to add to existing content
- *
- * @return string both strings appended
- */
- public function appendWithLineBreakIfNeeded($oldContent, $newContent)
- {
- return ($oldContent) ? $oldContent ."\n". $newContent : $newContent;
- }
-
- /**
- * Convert human readable (e.g. 'This string') strings to camelCase
- * representation (e.g. 'thisString')
- *
- * @param string $str input string
- *
- * @return string camelCase string
- */
- public static function camelize($str)
- {
- $str = str_replace(' ', '_', $str);
- $str = sfInflector::camelize($str);
- $str = lcfirst($str);
-
- return $str;
- }
-
- /**
* Map a value to its corresponding term name then return the term ID
* corresponding to the term name
*
--
You received this message because you are subscribed to the Google Groups
"Qubit Toolkit Commits" group.
To post to this group, send email to [email protected].
To unsubscribe from this group, send email to
[email protected].
For more options, visit this group at
http://groups.google.com/group/qubit-commits?hl=en.
