Author: david Date: Tue May 17 16:16:51 2011 New Revision: 9104 Log: Commit MJ's bulk import task (not working?).
Added: trunk/lib/task/importBulkTask.class.php (contents, props changed) Added: trunk/lib/task/importBulkTask.class.php ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ trunk/lib/task/importBulkTask.class.php Tue May 17 16:16:51 2011 (r9104) @@ -0,0 +1,121 @@ +<?php + +/* + * This file is part of Qubit Toolkit. + * + * Qubit Toolkit is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Qubit Toolkit is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Qubit Toolkit. If not, see <http://www.gnu.org/licenses/>. + */ + +class importBulkTask extends sfBaseTask +{ + protected function configure() + { + $this->addArguments(array( +// new sfCommandArgument('application', sfCommandArgument::REQUIRED, 'Changes the application context of the task'), + new sfCommandArgument('folder', sfCommandArgument::OPTIONAL, 'The import folder', sfConfig::get('sf_upload_dir')) + )); + + $this->addOptions(array( + new sfCommandOption('application', null, sfCommandOption::PARAMETER_OPTIONAL, 'The application', 'qubit'), + new sfCommandOption('noindex', null, sfCommandOption::PARAMETER_OPTIONAL, 'Set to \'true\' to skip indexing on imported objects'), + new sfCommandOption('output', null, sfCommandOption::PARAMETER_OPTIONAL, 'Filename to output results in CSV format'), + new sfCommandOption('v', null, sfCommandOption::PARAMETER_OPTIONAL, 'Verbose output'), + )); + + $this->namespace = 'import'; + $this->name = 'bulk'; + $this->briefDescription = 'Bulk import multiple XML files at once'; + $this->detailedDescription = <<<EOF +Bulk import multiple XML files at once +EOF; + } + + protected function execute($arguments = array(), $options = array()) + { + $timer = new QubitTimer; // overall timing + + sfContext::createInstance($this->configuration); + + if (empty($arguments['folder']) || !file_exists($arguments['folder']) || !is_dir($arguments['folder'])) { + throw new sfException('You must specify a valid import folder'); + } + + // set indexing preference + if ($options['noindex']) { + sfContext::getInstance()->getRequest()->setParameter('disableSearchIndex', true); + } + + // recurse into the import folder + $files = $this->dir_tree(rtrim($arguments['folder'], '/')); + + // TODO: add some colour + $this->log("Importing ".count($files)." files into ".$arguments['folder']." (indexing is ".($options['noindex'] ? "DISABLED" : "ENABLED").") ...\n"); + + $count = 0; + foreach ($files as $file) { + $start = microtime(true); + $import = QubitXmlImport::execute(file_get_contents($file), array('strictXmlParsing' => true)); + $split = microtime(true) - $start; + + $count++; + // store details if output is specified + if ($options['output']) { + $rows[] = array($count, $split, memory_get_usage()); + } + + if ($options['v']) { + $this->log($count.". '".basename($file)."' imported in ".$split." s."); + } + } + + // create/open output file if specified + if ($options['output']) { + $fh = fopen($options['output'], 'w+'); + foreach ($rows as $row) { + fputcsv($fh, $row); + } + fputcsv($fh, array('', $timer->elapsed(), memory_get_peak_usage())); + fclose($fh); + } + + $this->log("\nSuccessfully imported ".$count." XML files in ".$timer->elapsed()." s. ".memory_get_peak_usage()." bytes used."); + } + + protected function dir_tree($dir) { + $path = ''; + $stack[] = $dir; + while ($stack) { + $thisdir = array_pop($stack); + if ($dircont = scandir($thisdir)) { + $i=0; + while (isset($dircont[$i])) { + if ($dircont[$i] !== '.' && $dircont[$i] !== '..' + // ignore system/hidden files + && !preg_match('/^\..*/', $dircont[$i])) { + + $current_file = "{$thisdir}/{$dircont[$i]}"; + if (is_file($current_file)) { + $path[] = "{$thisdir}/{$dircont[$i]}"; + } elseif (is_dir($current_file)) { + $stack[] = $current_file; + } + } + $i++; + } + } + } + return $path; + } + +} -- You received this message because you are subscribed to the Google Groups "Qubit Toolkit Commits" group. To post to this group, send email to [email protected]. To unsubscribe from this group, send email to [email protected]. For more options, visit this group at http://groups.google.com/group/qubit-commits?hl=en.
