How big is the data file? MM
On Mon, Jul 4, 2011 at 8:55 AM, DanielMedia <[email protected]> wrote: > Here's the entire shell script if that helps: > > <? > > class ParseCusipAttributesTask extends Shell { > > var $uses = array('MasterIssue'); > > function execute($args = array()) { > > $file = '/home/dan/cusip_master/CUMASTER_ATTRIBUTE.PIP'; > > $record_count = explode(' ', exec("wc -l $file")); > $record_count = number_format($record_count[0]); > > $handle = @fopen($file, 'r'); > > if($handle) { > > $i = 0; > > while(!feof($handle)) { > > $buffer = fgets($handle, 4096); > > $segments = explode('|', $buffer); > > $cusip_prefix = $segments[0] . $segments[1]; > > $data = array(); > $data['MasterIssue']['alternative_min_tax'] > = $segments[2]; > $data['MasterIssue']['bank_q'] = > $segments[3]; > $data['MasterIssue']['callable'] = > $segments[4]; > $data['MasterIssue']['activity_date'] = > $segments[5]; > $data['MasterIssue']['first_coupon_date'] = > $segments[6]; > $data['MasterIssue']['init_pub_off'] = > $segments[7]; > $data['MasterIssue']['payment_frequency'] = > $segments[8]; > $data['MasterIssue']['currency_code'] = > $segments[9]; > $data['MasterIssue']['domicile_code'] = > $segments[10]; > $data['MasterIssue']['underwriter'] = > $segments[11]; > $data['MasterIssue']['us_cfi_code'] = > $segments[12]; > $data['MasterIssue']['closing_date'] = > $segments[13]; > $data['MasterIssue']['ticker_symbol'] = > $segments[14]; > $data['MasterIssue']['iso_cfi'] = > $segments[15]; > $data['MasterIssue']['depos_eligible'] = > $segments[16]; > $data['MasterIssue']['pre_refund'] = > $segments[17]; > $data['MasterIssue']['refundable'] = > $segments[18]; > $data['MasterIssue']['remarketed'] = > $segments[19]; > $data['MasterIssue']['sinking_fund'] = > $segments[20]; > $data['MasterIssue']['taxable'] = > $segments[21]; > $data['MasterIssue']['form'] = > $segments[22]; > $data['MasterIssue']['enhancements'] = > $segments[23]; > $data['MasterIssue']['fund_distrb_policy'] = > $segments[24]; > $data['MasterIssue']['fund_inv_policy'] = > $segments[25]; > $data['MasterIssue']['fund_type'] = > $segments[26]; > $data['MasterIssue']['guarantee'] = > $segments[27]; > $data['MasterIssue']['income_type'] = > $segments[28]; > $data['MasterIssue']['insured_by'] = > $segments[29]; > $data['MasterIssue']['ownership_restr'] = > $segments[30]; > $data['MasterIssue']['payment_status'] = > $segments[31]; > $data['MasterIssue']['preferred_type'] = > $segments[32]; > $data['MasterIssue']['putable'] = > $segments[33]; > $data['MasterIssue']['rate_type'] = > $segments[34]; > $data['MasterIssue']['redemption'] = > $segments[35]; > $data['MasterIssue']['source_doc'] = > $segments[36]; > $data['MasterIssue']['sponsoring'] = > $segments[37]; > $data['MasterIssue']['voting_rights'] = > $segments[38]; > $data['MasterIssue']['warrant_assets'] = > $segments[39]; > $data['MasterIssue']['warrant_status'] = > $segments[40]; > $data['MasterIssue']['warrant_type'] = > $segments[41]; > $data['MasterIssue']['where_traded'] = > $segments[42]; > $data['MasterIssue']['auditor'] = > $segments[43]; > $data['MasterIssue']['paying_agent'] = > $segments[44]; > $data['MasterIssue']['tender_agent'] = > $segments[45]; > $data['MasterIssue']['xfer_agent'] = > $segments[46]; > $data['MasterIssue']['bond_counsel'] = > $segments[47]; > $data['MasterIssue']['financial_advisor'] = > $segments[48]; > $data['MasterIssue']['municipal_sale_date'] > = $segments[49]; > $data['MasterIssue']['sale_type'] = > $segments[50]; > $data['MasterIssue']['offering_amount'] = > $segments[51]; > $data['MasterIssue']['offering_amount_code'] > = $segments[52]; > > if(strlen($cusip_prefix) == 8) { > > $this->MasterIssue->cacheQueries = > false; > > // This causes memory issues > $issues = > $this->MasterIssue->find('all', array( > 'conditions' => > array('MasterIssue.cusip LIKE' => > $cusip_prefix . '%'), > 'fields' => > array('MasterIssue.id', 'MasterIssue.cusip') > )); > > // This does not cause memory > issues > //$issues = > $this->MasterIssue->query("SELECT * FROM > master_issues WHERE cusip LIKE '$cusip_prefix%'", false); > > foreach($issues as $issue) { > $this->out(number_format($i) > . ' of ' . $record_count . ' : ' . > $issue['MasterIssue']['cusip']); > $this->MasterIssue->id = > $issue['MasterIssue']['id']; > > $this->MasterIssue->cacheQueries = false; > > $this->MasterIssue->save($data); > } > > } > > $i++; > > $this->out('Memory used: ' . > memory_get_usage(true)); > > $this- > >out('---------------------------------------------------------'); > > } > > $this->out("Finished processing $i records"); > > } > > fclose($handle); > > } > > } > > ?> > > -- > Our newest site for the community: CakePHP Video Tutorials > http://tv.cakephp.org > Check out the new CakePHP Questions site http://ask.cakephp.org and help > others with their CakePHP related questions. > > > To unsubscribe from this group, send email to > [email protected] For more options, visit this group > at http://groups.google.com/group/cake-php > -- Our newest site for the community: CakePHP Video Tutorials http://tv.cakephp.org Check out the new CakePHP Questions site http://ask.cakephp.org and help others with their CakePHP related questions. To unsubscribe from this group, send email to [email protected] For more options, visit this group at http://groups.google.com/group/cake-php
