Hello, thanks for all the hard work, glad to see it available as a gem
too :)
I'm getting an error about nesting of json when I try and upgrade. I
figure the solution is to refactor the process so the nesting isn't so
deep, but wondering if there is anthing else that may have caused it.
Below the stacktrace is the partners_phase.rb file.
** Invoke rt:compile_properties (first_time)
** Invoke /Users/francisco/realtravel/rtsite/maven.properties
(first_time, not_needed)
** Invoke /Users/francisco/realtravel/rtsite/realtravel.com/config/
resoures/config/application.properties (first_time)
** Execute /Users/francisco/realtravel/rtsite/realtravel.com/config/
resoures/config/application.properties
** Execute rt:compile_properties
Loading properties: /Users/francisco/realtravel/rtsite/
maven.properties
Loaded 438 properties
Loading properties: /Users/francisco/realtravel/rtsite/
maven.properties
Loaded 438 properties
** Invoke rt:restore:all (first_time)
** Invoke rt:ruote:load (first_time)
** Execute rt:ruote:load
rake aborted!
nesting of 20 is too deep
/Users/francisco/realtravel/rtsite/realtravel.com/gems/ruby/1.8/gems/
json-1.2.0/lib/json/common.rb:123:in `parse'
/Users/francisco/realtravel/rtsite/realtravel.com/gems/ruby/1.8/gems/
json-1.2.0/lib/json/common.rb:123:in `parse'
/Users/francisco/realtravel/rtsite/realtravel.com/gems/ruby/1.8/gems/
rufus-json-0.1.0/lib/rufus/json.rb:35
/Users/francisco/realtravel/rtsite/realtravel.com/gems/ruby/1.8/gems/
rufus-json-0.1.0/lib/rufus/json.rb:113:in `call'
/Users/francisco/realtravel/rtsite/realtravel.com/gems/ruby/1.8/gems/
rufus-json-0.1.0/lib/rufus/json.rb:113:in `decode'
/Users/francisco/realtravel/rtsite/realtravel.com/gems/ruby/1.8/gems/
rufus-json-0.1.0/lib/rufus/json.rb:122:in `dup'
/Users/francisco/realtravel/rtsite/realtravel.com/gems/ruby/1.8/gems/
rufus-cloche-0.1.13/lib/rufus/cloche.rb:71:in `put'
/Users/francisco/realtravel/rtsite/realtravel.com/gems/ruby/1.8/gems/
ruote-2.1.4/lib/ruote/storage/fs_storage.rb:57:in `put'
/Users/francisco/realtravel/rtsite/realtravel.com/gems/ruby/1.8/gems/
ruote-2.1.4/lib/ruote/storage/base.rb:191:in `put_engine_variable'
/Users/francisco/realtravel/rtsite/realtravel.com/gems/ruby/1.8/gems/
ruote-2.1.4/lib/ruote/engine.rb:337:in `[]='
/Users/francisco/realtravel/rtsite/realtravel.com/lib/ruote/
partners_phase.rb:1
/Library/Ruby/Site/1.8/rubygems/custom_require.rb:31:in
`gem_original_require'
/Library/Ruby/Site/1.8/rubygems/custom_require.rb:31:in `require'
/Users/francisco/realtravel/rtsite/realtravel.com/gems/ruby/1.8/gems/
activesupport-2.3.4/lib/active_support/dependencies.rb:156:in
`require'
/Users/francisco/realtravel/rtsite/realtravel.com/gems/ruby/1.8/gems/
activesupport-2.3.4/lib/active_support/dependencies.rb:521:in
`new_constants_in'
/Users/francisco/realtravel/rtsite/realtravel.com/gems/ruby/1.8/gems/
activesupport-2.3.4/lib/active_support/dependencies.rb:156:in
`require'
/Users/francisco/realtravel/rtsite/realtravel.com/lib/ruote/
participants.rb:16
/Library/Ruby/Site/1.8/rubygems/custom_require.rb:31:in
`gem_original_require'
/Library/Ruby/Site/1.8/rubygems/custom_require.rb:31:in `require'
/Users/francisco/realtravel/rtsite/realtravel.com/gems/ruby/1.8/gems/
activesupport-2.3.4/lib/active_support/dependencies.rb:156:in
`require'
/Users/francisco/realtravel/rtsite/realtravel.com/gems/ruby/1.8/gems/
activesupport-2.3.4/lib/active_support/dependencies.rb:521:in
`new_constants_in'
/Users/francisco/realtravel/rtsite/realtravel.com/gems/ruby/1.8/gems/
activesupport-2.3.4/lib/active_support/dependencies.rb:156:in
`require'
/Users/francisco/realtravel/rtsite/realtravel.com/lib/tasks/ruote.rake:
17
/Library/Ruby/Gems/1.8/gems/rake-0.8.7/lib/rake.rb:636:in `call'
/Library/Ruby/Gems/1.8/gems/rake-0.8.7/lib/rake.rb:636:in `execute'
/Library/Ruby/Gems/1.8/gems/rake-0.8.7/lib/rake.rb:631:in `each'
/Library/Ruby/Gems/1.8/gems/rake-0.8.7/lib/rake.rb:631:in `execute'
/Library/Ruby/Gems/1.8/gems/rake-0.8.7/lib/rake.rb:597:in
`invoke_with_call_chain'
/System/Library/Frameworks/Ruby.framework/Versions/1.8/usr/lib/ruby/
1.8/monitor.rb:242:in `synchronize'
/Library/Ruby/Gems/1.8/gems/rake-0.8.7/lib/rake.rb:590:in
`invoke_with_call_chain'
/Library/Ruby/Gems/1.8/gems/rake-0.8.7/lib/rake.rb:607:in
`invoke_prerequisites'
/Library/Ruby/Gems/1.8/gems/rake-0.8.7/lib/rake.rb:604:in `each'
/Library/Ruby/Gems/1.8/gems/rake-0.8.7/lib/rake.rb:604:in
`invoke_prerequisites'
/Library/Ruby/Gems/1.8/gems/rake-0.8.7/lib/rake.rb:596:in
`invoke_with_call_chain'
/System/Library/Frameworks/Ruby.framework/Versions/1.8/usr/lib/ruby/
1.8/monitor.rb:242:in `synchronize'
/Library/Ruby/Gems/1.8/gems/rake-0.8.7/lib/rake.rb:590:in
`invoke_with_call_chain'
/Library/Ruby/Gems/1.8/gems/rake-0.8.7/lib/rake.rb:583:in `invoke'
/Library/Ruby/Gems/1.8/gems/rake-0.8.7/lib/rake.rb:2051:in
`invoke_task'
/Library/Ruby/Gems/1.8/gems/rake-0.8.7/lib/rake.rb:2029:in `top_level'
/Library/Ruby/Gems/1.8/gems/rake-0.8.7/lib/rake.rb:2029:in `each'
/Library/Ruby/Gems/1.8/gems/rake-0.8.7/lib/rake.rb:2029:in `top_level'
/Library/Ruby/Gems/1.8/gems/rake-0.8.7/lib/rake.rb:2068:in
`standard_exception_handling'
/Library/Ruby/Gems/1.8/gems/rake-0.8.7/lib/rake.rb:2023:in `top_level'
/Library/Ruby/Gems/1.8/gems/rake-0.8.7/lib/rake.rb:2001:in `run'
/Library/Ruby/Gems/1.8/gems/rake-0.8.7/lib/rake.rb:2068:in
`standard_exception_handling'
/Library/Ruby/Gems/1.8/gems/rake-0.8.7/lib/rake.rb:1998:in `run'
/Library/Ruby/Gems/1.8/gems/rake-0.8.7/bin/rake:31
/usr/bin/rake:19:in `load'
/usr/bin/rake:19
$engine.variables['partners_phase'] = Ruote.process_definition :name
=> 'partners_phase' do
define :partner do
sequence do
_if :test => "${f:restore_process} is set" do
sequence do
set :f => "process", :value => "${f:restore_process}"
subprocess :ref => "restore_process"
end
end
_if :test => "${f:filters} is not set" do
set :f => "filters", :value => ".*"
end
_if :test => "${f:processes} is set" do
iterator :on_value => "${f:processes}", :to_field => "process"
do
subprocess :ref => 'partner_step'
end
#else
_if :test => "${f:archive_process} is set" do
sequence do
set :f => "process", :value => "${f:archive_process}"
subprocess :ref => 'archive_process'
end
end
end
_if :test => "${f:archive_partner} is set" do
sequence do
_if :test => "${f:archive} is set" do
sequence do
set :f => 'archive_tmp', :value => '${f:archive}'
end
end
_if :test => "${f:archive_partner} == true" do
set :f => 'archive', :value => '${f:partner}'
set :f => 'archive', :value => '${f:archive_partner}'
end
echo "archive ${f:partner} @ ${f:archive}"
_if :test => "${f:archive_partner_sync} is set" do
set :field => 'archive_sync', :value => 'true'
end
subprocess :ref => "archive_phase"
_if :test => "${f:archive_tmp} is set" do
sequence do
set :f => 'archive', :value => '${f:archive_tmp}'
end
end
end
end
end
end
define :partner_step do
#echo "HERE ${f:step}"
sequence do
_if :test => "${f:skip_exec} is set" do
echo "Skipping execution of ${f:process} for ${f:partner}"
#else
sequence do
set :f => "args", :value => "-Dpartner=${f:partner}"
_if :test => "${f:process} == raw" do
subprocess :ref => 'partner_exec'
#else
sequence do
_if :test => "${f:entities} is not set" do
set :f => 'entities', :value => "locations"
end
echo "${f:partner} ${f:process}: ${f:entities}"
iterator :on_value => '${f:entities}', :to_field =>
"entity" do
subprocess :ref => 'partner_${f:process}_entity'
end
end
end
end
end
_if :test => "${f:archive_process} is set" do
subprocess :ref => 'archive_process'
end
end
end
define :partner_extract_entity do
sequence do
subprocess :ref => 'partner_exec'
=begin
_if :test => "${f:synchronous}" do
iterator :on_value => "${f:filters}", :to_field => "filter" do
subprocess :ref => 'partner_exec'
end
concurrent_iterator :on_value => "${f:filters}", :to_field =>
"filter" do
subprocess :ref => 'partner_exec'
end
end
=end
end
end
define :partner_transform_entity do
sequence do
_if :test => "${f:synchronous} is set" do
iterator :on_value => "${f:filters}", :to_field => "filter" do
subprocess :ref => 'partner_exec'
end
concurrent_iterator :on_value => "${f:filters}", :to_field =>
"filter" do
subprocess :ref => 'partner_exec'
end
end
end
end
define :partner_exec do
sequence do
# Checks technology to use to execute process
subprocess :ref => 'process_reset'
check_partner_exec
echo "response: ${f:__result__}<<"
_if :test => "${f:__result__} == kettle" do
sequence do
_if :test => '${f:process} == raw' do
set :f => 'file', :value => "partners/${f:partner}/$
{f:process}/${f:partner}_${f:process}"
set :f => 'file', :value => "partners/${f:partner}/$
{f:process}/${f:partner}_${f:entity}_${f:process}"
end
echo "Kettle: ${f:process} ${f:partner} > ${f:file}"
subprocess :ref => 'kettle_exec'
end
sequence do
echo "OpenWFE: ${f:process} ${f:partner}"
participant "${f:partner}_wfe"
end
end
unset :f => 'args'
end
end
define :process_reset do
sequence do
set :f => 'task', :value => "rt:partner:${f:process}:reset"
_if :test => '${f:args} is set' do
set :f => 'args_tmp', :value => '${f:args}'
end
_if :test => '${f:filter_seq} is not set' do
set :f => 'filter_seq', :value => '0'
end
_if :test => '${f:entity} is set' do
set :f => 'args', :value => "partner=${f:partner} entity=$
{f:entity} filter_seq=${f:filter_seq}"
set :f => 'args', :value => "partner=${f:partner} filter_seq=$
{f:filter_seq}"
end
rake
_if :test => '${f:args_tmp} is set' do
set :f => 'args', :value => '${f:args_tmp}'
unset :f => 'args'
end
end
end
define :destinations_load do
sequence do
set :f => "args", :value => "-Dpartner=${f:partner} \\\\\"-
Dconditions=(destination_type = 10 or destination_type = 20)\\\\\""
set :f => 'file', :value => 'realtravel/${f:entity}/load/$
{f:entity}_match'
subprocess :ref => 'kettle_run'
_if :test => '${f:skip_merge} is not set' do
sequence do
set :f => 'file', :value => "realtravel/${f:entity}/load/rt_$
{f:entity}_load"
subprocess :ref => 'kettle_exec'
end
end
set :f => "args", :value => "-Dpartner=${f:partner} \\\\\"-
Dconditions=(destination_type = 30)\\\\\""
set :f => 'file', :value => 'realtravel/${f:entity}/load/$
{f:entity}_match'
subprocess :ref => 'kettle_run'
_if :test => '${f:skip_merge} is not set' do
sequence do
set :f => 'file', :value => "realtravel/${f:entity}/load/rt_$
{f:entity}_load"
subprocess :ref => 'kettle_exec'
end
end
set :f => "args", :value => "-Dpartner=${f:partner} \\\\\"-
Dconditions=destination_type = 40\\\\\""
set :f => 'file', :value => 'realtravel/${f:entity}/load/$
{f:entity}_match'
subprocess :ref => 'kettle_run'
_if :test => '${f:skip_merge} is not set' do
sequence do
set :f => 'file', :value => "realtravel/${f:entity}/load/$
{f:entity}_add"
subprocess :ref => 'kettle_exec'
set :f => 'file', :value => "realtravel/${f:entity}/load/rt_$
{f:entity}_load"
subprocess :ref => 'kettle_exec'
end
end
set :f => "args", :value => "-Dpartner=${f:partner} \\\\\"-
Dconditions=destination_type is null\\\\\""
set :f => 'file', :value => 'realtravel/${f:entity}/load/$
{f:entity}_match'
subprocess :ref => 'kettle_run'
_if :test => '${f:skip_merge} is not set' do
sequence do
set :f => 'file', :value => "realtravel/${f:entity}/load/$
{f:entity}_add"
subprocess :ref => 'kettle_exec'
set :f => 'file', :value => "realtravel/${f:entity}/load/rt_$
{f:entity}_load"
subprocess :ref => 'kettle_exec'
end
end
end
end
define :partner_load_entity do
sequence do
echo "loading ${f:entity} skip rtops: ${f:skip_rtops} skip
rtsite: ${f:skip_rtsite}"
set :f => "args", :value => "-Dpartner=${f:partner}"
_if :test => "${f:skip_ext} is not set" do
sequence do
_if :test => "${f:skip_load} is not set" do
sequence do
current_time
set :f => 'time', :value => '${f:__result__}'
subprocess :ref => 'process_reset'
sequence do
set :f => 'sql', :value => "update ext_${f:entity} set
flags = flags & ~(1::bigint << 2) where flags & (1::bigint << 2) != 0
and ext_id_owner ilike '${f:partner}'"
sql
end
set :f => 'file', :value => "realtravel/${f:entity}/load/
${f:entity}_load"
subprocess :ref => 'kettle_run'
set :f => 's_size', :value => RT.property('sample.size')
sequence do
_if :test => "${f:s_size} == 0" do
_if :test => "${f:partner} != realtravel" do
sequence do
set :f => 'sql', :value => "DELETE FROM ext_$
{f:entity} where (updated_at is null or extract(EPOCH from
updated_at) < ${f:time}) and ext_id_owner ilike '${f:partner}'"
echo ">${f:sql}"
sql
end
end
end
end
end
end
sequence do
_if :test => "${f:skip_geo} is not set" do
sequence do
set :f => 'file', :value => 'realtravel/${f:entity}/
load/${f:entity}_geocode'
subprocess :ref => 'kettle_run'
end
end
end
=begin
_if :test => "${f:entity} == locations" do
_if :test => "${f:skip_assets} is not set" do
sequence do
set :f => 'file', :value => 'realtravel/${f:entity}/
load/ext_${f:entity}_assets'
subprocess :ref => 'kettle_run'
end
end
end
=end
end
end
_if :test => "${f:entity} != partners" do
_if :test => "${f:skip_rtsite} is not set" do
sequence do
_if :test => "${f:skip_match} is not set" do
sequence do
set :f => 'file', :value => 'realtravel/${f:entity}/
load/${f:entity}_match'
subprocess :ref => 'kettle_run'
set :f => 's_size', :value => RT.property
('sample.size')
_if :test => "${f:s_size} == 0" do
sequence do
set :f => 'file', :value => 'realtravel/$
{f:entity}/load/${f:entity}_remove_invalid_matches'
subprocess :ref => 'kettle_run'
end
end
end
end
_if :test => "${f:skip_add} is not set" do
sequence do
# _if :test => "${f:heap} is set" do
# set :f => "kettle_heap_temp", :value => '${f:heap}'
# end
# set :f => "heap", :value => RT.property
('kettle.high')
set :f => 'file', :value => "realtravel/${f:entity}/
update/ext_locations_update_method"
subprocess :ref => 'kettle_run'
#set :f => 'entities', :value => '${f:entity}'
#dataimport
end
end
_if :test => "${f:kettle_heap_temp} is set" do
set :f => "heap", :value => '${f:kettle_heap_temp}'
end
end
end
end
=begin
set :f => 'file', :value => "realtravel/tags/load/
tags_partners_add"
subprocess :ref => 'kettle_exec'
_if :test => "${f:kettle_heap_temp} is set" do
set :f => "kettle_heap", :value => '$
{f:kettle_heap_temp}'
unset :f => "kettle_heap"
end
set :f => 'file', :value => "realtravel/${f:entity}/load/$
{f:entity}_tags"
subprocess :ref => 'kettle_run'
=end
#
# _if :test => "${f:entity} == documents" do
# sequence do
# set :f => 'file', :value => "realtravel/${f:entity}/
load/${f:entity}_parent"
# subprocess :ref => 'kettle_run'
# end
# end
##optimizesearch
#reloadsearch
end
end
define :archive_process do
sequence do
set :f => 'dir', :value => "#{RT.property('rt.data')}/partners/$
{f:partner}"
set :f => 'size_ext', :value => ''
_if :test => '${f:process} == extract' do
sequence do
set :f => 'entity', :value => "locations" # FIXME
set :f => 'size', :value => RT.property('sample.size')
_if :test => "${f:size} != 0" do
set :f => 'size_ext', :value => '_${f:size}'
end
end
end
_if :test => '${f:process} == extract' do
sequence do
# archive the partner table
set :f => 'filter', :value => "db"
set :f => 'extension', :value => "${f:size_ext}"
set :f => 'archive_dir', :value => "${f:dir}/${f:process}/$
{f:entity}"
set :f => 'table', :value => "ext_${f:partner}_${f:entity}"
set :f => 'db', :value => "db.ops.db"
set :f => 'archive', :value => "${f:process}"
archive
unset :f => 'extension'
unset :f => 'archive_dir'
unset :f => 'table'
unset :f => 'db'
unset :f => 'archive'
end
end
set :f => 'file', :value => "${f:partner}_${f:process}$
{f:size_ext}"
set :f => 'file_ext', :value => ".tgz"
set :f => 'sync_direction', :value => 'up'
set :f => 'compress', :value => '${f:process}'
sync
end
end
define :remove_invalid_links do
sequence do
set :f => 'sql', :value => 'TRUNCATE tag_dimensions'
sql
end
end
define :restore_process do
sequence do
echo "Restoring process ${f:process} for partner: ${f:partner}"
set :f => 'size_ext', :value => ''
_if :test => '${f:process} == extract' do
sequence do
set :f => 'entity', :value => "locations" # FIXME
set :f => 'size', :value => RT.property('sample.size')
_if :test => "${f:size} != 0" do
set :f => 'size_ext', :value => '_${f:size}'
end
end
end
set :f => 'process', :value => '${f:restore_process}'
set :f => 'dir', :value => "#{RT.property('rt.data')}/partners/$
{f:partner}"
set :f => 'file', :value => "${f:partner}_${f:process}$
{f:size_ext}"
set :f => 'file_ext', :value => ".tgz"
set :f => 'sync_direction', :value => 'down'
set :f => 'compress', :value => '${f:process}'
sync
_if :test => '${f:process} == extract' do
sequence do
# restore the partner table
set :f => 'filter', :value => "db"
set :f => 'extension', :value => "${f:size_ext}"
set :f => 'archive_dir', :value => "${f:dir}/${f:process}/$
{f:entity}"
set :f => 'archive', :value => "${f:process}"
set :f => 'table', :value => "ext_${f:partner}_${f:entity}"
set :f => 'db', :value => "db.ops.db"
rstr
unset :f => 'extension'
unset :f => 'archive_dir'
unset :f => 'table'
unset :f => 'db'
unset :f => 'archive'
end
end
end
end
define :archive_phase do
sequence do
set :field => 'archive_dbs', :value =>
"db.src:db.db,db.ops.src:db.ops.db"
set :f => 'archive_index', :value => 'true'
set :f => 'archive_logs', :value => 'true'
set :f => 'archive_reports', :value => 'true'
_if :test => "${f:archive} == true" do
set :f => 'archive', :value => 'partners'
end
archiveprocess
end
end
define :restore do
sequence do
_if :test => "${f:restore} == true" do
set :f => "restore", :value => "cleanse"
end
set :f => 'restore_index', :value => 'true'
set :f => 'restore_dbs', :value => "db.src:db.db"
restoreprocess
end
end
define :rt_load_type do
sequence do
set :f => "args", :value => "-Dpartner=${f:partner} -Dtype=$
{f:type}"
subprocess :ref => 'kettle_exec'
end
end
define :kettle_exec do
sequence do
kettle
_if :test => "${f:__result__} == 1" do
_cancel_process
end
end
end
define :kettle_run do
sequence do
_if :test => "${f:synchronous} is set" do
iterator :on_value => "${f:filters}", :to_field => "filter" do
subprocess :ref => 'kettle_exec'
end
concurrent_iterator :on_value => "${f:filters}", :to_field =>
"filter" do
subprocess :ref => 'kettle_exec'
end
end
end
end
sequence do
set :f => "timer", :value => "Partners Phase"
set :f => "sample_size", :value => RT.property('sample.size')
set :f => "skip_tags", :value => 'true'
_if :test => "${f:phase} is not set" do
set :f => "phase", :value => "partners"
end
set :f => 'log_file', :value => '${f:phase}.out'
_if :test => "${f:console} is not set" do
set :f => 'console', :value => RT.property('log.console')
end
start_process
_if :test => "${f:restore} is set" do
subprocess :ref => "restore"
# else
end
iterator :on_value => "${f:partners}", :to_field => "partner" do
subprocess :ref => 'partner'
end
# Archive Phase
_if :test => "${f:archive} is set" do
subprocess :ref => "archive_phase"
end
stop_process
end
end
--
you received this message because you are subscribed to the "ruote users" group.
to post : send email to [email protected]
to unsubscribe : send email to [email protected]
more options : http://groups.google.com/group/openwferu-users?hl=en
