From: Jan Provaznik <[email protected]>
Repository xmls are now downloaded and parsed in rake task (rake
dc:prepare_repos) and parsed groups and packages are saved marshalized.
This method is faster and requires significantly less memory than parsing
primary.xml for each request. Anyway this is solution for beta release, final
solution will depend on UI selection model.
Also not all packages were listed in software selection, and if we list all
packages (really long list - 16000 pkgs), it's quite slow, so for beta release
this patch displays only grouped packages and link "Show unsorted packages" at
the end (when clicking on this link all packages are showed).
---
src/app/util/repository_manager.rb | 11 --
.../util/repository_manager/abstract_repository.rb | 3 +-
.../util/repository_manager/comps_repository.rb | 139 ++++++++++----------
src/app/util/repository_manager/pulp_repository.rb | 13 --
src/app/views/templates/_content_selection.haml | 12 ++-
src/app/views/templates/new.haml | 13 ++-
src/lib/tasks/dc_tasks.rake | 19 +---
7 files changed, 90 insertions(+), 120 deletions(-)
diff --git a/src/app/util/repository_manager.rb
b/src/app/util/repository_manager.rb
index d34ccf5..e165744 100644
--- a/src/app/util/repository_manager.rb
+++ b/src/app/util/repository_manager.rb
@@ -47,17 +47,6 @@ class RepositoryManager
return @all_groups
end
- def all_packages(repository = nil)
- unless @all_packages
- @all_packages = []
- repositories.each do |r|
- next if repository and repository != 'all' and repository != r.id
- @all_packages += r.packages
- end
- end
- return @all_packages
- end
-
def all_groups_with_tagged_selected_packages(pkgs, repository = nil)
groups = all_groups(repository)
groups.each_value do |group|
diff --git a/src/app/util/repository_manager/abstract_repository.rb
b/src/app/util/repository_manager/abstract_repository.rb
index baa72e7..5400ae1 100644
--- a/src/app/util/repository_manager/abstract_repository.rb
+++ b/src/app/util/repository_manager/abstract_repository.rb
@@ -1,10 +1,11 @@
class AbstractRepository
- attr_reader :id, :name, :baseurl, :yumurl
+ attr_reader :id, :name, :baseurl, :yumurl, :type
def initialize(conf)
@id = conf['id']
@name = conf['name']
@baseurl = conf['baseurl']
@yumurl = conf['yumurl'] || conf['baseurl']
+ @type = conf['type']
end
end
diff --git a/src/app/util/repository_manager/comps_repository.rb
b/src/app/util/repository_manager/comps_repository.rb
index 4454e5a..80391ef 100644
--- a/src/app/util/repository_manager/comps_repository.rb
+++ b/src/app/util/repository_manager/comps_repository.rb
@@ -5,113 +5,108 @@ class CompsRepository < AbstractRepository
def initialize(conf)
super
@baseurl = conf['baseurl']
+ @cache_dir = "#{RAILS_ROOT}/config/image_descriptor_xmls"
+ @cache_file = File.join(@cache_dir, "#...@id}.data")
+ @repomd_url = File.join(@baseurl, 'repodata', 'repomd.xml')
end
- def packages
- packages = []
- get_packages_nodes.each do |node|
- name = node.at_xpath('./xmlns:name/child::text()')
- group = node.at_xpath('./xmlns:format/rpm:group/child::text()')
- description = node.at_xpath('./xmlns:description/child::text()')
- next unless name and group
- packages << {
- :repository_id => @id,
- :name => name.text,
- :description => description ? description.text : '',
- }
+ def groups
+ begin
+ @groups ||= Marshal.load(File.open(@cache_file, 'r'))
+ rescue Errno::ENOENT
+ raise "failed to read cached packages info, run 'rake dc:prepare_repos'"
end
- return packages
end
- def groups
- groups = {}
- get_groups_nodes.each do |g|
- pkgs = get_group_packages(g)
+ def prepare_repo
+ grps = {}
+ @all_pkgs = pkg_names
+ group_nodes.each do |g|
+ pkgs = group_packages(g)
next if pkgs.empty?
name = g.at_xpath('name').text
- groups[name] = {
+ grps[name] = {
:name => name,
- :description => (t = g.at_xpath('description')) ? t.text : '',
:repository_id => @id,
- :packages => pkgs,
+ :packages => pkgs
}
end
- return groups
- end
- def download_xml(type)
- begin
- url = get_url(type)
- rescue
- return ''
+ other = {}
+ @all_pkgs.each do |pkg, val|
+ other[pkg] = {:type => 'optional'} if val != :listed
end
-
- xml_data = open(url)
- if url =~ /\.gz$/
- return Zlib::GzipReader.new(xml_data).read
- else
- return xml_data.read
+ if other.size > 0
+ name = 'unsorted'
+ grps[name] = {
+ :name => name,
+ :repository_id => @id,
+ :packages => other
+ }
end
+
+ Dir.mkdir(@cache_dir) unless File.directory?(@cache_dir)
+ Marshal.dump(grps, File.open(@cache_file, 'w'))
end
private
- def get_xml(type)
- # FIXME: I'm not sure config is right dir for automatic storing of
- # xml files, but this should be temporary solution until Image Repo is
- # done
- xml_dir = "#{RAILS_ROOT}/config/image_descriptor_xmls"
- xml_file = "#{xml_dir}/#...@id}.#{type}.xml"
- begin
- return File.open(xml_file) { |f| f.read }
- rescue
- xml = download_xml(type)
- Dir.mkdir(xml_dir) unless File.directory?(xml_dir)
- File.open(xml_file, 'w') { |f| f.write xml }
- return xml
- end
+ def group_nodes
+ return [] unless data = group_xml
+ xml = Nokogiri::XML(data)
+ xml.xpath('/comps/group')
end
- def get_group_packages(group_node)
+ def pkg_names
pkgs = {}
- group_node.xpath('packagelist/packagereq').each do |p|
- (pkgs[p.text] ||= {})[:type] = p.attr('type')
+ xml = Nokogiri::XML(primary_xml)
+ xml.xpath('/xmlns:metadata/xmlns:package').each do |node|
+ next unless name = node.at_xpath('./xmlns:name/child::text()')
+ pkgs[name.text] = true
end
return pkgs
end
- def get_packages_nodes
- unless @packages_nodes
- data = get_xml('primary')
- xml = Nokogiri::XML(data)
- @packages_nodes = xml.xpath('/xmlns:metadata/xmlns:package')
+ def download_xml(url)
+ resp = Typhoeus::Request.get(url, :timeout => 30000, :follow_location =>
true, :max_redirects => 3)
+ unless resp.code == 200
+ raise "failed to fetch #{url}: #{resp.body}"
+ end
+ if url =~ /\.gz$/
+ return Zlib::GzipReader.new(StringIO.new(resp.body)).read
+ else
+ return resp.body
end
- return @packages_nodes
end
- def get_groups_nodes
- unless @groups_nodes
- data = get_xml('group')
- xml = Nokogiri::XML(data)
- @groups_nodes = xml.xpath('/comps/group')
+ def repomd
+ @repomd ||= Nokogiri::XML(download_xml(@repomd_url))
+ end
+
+ def primary_xml
+ unless primary =
repomd.xpath("/xmlns:repomd/xmlns:da...@type=\"primary\"]/xmlns:location").first
+ raise "there is no 'primary' info in the repomd.xml (#...@repomd_url})"
end
- return @groups_nodes
+ download_xml(File.join(@baseurl, primary['href']))
end
- def get_url(type)
- if type == 'repomd'
- return File.join(@baseurl, 'repodata', 'repomd.xml')
+ def group_xml
+ # we don't raise exception if group is missing - group is not required
+ if group =
repomd.xpath("/xmlns:repomd/xmlns:da...@type=\"group\"]/xmlns:location").first
+ download_xml(File.join(@baseurl, group['href']))
else
- location =
repomd.xpath("/xmlns:repomd/xmlns:da...@type=\"#{type}\"]/xmlns:location").first
- raise "location for #{type} data not found" unless location
- return File.join(@baseurl, location['href'])
+ nil
end
end
- def repomd
- unless @repomd
- @repomd = Nokogiri::XML(get_xml('repomd'))
+ def group_packages(group_node)
+ pkgs = {}
+ group_node.xpath('packagelist/packagereq').each do |p|
+ pkg_name = p.text
+ next unless val = @all_pkgs[pkg_name]
+ val = :listed
+ (pkgs[pkg_name] ||= {})[:type] = p.attr('type')
end
- return @repomd
+ return pkgs
end
end
diff --git a/src/app/util/repository_manager/pulp_repository.rb
b/src/app/util/repository_manager/pulp_repository.rb
index 8dbd9ec..a0ea005 100644
--- a/src/app/util/repository_manager/pulp_repository.rb
+++ b/src/app/util/repository_manager/pulp_repository.rb
@@ -29,22 +29,9 @@ class PulpRepository < AbstractRepository
def initialize(conf)
super
- @packages_url = File.join(strip_path(@baseurl), conf['packages'])
@groups_url = File.join(strip_path(@baseurl), conf['packagegroups'])
end
- def packages
- packages = []
- WrappedRestClient.get(@packages_url, HTTP_OPTS).each_value do |info|
- packages << {
- :repository_id => @id,
- :name => info['name'],
- :description => info['description'],
- }
- end
- return packages
- end
-
def groups
groups = {}
WrappedRestClient.get(@groups_url, HTTP_OPTS).each do |id, info|
diff --git a/src/app/views/templates/_content_selection.haml
b/src/app/views/templates/_content_selection.haml
index e43b7a2..2372a4a 100644
--- a/src/app/views/templates/_content_selection.haml
+++ b/src/app/views/templates/_content_selection.haml
@@ -28,19 +28,25 @@
-# = submit_tag "Add Selected", :name => "add_selected", :class =>
"grid_2 alpha", :id => "do_add_software"
-# = submit_tag "Cancel", :name => "cancel", :class => "grid_2", :id =>
"cancel_add_software"
%ul.softwaregroups
- - @groups.keys.sort.each do |group|
+ - groups = @groups.keys.sort
+ - unsorted = groups.delete('unsorted')
+ - groups.push('unsorted') if params[:show_unsorted] and unsorted
+ - groups.each do |group|
- group_sel = @groups[group][:selected]
- group_id = group.gsub(/\s/, '_')
%li
= check_box_tag 'groups[]', group, group_sel, :disabled =>
group_sel, :id => "group_#{group_id}"
= label_tag "group_#{group_id}", group
%ul{:class => "packages group_#{group_id}"}
- - @groups[group][:packages].each do |pkg, info|
- - pkg_sel = info[:selected] ? true : false
+ - pkgs = @groups[group][:packages]
+ - pkgs.keys.sort.each do |pkg|
+ - pkg_sel = pkgs[pkg][:selected] ? true : false
- pkg_id = pkg.gsub(/\s/, '_')
%li
= check_box_tag 'packages[]', pkg, pkg_sel, :disabled =>
pkg_sel, :id => "package_#{pkg_id}"
= label_tag "package_#{pkg_id}", pkg
+
+ = link_to "Show unsorted packages", {:action => 'create',
:add_software_form => true, :show_unsorted => true, 'tpl[id]' => @tpl.id}, {:id
=> 'switch_all_link'} unless params[:show_unsorted]
%fieldset.clearfix
= submit_tag "Add Selected", :name => "add_selected", :class => "grid_2
alpha", :id => "do_add_software"
= submit_tag "Cancel", :name => "cancel", :class => "grid_2", :id =>
"cancel_add_software"
diff --git a/src/app/views/templates/new.haml b/src/app/views/templates/new.haml
index 4963d72..42fc85d 100644
--- a/src/app/views/templates/new.haml
+++ b/src/app/views/templates/new.haml
@@ -2,13 +2,14 @@
$(document).ready(function() {
var $container = $('#package_selection_list'),
$submit = $('#add_software_button');
- $submit.click(function(e) {
- var list_url = '#{url_for :action => 'create', :ajax => true,
:add_software_form => true}';
- var list_data = {'tpl[id]': $("input[name='tpl[id]']").val() || ''};
+ $submit.click(function(e, show_all) {
+ var list_url = '#{url_for :action => 'create', :add_software_form =>
true}';
+ var list_all_url = '#{url_for :action => 'create', :add_software_form =>
true, :show_unsorted => true}';
+ var list_data = {'tpl[id]': $("input[name='tpl[id]']").val() || '',
ajax: true};
e.preventDefault();
$(this).hide();
$container.empty().show().addClass('loading');
- $container.load(list_url, list_data, function() {
+ $container.load(show_all ? list_all_url : list_url, list_data,
function() {
$container.removeClass('loading');
$('#do_add_software').click(function(e) {
var url = '#{url_for :action => 'add_software', :ajax => true,
:add_selected => true}';
@@ -24,6 +25,10 @@
$submit.show();
});
});
+ $('#switch_all_link').click(function(e) {
+ e.preventDefault();
+ $submit.trigger('click', [true]);
+ });
$('#cancel_add_software').click(function(e) {
e.preventDefault();
$container.hide();
diff --git a/src/lib/tasks/dc_tasks.rake b/src/lib/tasks/dc_tasks.rake
index 4b9d4e4..eab7001 100644
--- a/src/lib/tasks/dc_tasks.rake
+++ b/src/lib/tasks/dc_tasks.rake
@@ -52,23 +52,10 @@ namespace :dc do
end
- desc 'Download repository xml files'
- task :download_repos => :environment do |t, args|
+ desc 'Download and parse repository xml files'
+ task :prepare_repos => :environment do |t, args|
require 'util/repository_manager'
-
- base_dir = "#{RAILS_ROOT}/config/image_descriptor_xmls"
- Dir.mkdir(base_dir) unless File.directory?(base_dir)
-
- mgr = RepositoryManager.new
- mgr.repositories.keys.each do |repid|
- rep = mgr.get_repository(repid)
-
- %w(repomd primary group).each do |type|
- path = "#{base_dir}/#{repid}.#{type}.xml"
- puts "Downloading #{type} file for #{repid} repository -> #{path}"
- File.open(path , "w") { |f| f.write rep.download_xml(type) }
- end
- end
+ RepositoryManager.new.repositories.each { |repo| repo.prepare_repo if
repo.type == 'xml' }
end
desc 'Create user "admin" for CloudEngine'
--
1.7.2.3
_______________________________________________
deltacloud-devel mailing list
[email protected]
https://fedorahosted.org/mailman/listinfo/deltacloud-devel
