I had to make slight changes because in my version Repository does not have 
@property repo_path anymore (or not yet).
It now also assigns one category (which has to exist before).

thank you very much,
ido


#!/usr/bin/env python

'''
This script requires the Galaxy instance to use Postgres for database storage.  
To run this script, use "sh copy_repository.sh" from this directory
'''

import sys, os, ConfigParser

assert sys.version_info[:2] >= ( 2, 4 )
new_path = [ os.path.join( os.getcwd(), "lib" ) ]
new_path.extend( sys.path[1:] ) # remove scripts/ from the path
sys.path = new_path

from galaxy import eggs

import pkg_resources
pkg_resources.require( "psycopg2" )
import psycopg2

pkg_resources.require( 'elementtree' )
from elementtree import ElementTree, ElementInclude

import galaxy.webapps.community.app
from galaxy import util
from mercurial import hg, ui, httprepo, commands

def directory_hash_id( id ):
    s = str( id )
    l = len( s )
    # Shortcut -- ids 0-999 go under ../000/
    if l < 4:
        return [ "000" ]
    # Pad with zeros until a multiple of three
    padded = ( ( ( 3 - len( s ) ) % 3 ) * "0" ) + s
    # Drop the last three digits -- 1000 files per directory
    padded = padded[:-3]
    # Break into chunks of three
    return [ padded[i*3:(i+1)*3] for i in range( len( padded ) // 3 ) ]
def add_hgweb_config_entry( repository, repository_path ):
    # Add an entry in the hgweb.config file for a new repository.  This enables calls to repository.repo_path.
    # An entry looks something like: repos/test/mira_assembler = database/community_files/000/repo_123
    hgweb_config = "%s/hgweb.config" %  os.getcwd()
    entry = "repos/%s/%s = %s" % ( repository.user.username, repository.name, repository_path.lstrip( './' ) )
    if os.path.exists( hgweb_config ):
        output = open( hgweb_config, 'a' )
    else:
        output = open( hgweb_config, 'w' )
        output.write( '[paths]\n' )
    output.write( "%s\n" % entry )
    output.close()
def create_hgrc_file( repo_path, username, reponame ):
    # At this point, an entry for the repository is required to be in the hgweb.config
    # file so we can call repository.repo_path.
    # Create a .hg/hgrc file that looks something like this:
    # [web]
    # allow_push = test
    # name = convert_characters1
    # push_ssl = False
    # Upon repository creation, only the owner can push to it ( allow_push setting ),
    # and since we support both http and https, we set push_ssl to False to override
    # the default (which is True) in the mercurial api.
    hgrc_file = os.path.abspath( os.path.join( repo_path, ".hg", "hgrc" ) )
    output = open( hgrc_file, 'w' )
    output.write( '[web]\n' )
    output.write( 'allow_push = %s\n' % username )
    output.write( 'name = %s\n' % reponame )
    output.write( 'push_ssl = false\n' )
    output.flush()
    output.close()
def clone_repository( repository_clone_url, repository_path ):
    """Clone the repository up to the specified changeset_revision.  No subsequent revisions will be present in the cloned repository."""
    print( str( repository_clone_url )+" "+str( repository_path ))
    commands.clone( get_configured_ui(),
                    str( repository_clone_url ),
                    dest=str( repository_path ),
                    pull=True,
                    noupdate=False,
                    rev=[] )
def contains( containing_str, contained_str ):
    return containing_str.lower().find( contained_str.lower() ) >= 0
def copy_repository( app, repository_clone_url, name, description, user_id, username, catname ):
    sa_session = app.model.context
    # Add the repository record to the db
    print "Adding new database record for repository: ", name
    repository = app.model.Repository( name=name,
                                       description=description,
                                       long_description=description,
                                       user_id=user_id )
    
 #   repopath = app.hgweb_config_manager.get_entry( os.path.join( "repos", username, name ) )
    
    # Flush to get the id
    sa_session.add( repository )
    sa_session.flush()
  
    category = sa_session.query( app.model.Category ).filter( app.model.Category.table.c.name == catname ).all()[0]
    assoc = app.model.RepositoryCategoryAssociation(repository, category)
    sa_session.add(assoc)
    sa_session.flush()

    # Determine the repository's repo_path on disk
    clone_dir = os.path.join( app.config.file_path, *directory_hash_id( repository.id ) )
    # Create directory if it does not exist
    if not os.path.exists( clone_dir ):
        os.makedirs( clone_dir )
    # Define repo name inside hashed directory
    repository_path = os.path.join( clone_dir, "repo_%d" % repository.id )
    # Create local repository directory
    if not os.path.exists( repository_path ):
        os.makedirs( repository_path )
    print "Creating local repository at: ", repository_path
    # Clone the repository
    clone_repository( repository_clone_url, os.path.abspath( repository_path ) )
    repo = hg.repository( get_configured_ui(), repository_path )
    # Update the repository files for browsing.
    print "Updating repository"
    update_repository( repo )
    # Add an entry in the hgweb.config file for the local repository.  This enables calls to repository.repo_path
    print "Adding entry to hgweb.config"
    add_hgweb_config_entry( repository, repository_path )
    # Create a .hg/hgrc file for the local repository
    print "Creating hgrc file"
    create_hgrc_file( repository_path, username, name )
    print "Repository '%s' has been copied." % repository.name
def create_user( app, email, password, username ):
    sa_session = app.model.context
    user = app.model.User( email=email )
    user.set_password_cleartext( password )
    user.username = username
    sa_session.add( user )
    sa_session.flush()
    return user
def get_configured_ui():
    # Configure any desired ui settings.
    _ui = ui.ui()
    # The following will suppress all messages.  This is
    # the same as adding the following setting to the repo
    # hgrc file' [ui] section:
    # quiet = True
    _ui.setconfig( 'ui', 'quiet', True )
    return _ui
def get_user( app, email ):
    sa_session = app.model.context
    return sa_session.query( app.model.User ) \
                     .filter( app.model.User.table.c.email == email ) \
                     .first()
def update_repository( repo, ctx_rev=None ):
    commands.update( get_configured_ui(),
                     repo,
                     rev=ctx_rev )

def main():
    if len( sys.argv ) < 3:
        print "Usage: python %s <Tool shed config file> copy_repository.xml" % sys.argv[0]
        sys.exit( 0 )
    email = 'user@changeme'
    password = 'password'
    username = 'username'
    catname = 'category'
    # community_wsgi.ini file
    ini_file = sys.argv[ 1 ]
    repository_xml_file = sys.argv[ 2 ]
    conf_parser = ConfigParser.ConfigParser( { 'here' : os.getcwd() } )
    conf_parser.read( ini_file )
    try:
        db_conn_str = conf_parser.get( "app:main", "database_connection" )
    except ConfigParser.NoOptionError, e:
        db_conn_str = conf_parser.get( "app:main", "database_file" )
    print 'DB Connection: ', db_conn_str
    # Determine db connection - only postgres is supported
    if contains( db_conn_str, '///' ) and contains( db_conn_str, '?' ) and contains( db_conn_str, '&' ):
        # postgres:///galaxy_test?user=postgres&password=postgres 
        db_str = db_conn_str.split( '///' )[ 1 ]
        db_name = db_str.split( '?' )[ 0 ]
        db_user = db_str.split( '?' )[ 1 ].split( '&' )[ 0 ].split( '=' )[ 1 ]
        db_password = db_str.split( '?' )[ 1 ].split( '&' )[ 1 ].split( '=' )[ 1 ]
    elif contains( db_conn_str, '//' ) and contains( db_conn_str, ':' ):
        # dialect://user:password@host/db_name
        db_name = db_conn_str.split( '/' )[ -1 ]
        db_user = db_conn_str.split( '//' )[ 1 ].split( ':' )[ 0 ]
    # Instantiate app
    configuration = {}
    for key, value in conf_parser.items( "app:main" ):
        configuration[ key ] = value
    app = galaxy.webapps.community.app.UniverseApplication( global_conf=dict( __file__=ini_file ), **configuration )
    sa_session = app.model.context
    # Make sure we have a user.
    user = get_user( app, email )
    if user is None:
        user = create_user( app, email, password, username )
    hgweb_config = "%s/hgweb.config" %  os.getcwd()
    # Parse the copy_repository.xml file to determine what repositories should be copied.
    tree = util.parse_xml( repository_xml_file )
    root = tree.getroot()
    tool_shed_url = root.get( 'url' )
    for repository_elem in root:
        repository_name = repository_elem.get( 'name' )
        repository_description = repository_elem.get( 'description' )
        repository_owner = repository_elem.get( 'owner' )
        repository_clone_url = os.path.join( tool_shed_url, 'repos', repository_owner, repository_name )
        copy_repository( app, repository_clone_url, repository_name, repository_description, user.id, user.username, catname )
    app.shutdown()
    sys.exit(0)

if __name__ == "__main__":
    main()



On Dec 4, 2012, at 5:27 PM, Greg Von Kuster wrote:

> Hi Ido,
> 
> On Dec 4, 2012, at 10:26 AM, Ido Tamir wrote:
> 
> > Hi,
> > how do I get tools from one local toolshed (or the main galaxy toolshed)
> > into another tool-shed? Is it possible to clone the content of a complete 
> > tool-shed - all repositories
> > at once into another tool-shed?
> 
> Not at the current time, but I've attached a script that I use in my 
> development environment that sort of does what you want.  
> 
> When you run it for the first time, make sure it installs to a tool shed you 
> don't care about destroying just so you can see how it works.  It uses the 
> configuration settings in commnity_wsgi.ini, so make sure you set things 
> accordingly.
> 
> here's what you do:
> 
> 1. Place the 3 attached files in ~/scripts/tool_shed
> 2. Define the tool shed you want to copy repositories form in 
> copy_repository.xml
> 3. Define any number of valid <repository> tags in copy_repository.xml
> 4. run the copy_repositoyr.sh shell script from the ~/scripts/tool_shed 
> directory
> 5. When all repositories have been copied into you local tool shed, start it 
> up and make sure to reset all metadata on all repositories as the script does 
> not do so.  You can dfo this from the Tool Shed's admin interface.
> 
> I do not have time to support it, so you're on your own if you want to use it 
> as is.  If you enhance it to the point that it would be useful to the 
> community, please contribute it back to me and I'll make sure it is included 
> in the Galaxy code base.
> 
> 
> 
> 
> 
> 
> 
> 
> 
> > Can I then import all the tools at once into a local galaxy instance?
> 
> Not currently, but this is planned for the near future ( within a few months 
> ).
> 
> > 
> > We have a group with their own tool-shed. Unfortunately, I accessed it 
> > directly
> > without a proxy on its port which leads to a problem described in issue 825 
> > in galaxy-central.
> > 
> > I want to 
> > a) clone the tool-shed as it is into my own tool-shed (rsync?)
> > b) selectively or bulk install these tools into my galaxy server.
> > 
> > thank you very much,
> > ido
> > 
> > 
> > 
> > ___________________________________________________________
> > Please keep all replies on the list by using "reply all"
> > in your mail client.  To manage your subscriptions to this
> > and other Galaxy lists, please use the interface at:
> > 
> >  http://lists.bx.psu.edu/
> 
> <copy_repository.py><copy_repository.sh><copy_repository.xml>

___________________________________________________________
Please keep all replies on the list by using "reply all"
in your mail client.  To manage your subscriptions to this
and other Galaxy lists, please use the interface at:

  http://lists.bx.psu.edu/

Reply via email to