Re: [MarkLogic Dev General] user installation using a query

Eric Bloch Wed, 03 Mar 2010 22:15:35 -0800

Hi Karl,

Here's what we do via puppet, which I'm hoping you can kind of read as pseudo code. In particular, I've removed our credentials. If you copy any of the code, note that you'd want to use your own USER and PASSWORD (and url encode correctly in one place).

In a nut-shell, in puppet we:

1) pull down our app via svn
2) install the ML rpm
3) copy a setup script into /opt/MarkLogic/Admin
4) create a link from /opt/MarkLogic/Modules to our app code that needs amps
4) define a bunch of stuff to use
5) exec the commands we defined with delays as need be.

I've attached a copy of setup-markmail.xqy, which has a whole bunch of ML admin API calls - you can see us clearing out the default 4.[01] stuff there and creating app servers that point to our app in the filesystem as well. You'll note that it has two phases and we run some stuff in between the two phases. In particular, we run another .xqy on an app server we create during stage 1. That script creates a whole bunch of roles, amps, users, etc... ; you probably don't need that- I'm just explaining in case you wonder what's going on here.

The puppet code is below.

Good luck!
-Eric

PS. Credit for the initial work here goes to Ryan Grimm - I've just been a happy user and maintainer.

define marklogic(
    $version,
    $licensee,
    $license_key,
    $deployment = "development"
) {
    include remove_httpd


    svn_client { "/usr/local/markmail/xquery":
        source => "http://svn.markmail.int/code/ark/trunk/xquery",
        path => "/usr/local/markmail",
        directory_name => "xquery,
        user => "puppet",
        password =>"master"
    }

    package { "MarkLogic":
        ensure => "installed",
        provider => "rpm",
        source => "http://puppet.markmail.int/marklogic/files/MarkLogic-$version.x86_64.rpm",
    }

    file { "/opt/MarkLogic/Admin/setup-markmail.xqy":
        source => "puppet://puppet.markmail.int/marklogic/setup-markmail.xqy",
        require => Package["MarkLogic"]
    }

    file { "/usr/local/markmail/xquery/userui/lib/linked":
        ensure => "/opt/MarkLogic/modules/MarkMail"
    }

    $license = "/usr/bin/wget --quiet -O - \"http://localhost:8001/license-go.xqy?licensee=$licensee&license-key=$license_key\" > /dev/null"
    $accept = $deployment ? {
        development => "/usr/bin/wget --quiet -O - \"http://localhost:8001/agree-go.xqy?accepted-agreement=standard&ok=accept\" > /dev/null",
        default => "/bin/true"
    }
    $initialize = $deployment ? {
        development => "/usr/bin/wget --quiet -O - \"http://localhost:8001/initialize-go.xqy\" > /dev/null",
        default => "/bin/true"
    }
    $join = $deployment ? {
        development => "/usr/bin/wget --quiet -O - \"http://localhost:8001/join-admin-go.xqy?new-server=$fqdn&new-server-port=8001&bind=7999&connect=7999&server=&port=8001&cancel=cancel\" > /dev/null",
        default => "/bin/true"
    }
    $security = $deployment ? {
        development => "/usr/bin/wget --user=USER --password=PASSWORD --quiet -O - \"http://localhost:8001/security-install-go.xqy?user=USER&password1=PASSWORD&password2=PASSWORD&realm=public\" > /dev/null",
        default => "/bin/true"
    }
    $setup = $deployment ? {
        development => "/usr/bin/wget --user=USER --password=PASSWORD --quiet -O - http://localhost:8001/setup-markmail.xqy?stage=1 > /dev/nul
l",
        default => "/bin/true"
    }    $app_security = $deployment ? {
        development => "/usr/bin/wget --user=USER --password=PASSWORD --quiet -O - \"http://localhost:8003/security/sec-setup.xqy?mode=update&
config=mm-config_4.0.xml\" > /dev/null",
        default => "/bin/true"
    }
    $set_default_user = $deployment ? {
        development => "/usr/bin/wget --user=USER --password=PASSWORD --quiet -O - http://localhost:8001/setup-markmail.xqy?stage=2 > /dev/nul
l",
        default => "/bin/true"
    }

    exec { "enter_marklogic_license_info":
        command => "$license && /bin/sleep 20",
        refreshonly => true,
        require => Service["MarkLogic"],
        creates => "/var/opt/MarkLogic/server.xml",
        notify => Exec["accept_marklogic_license"]
    }

    exec { "accept_marklogic_license":
        command => "$accept && /bin/sleep 20",
        refreshonly => true,
        require => [
            Service["MarkLogic"],
            Exec["enter_marklogic_license_info"]
        ],
        notify => Exec["initialize_marklogic"]
    }

    exec { "initialize_marklogic":
        command => "$initialize && /bin/sleep 20",
        refreshonly => true,
        require => [
            Service["MarkLogic"],
            Exec["accept_marklogic_license"]
        ],
        notify => Exec["join_marklogic_cluster"]
    }

    exec { "join_marklogic_cluster":
        command => "$join && /bin/sleep 5",
        refreshonly => true,
        require => [
            Service["MarkLogic"],
            Exec["initialize_marklogic"]
        ],
        notify => Exec["install_marklogic_security"]
    }

    exec { "install_marklogic_security":
        command => "$security && /bin/sleep 5",
        refreshonly => true,
        require => [
            Service["MarkLogic"],
            Exec["join_marklogic_cluster"]
        ],
        notify => Exec["setup_markmail_application"]
    }

    exec { "setup_markmail_application":
        command => "$setup && /bin/sleep 20",
        refreshonly => true,
        require => [
            Service["MarkLogic"],
            Exec["install_marklogic_security"]
        ],
        notify => Exec["setup_markmail_security"]
    }

    exec { "setup_markmail_security":
        command => "$app_security && /bin/sleep 5",
        refreshonly => true,
        require => [
            Service["MarkLogic"],
            Exec["setup_markmail_application"]
        ],
        notify => Exec["set_markmail_default_user"]
    }

    exec { "set_markmail_default_user":
        command => "$set_default_user",
        refreshonly => true,
        require => [
            Service["MarkLogic"],
            Exec["setup_markmail_security"]
        ]
    }

    service { "MarkLogic":
        name => "MarkLogic",
        enable => "true",
        require => [
            Package["MarkLogic"],
            File["/opt/MarkLogic/Admin/setup-markmail.xqy"]
        ],
        notify => Exec["enter_marklogic_license_info"],
        ensure => "running"
    }

-Eric

Karl Erisman wrote:

Eric,

The procedure you describe sounds close to what I've been imagining.
If you get a chance, I would be interested in any other details you
care to provide (what you do to work around the server restarting,
which default dbs/forests/servers you wipe out, etc.).

Karl

On Tue, Mar 2, 2010 at 11:54 AM, Eric Bloch <[email protected]> wrote:

Here's what we do in markmail to add a new node.  We use puppet to drive, but you could do this in a scripting language and/or installer tool of your choice.

0) checkout (svn co) our application code on to the box
1) copy marklogic rpm  to the box and install
2) wget a few URLs within the ML Admin app to do a few things, including install our license
3) copy a setup-markmail.xqy script to the Admin app filesystem
4) wget the url of the setup script
* this script wipes out default dbs, forests, and app servers we don't want
* set up app-servers, dbs, forests,  index settings, and the like.
* setup security users, roles, amps, etc...

If you're interested in details, I can share more.  It's a little tricky as some of the admin ops result in the server restarting.

-Eric

________________________________________
From: [email protected] [[email protected]] On Behalf Of Karl Erisman [[email protected]]
Sent: Tuesday, March 02, 2010 9:35 AM
To: General Mark Logic Developer Discussion
Subject: Re: [MarkLogic Dev General] user installation using a query

If your customer does not yet have MarkLogic installed, they'll have
to use the administrative interface to configure MarkLogic itself, so
the procedure you've outlined seems straightforward enough as long as
your install.xqy itself is "user-friendly" (which should probably mean
that it presents a simple interface, does not ask questions that are
excessively technical, and shows clear confirmation or error messages
based on what happens).

I've been wondering about a broader, but closely-related topic:
release management of production MarkLogic applications in general.
How do others package code for system admins (customers themselves in
Andrew's case; but probably developers or an internal team of
specialists at most organizations) to deploy?  This applies to new
releases or upgrades.

Do you create, for each version of your application, an XQuery install
script (much like the one Andrew plans to use) that performs steps
such as manipulation of app servers, databases, etc. to bring the
configuration up to date with respect to source code changes?  Any
other hints on streamlining the process?

Karl

On Tue, Mar 2, 2010 at 10:41 AM, Andrew Welch <[email protected]> wrote:

Hi,

It seems that everything you can do through the admin screen you can
do through a query... so rather give the customer a list of steps to
follow (create http server, create webdav server etc), I can shield
them from the admin screens and give them a query to run instead.

The question is, what is the user-friendliest way for them to run that
xquery, when the person installing the application is likely to know
very little about MarkLogic?

Is using the Docs dir the best way:

1) Install MarkLogic
2) copy the "install.xqy" to [install dir]/Docs
2) navigate to [hostname]:8000/install.xqy

...or is there another way?



thanks
--
Andrew Welch
http://andrewjwelch.com
Kernow: http://kernowforsaxon.sf.net/
_______________________________________________
General mailing list
[email protected]
http://xqzone.com/mailman/listinfo/general

_______________________________________________
General mailing list
[email protected]
http://xqzone.com/mailman/listinfo/general_______________________________________________
General mailing list
[email protected]
http://xqzone.com/mailman/listinfo/general

_______________________________________________
General mailing list
[email protected]
http://xqzone.com/mailman/listinfo/general

import module namespace admin = "http://marklogic.com/xdmp/admin"; at 
"/MarkLogic/admin.xqy"


let $stage := xs:integer(xdmp:get-request-field("stage", "1"))
let $config := admin:get-configuration()
let $defaultGroupId := admin:group-get-id($config, "Default")
return
if($stage = 1)
then
        (: Remove the Docs server, Documents database and forest :)
        let $config := admin:appserver-delete($config, 
admin:appserver-get-id($config, $defaultGroupId, "Docs"))
        let $save := admin:save-configuration-without-restart($config)

        let $config := admin:database-delete($config, 
xdmp:database("Documents"))
        let $save := admin:save-configuration-without-restart($config)

        let $config := admin:forest-delete($config, 
admin:forest-get-id($config, "Documents"), true())
        let $save := admin:save-configuration-without-restart($config)
        return ()
else
        ();

import module namespace admin = "http://marklogic.com/xdmp/admin"; at 
"/MarkLogic/admin.xqy"


let $stage := xs:integer(xdmp:get-request-field("stage", "1"))
let $config := admin:get-configuration()
let $defaultGroupId := admin:group-get-id($config, "Default")
return
if($stage = 1)
then
        (: Create the MarkMail forests, databases (including Users and 
Sessions) and app servers :)
        let $config := admin:forest-create($config, "MarkMail-001", 
xdmp:host(), ())
        let $config := admin:forest-create($config, "Users-001", xdmp:host(), 
())
        let $config := admin:forest-create($config, "Sessions-001", 
xdmp:host(), ())
        let $config := admin:database-create($config, "MarkMail", 
xdmp:database("Security"), xdmp:database("Schemas"))
        let $config := admin:database-create($config, "Users", 
xdmp:database("Security"), xdmp:database("Schemas"))
        let $config := admin:database-create($config, "Sessions", 
xdmp:database("Security"), xdmp:database("Schemas"))
        let $save := admin:save-configuration-without-restart($config)
        let $config := admin:database-attach-forest($config, 
xdmp:database("MarkMail"), xdmp:forest("MarkMail-001"))
        let $config := admin:database-attach-forest($config, 
xdmp:database("Users"), xdmp:forest("Users-001"))
        let $config := admin:database-attach-forest($config, 
xdmp:database("Sessions"), xdmp:forest("Sessions-001"))
        let $save := admin:save-configuration-without-restart($config)

        let $config := admin:database-set-stemmed-searches($config, 
xdmp:database("MarkMail"), "advanced")
        let $config := admin:database-set-word-searches($config, 
xdmp:database("MarkMail"), true())
        let $config := admin:database-set-word-positions($config, 
xdmp:database("MarkMail"), false())
        let $config := admin:database-set-fast-phrase-searches($config, 
xdmp:database("MarkMail"), true())
        let $config := admin:database-set-fast-case-sensitive-searches($config, 
xdmp:database("MarkMail"), false())
        let $config := 
admin:database-set-fast-diacritic-sensitive-searches($config, 
xdmp:database("MarkMail"), true())

        let $config := admin:database-set-fast-element-word-searches($config, 
xdmp:database("MarkMail"), true())
        let $config := admin:database-set-element-word-positions($config, 
xdmp:database("MarkMail"), true())
        let $config := admin:database-set-fast-element-phrase-searches($config, 
xdmp:database("MarkMail"), true())
        let $config := admin:database-set-element-value-positions($config, 
xdmp:database("MarkMail"), true())

        let $config := admin:database-set-trailing-wildcard-searches($config, 
xdmp:database("MarkMail"), true())
        let $config := 
admin:database-set-trailing-wildcard-word-positions($config, 
xdmp:database("MarkMail"), true())
        let $config := 
admin:database-set-fast-element-trailing-wildcard-searches($config, 
xdmp:database("MarkMail"), true())
        let $config := admin:database-set-three-character-searches($config, 
xdmp:database("MarkMail"), false())
        let $config := 
admin:database-set-three-character-word-positions($config, 
xdmp:database("MarkMail"), false())
        let $config := admin:database-set-two-character-searches($config, 
xdmp:database("MarkMail"), false())
        let $config := admin:database-set-one-character-searches($config, 
xdmp:database("MarkMail"), false())
        let $config := 
admin:database-set-fast-element-character-searches($config, 
xdmp:database("MarkMail"), false())

        let $config := admin:database-set-uri-lexicon($config, 
xdmp:database("MarkMail"), true())
        let $config := admin:database-set-collection-lexicon($config, 
xdmp:database("MarkMail"), false())

        let $config := admin:database-set-reindexer-enable($config, 
xdmp:database("MarkMail"), true())
        let $config := admin:database-set-reindexer-throttle($config, 
xdmp:database("MarkMail"), 3)

        let $config := admin:database-set-directory-creation($config, 
xdmp:database("MarkMail"), "manual")
        let $config := admin:database-set-maintain-last-modified($config, 
xdmp:database("MarkMail"), false())
        let $config := 
admin:database-set-maintain-directory-last-modified($config, 
xdmp:database("MarkMail"), false())
        let $config := admin:database-set-inherit-permissions($config, 
xdmp:database("MarkMail"), false())
        let $config := admin:database-set-inherit-collections($config, 
xdmp:database("MarkMail"), false())
        let $config := admin:database-set-inherit-quality($config, 
xdmp:database("MarkMail"), false())

        let $config := admin:database-set-preallocate-journals($config, 
xdmp:database("MarkMail"), true())
        let $config := admin:database-set-preload-mapped-data($config, 
xdmp:database("MarkMail"), true())

        let $emailewqt := admin:database-element-word-query-through("", "email")
        let $config := admin:database-add-element-word-query-through($config, 
xdmp:database("MarkMail"), $emailewqt)

        let $urlewqt := admin:database-element-word-query-through("", "url")
        let $config := admin:database-add-element-word-query-through($config, 
xdmp:database("MarkMail"), $urlewqt)


        let $spec := admin:database-range-element-attribute-index("dateTime", 
"", "message", "", "date", "", false())
        let $config := 
admin:database-add-range-element-attribute-index($config, 
xdmp:database("MarkMail"), $spec)

        let $spec := admin:database-range-element-attribute-index("string", "", 
"from", "", "personal", "http://marklogic.com/collation/codepoint";, false())
        let $config := 
admin:database-add-range-element-attribute-index($config, 
xdmp:database("MarkMail"), $spec)

        let $spec := admin:database-range-element-attribute-index("string", "", 
"attachment", "", "extension", "http://marklogic.com/collation/codepoint";, 
false())
        let $config := 
admin:database-add-range-element-attribute-index($config, 
xdmp:database("MarkMail"), $spec)

        let $spec := admin:database-range-element-attribute-index("string", "", 
"inner-attachment", "", "extension", 
"http://marklogic.com/collation/codepoint";, false())
        let $config := 
admin:database-add-range-element-attribute-index($config, 
xdmp:database("MarkMail"), $spec)

        let $spec := admin:database-range-element-attribute-index("string", "", 
"message", "", "list", "http://marklogic.com/collation/codepoint";, false())
        let $config := 
admin:database-add-range-element-attribute-index($config, 
xdmp:database("MarkMail"), $spec)

        let $spec := admin:database-range-element-attribute-index("string", "", 
"message", "", "type", "http://marklogic.com/collation/codepoint";, false())
        let $config := 
admin:database-add-range-element-attribute-index($config, 
xdmp:database("MarkMail"), $spec)

        let $spec := admin:database-range-element-attribute-index("string", "", 
"message", "", "year", "http://marklogic.com/collation/codepoint";, false())
        let $config := 
admin:database-add-range-element-attribute-index($config, 
xdmp:database("MarkMail"), $spec)

        let $spec := admin:database-range-element-attribute-index("string", "", 
"message", "", "year-month", "http://marklogic.com/collation/codepoint";, 
false())
        let $config := 
admin:database-add-range-element-attribute-index($config, 
xdmp:database("MarkMail"), $spec)

        let $spec := admin:database-range-element-attribute-index("string", "", 
"message", "", "year-month-day", "http://marklogic.com/collation/codepoint";, 
false())
        let $config := 
admin:database-add-range-element-attribute-index($config, 
xdmp:database("MarkMail"), $spec)

        let $config := admin:http-server-create($config, $defaultGroupId, 
"MarkMail-UI", "/usr/local/markmail/xquery/userui", 8002, "file-system", 
xdmp:database("MarkMail"))
        let $config := admin:http-server-create($config, $defaultGroupId, 
"MarkMail-Admin", "/usr/local/markmail/xquery/admin", 8003, "file-system", 
xdmp:database("MarkMail"))
        let $config := admin:xdbc-server-create($config, $defaultGroupId, 
"MarkMail-Load", "/usr/local/markmail/xquery/load", 8010, "file-system", 
xdmp:database("MarkMail"))

        let $config := admin:appserver-set-collation($config, 
admin:appserver-get-id($config, $defaultGroupId, "MarkMail-UI"), 
"http://marklogic.com/collation/codepoint";)
        let $config := admin:appserver-set-collation($config, 
admin:appserver-get-id($config, $defaultGroupId, "MarkMail-Admin"), 
"http://marklogic.com/collation/codepoint";)
        let $config := admin:appserver-set-collation($config, 
admin:appserver-get-id($config, $defaultGroupId, "MarkMail-Load"), 
"http://marklogic.com/collation/codepoint";)
        let $config := admin:appserver-set-authentication($config, 
admin:appserver-get-id($config, $defaultGroupId, "MarkMail-UI"), 
"application-level")

        let $config := admin:appserver-set-default-xquery-version($config, 
admin:appserver-get-id($config, $defaultGroupId, "MarkMail-UI"), "0.9-ml")
        let $config := admin:appserver-set-default-xquery-version($config, 
admin:appserver-get-id($config, $defaultGroupId, "MarkMail-Admin"), "0.9-ml")
        let $config := admin:appserver-set-default-xquery-version($config, 
admin:appserver-get-id($config, $defaultGroupId, "MarkMail-Load"), "0.9-ml")

        let $config := admin:group-add-namespace($config, $defaultGroupId, (
                <namespace xmlns="http://marklogic.com/xdmp/group";>
                        <prefix>user</prefix>
                        <namespace-uri>http://markmail.org/user</namespace-uri>
                </namespace>,
                <namespace xmlns="http://marklogic.com/xdmp/group";>
                        <prefix>set</prefix>
                        <namespace-uri>http://markmail.org/set</namespace-uri>
                </namespace>,
                <namespace xmlns="http://marklogic.com/xdmp/group";>
                        <prefix>session</prefix>
                        
<namespace-uri>http://markmail.org/session</namespace-uri>
                </namespace>,
                <namespace xmlns="http://marklogic.com/xdmp/group";>
                        <prefix>widget</prefix>
                        
<namespace-uri>http://markmail.org/widget</namespace-uri>
                </namespace>,
                <namespace xmlns="http://marklogic.com/xdmp/group";>
                        <prefix>identity</prefix>
                        
<namespace-uri>http://markmail.org/identity</namespace-uri>
                </namespace>,
                <namespace xmlns="http://marklogic.com/xdmp/group";>
                        <prefix>security</prefix>
                        
<namespace-uri>http://markmail.org/security</namespace-uri>
                </namespace>,
                <namespace xmlns="http://marklogic.com/xdmp/group";>
                        <prefix>profile</prefix>
                        
<namespace-uri>http://markmail.org/profile</namespace-uri>
                </namespace>
        ))

        (: Add Users and Sessions range indexes :)
        let $rangespec := admin:database-range-element-index("dateTime",
                        "http://markmail.org/user";,
                        "conf-next-msg", 
                                        "http://marklogic.com/collation/";,
                                        fn:false())
        let $config := admin:database-add-range-element-index($config, 
xdmp:database("Users"), $rangespec)

        let $rangespec := admin:database-range-element-index("dateTime",
                        "http://markmail.org/user";,
                        "conf-retire", 
                                        "http://marklogic.com/collation/";,
                                        fn:false())
        let $config := admin:database-add-range-element-index($config, 
xdmp:database("Users"), $rangespec)

        let $rangespec := admin:database-range-element-index("dateTime",
                        "http://markmail.org/user";,
                        "expires", 
                                        "http://marklogic.com/collation/";,
                                        fn:false())

        let $config := admin:database-add-range-element-index($config, 
xdmp:database("Sessions"), $rangespec)



        let $save := admin:save-configuration($config)
        return ()
else if($stage = 2)
then
        let $mmPublicUser := xdmp:eval('
                        import module namespace sec = 
"http://marklogic.com/xdmp/security"; at "/MarkLogic/security.xqy"
                        sec:uid-for-name("mm-public")
                ',(),  <options xmlns="xdmp:eval"><database>{ 
xdmp:security-database() }</database></options>)
        let $config := admin:appserver-set-default-user($config, 
admin:appserver-get-id($config, $defaultGroupId, "MarkMail-UI"), $mmPublicUser)
        let $save := admin:save-configuration($config)
        return ()
else ()

_______________________________________________
General mailing list
[email protected]
http://xqzone.com/mailman/listinfo/general

Re: [MarkLogic Dev General] user installation using a query

Reply via email to