Didi,
I've had to do something similar at Michigan Tech, but I found that
reading the LDIF is much faster. Perhaps this program will be of some
help in that regard.
It is based on the ldif2pw program which, I believe, comes with the
Net::LDAP bundle.
Regards,
Todd Piket
Sr. Programmer/Analyst
Michigan Technological University
Didi wrote:
Hi
I am trying to generate my /etc/passwd from my ldap as a backup
solution. For this I wrote a little script. Unfortunately with having
20530 users in my LDAP (expected to increase) the script takes
real 0m21.471s
to execute and uses up to 370 MB of Memory. (While using 100 % Percent
of one Core (AMD Athlon(tm) 64 X2 Dual Core Processor 4600+))
Has someone got an idea how I can make this a little more efficient. I
tried both
- foreach my $entry ($mesg->entries) {
- while (my $entry=$mesg->pop_entry ( )){
It seams like the search first fills it's buffer and then proceeds.
Thank you in advance.
Cheers for the great work
Didi
-------Code-----
#!/usr/bin/perl
# A little program that creates a /etc/passwd from ldap
use strict;
use Net::LDAP;
# Please modify appropriate
my $ldap = Net::LDAP->new("lxb5479.cern.ch") or die "Error in creating
new Connection: $@";;
$ldap->bind("cn=Manager,dc=example,dc=edu", password=>"no_you_dont");
my $mesg = $ldap->search(filter=>"(&(objectClass=posixAccount)(uid=*))",
base=>"ou=People,dc=example,dc=edu");
$mesg->code && die $mesg->error;
#########!!!DO NOT TOUCH!!!##########
#foreach my $entry ($mesg->entries) {
while (my $entry=$mesg->pop_entry ( )){
# Get rid of {crypt}
my $a = $entry->get_value("userPassword");
$a =~ s/\{crypt\}//g;
my $line = join (":", $entry->get_value("uid"),
$a ,
$entry->get_value("uidNumber") ,
$entry->get_value("gidNumber") ,
$entry->get_value("gecos") ,
$entry->get_value("homeDirectory") ,
$entry->get_value("loginShell") );
print $line . "\n";
}
$ldap->unbind;
#end
----
www.ribalba.de
Email / Jabber: [EMAIL PROTECTED]
Phone (Work) : +41 22 7679376
Skype : ribalba
#!/services/linux/support/perl/perl-5.8.5_AS3/bin/perl
#**********************************************************************
#
# File: generate_password_file.pl
#
# Author: Todd Piket
#
# Parameters: -h - help
#
# -m - min size
#
# -t - tag
#
# Description: Builds a UNIX-style passwd file without
# passwords in it as a backup for the sendmail
# servers.
#
# Last Modified: May 05, 2006
#
#**********************************************************************
#**********************************************************************
#
# This forces us to declare our variables using the 'my' keyword
# so our code will be a bit more stable. It also forces us to use
# fully qualified names for global variables and functions so we
# know EXACTLY where they are coming from.
#
#**********************************************************************
use strict;
#**********************************************************************
#
# This allows us to use actual names for "Special Variables" like
# $_. Instead of $_ we can use $ARG
#
#**********************************************************************
use English;
#**********************************************************************
#
# Various other modules to use in order to get our job done.
#
#**********************************************************************
use Getopt::Std;
use File::Copy;
use Sys::Syslog qw(:DEFAULT setlogsock);
use POSIX;
use Net::LDAP::Entry;
use Net::LDAP::LDIF;
use Net::LDAP::Constant qw(:all);
#**********************************************************************
#
# Local Variables...
#
#**********************************************************************
my ($timestamp, $check, $ldif_lockfile, $file, $min_size);
my ($sec, $min, $hour, $mday, $mon, $year, $wday, @junk) = localtime(time);
my ($diff_command, $diff_rc, $patch_command, $patch_rc, $host);
my ($passwd_lockfile);
my (@filestuff, @diff_results, @patch_results, @host_pieces);
my (%LDIF_PASSWD);
my ($LDIF_DUMP_DIR, $LDIF_DUMP_FILE);
#**********************************************************************
#
# Initialize Variables...
#
#**********************************************************************
$min_size = 0;
$mon++;
$year = $year + 1900;
$mon = "0" . $mon if ($mon < 10);
$mday = "0" . $mday if ($mday < 10);
$wday = "0" . $wday if ($wday < 10);
$hour = "0" . $hour if ($hour < 10);
$min = "0" . $min if ($min < 10);
$sec = "0" . $sec if ($sec < 10);
$timestamp = "$mon/$mday/$year $hour:$min:$sec";
$ldif_lockfile = "/export/misc/ldapmaster.mtu.edu/ldifdump/ldifdump.lock";
$passwd_lockfile = "/export/misc/ldapmaster.mtu.edu/ldifdump/passwd.lock";
$main::EXEC_DIR = "/services/linux/support/generate_password_file";
$main::LOCAL_DIR = "/tmp";
$main::PASSWD_DIR = "/etc";
$main::LDIF_DUMP_DIR = "/export/misc/ldapmaster.mtu.edu/ldifdump";
$main::LDIF_DUMP_FILE = "mtuldap_db2ldif";
$main::DIFF_CMD = "/usr/bin/diff";
$main::PATCH_CMD = "/usr/bin/patch";
$main::HOSTNAME_CMD = "/bin/hostname";
#**********************************************************************
#
# Begin Code...
#
#**********************************************************************
#
# Get passed in variables...
#
&getopts("hm:t:");
#
# If help was asked for, print and exit.
#
if (defined($Getopt::Std::opt_h))
{
&usage();
exit(0);
}
#
# Set the minimum size if it was given.
#
$min_size = $Getopt::Std::opt_m if (defined($Getopt::Std::opt_m));
#
# Make sure we are running this file from within the
# $main::EXEC_DIR
#
if (! chdir ("$main::EXEC_DIR"))
{
&logit("Can't change to directory $main::EXEC_DIR");
exit(1);
}
#
# Make sure the LDIF dump process isn't currently running
# before we attempt to use the file. If it is we have to abort.
#
if (-e $ldif_lockfile)
{
&logit("Could not read LDIF due to existing lock file");
exit(1);
}
#
# Get this host's name to do host specific stuff.
#
$host = `$main::HOSTNAME_CMD`;
chomp ($host);
(@host_pieces) = split (/\./, $host);
$host = $host_pieces[0];
#
# Am I already running?
#
if (-e "$passwd_lockfile.$host")
{
&logit ("Already running on $host...aborting.");
exit(1);
}
#
# Create this host's gen_passwd lockfile so the LDIF
# Extraction program can see it.
#
if (! open (LOCKFILE, ">$passwd_lockfile.$host"))
{
&logit ("Can't open $passwd_lockfile.$host for writing");
exit(1);
}
close (LOCKFILE);
#
# Make sure the file has data in it.
#
$check = 1;
while ($check)
{
$file = "$main::LDIF_DUMP_DIR/$main::LDIF_DUMP_FILE.$wday.ldif";
@filestuff = stat ($file);
if ($filestuff[7] <= 0)
{
$wday = $wday - 1;
$wday = "0" . $wday if ($wday < 10);
}
else
{
$check = 0;
}
}
#
# Build the password file from the ldif.
#
%LDIF_PASSWD = &ldif2pw($file, $Getopt::Std::opt_t);
#
# If we didn't generate enough entries we should stop.
#
if (scalar(keys(%LDIF_PASSWD)) < $min_size)
{
&logit ("Not enough entries...aborting.");
exit(1);
}
#
# Unlock the ldifdump file
#
unlink ("$passwd_lockfile.$host");
#
# Truncate the old passwd file.
#
if (! open ("PASSWD", ">$main::LOCAL_DIR/passwd"))
{
&logit ("Can't open $main::LOCAL_DIR/passwd file for writing");
exit(1);
}
#
# Create an hourly backup.
#
if (! open ("HPASSWD", ">$main::LOCAL_DIR/passwd.$hour"))
{
&logit ("Can't open $main::LOCAL_DIR/passwd.$hour file for writing");
exit(1);
}
#
# Open the file we just made and the static file so we can stick
# them together to make one big file.
#
if (! open ("STATIC", "<$main::EXEC_DIR/data/passwd.static"))
{
&logit ("Can't open $main::EXEC_DIR/data/passwd.static for reading!");
exit (1);
}
while (<STATIC>)
{
print PASSWD $_;
print HPASSWD $_;
}
close (STATIC);
foreach (sort (keys (%LDIF_PASSWD)))
{
print PASSWD $LDIF_PASSWD{$_};
print HPASSWD $LDIF_PASSWD{$_};
}
#
# All done generating.
#
close (PASSWD);
close (HPASSWD);
#
# Make a backup of the original passwd file.
#
if (! copy("$main::PASSWD_DIR/passwd", "$main::PASSWD_DIR/passwd.$hour"))
{
&logit ("Could not backup $main::PASSWD_DIR/passwd");
exit(1);
}
#
# Now we have to apply our changes so perform a diff/patch operation.
# This should be less harmful to the passwd file in the long run.
#
# Handle the diff operation.
#
$diff_command = "$main::DIFF_CMD $main::PASSWD_DIR/passwd
$main::LOCAL_DIR/passwd 2>&1";
@diff_results = `$diff_command`;
$diff_rc = $? >> 8;
#
# Check for errors
#
if ($diff_rc == 0)
{
&logit ("No differences found.");
exit(0);
}
elsif ($diff_rc == 1)
{
&logit ("Writing diff output to $main::LOCAL_DIR/passwd.$hour.diff");
if (! open(DIFF, ">$main::LOCAL_DIR/passwd.$hour.diff"))
{
&logit ("Could not write diff file
$main::LOCAL_DIR/passwd.$hour.diff");
exit(1);
}
foreach (@diff_results)
{
print DIFF "$_";
}
close(DIFF);
}
else
{
&logit ("Diff Error:" . join("\n", @diff_results));
}
#
# Handle the patch operation.
#
$patch_command = "$main::PATCH_CMD $main::PASSWD_DIR/passwd
$main::LOCAL_DIR/passwd.$hour.diff 2>&1";
@patch_results = `$patch_command`;
$patch_rc = $? >> 8;
#
# Check for errors
#
if ($patch_rc == 0)
{
&logit ("Patch successful.");
exit(0);
}
elsif ($patch_rc == 1)
{
&logit ("Problem patching /etc/passwd.");
}
else
{
&logit ("Patch Error:" . join("\n", @patch_results));
}
#**********************************************************************
#
# Begin Subroutines...
#
#**********************************************************************
#**********************************************************************
#
# Function: ldif2pw
#
# Parameters: $file - The LDIF file to parse.
#
# $tag - A tag that tells this function to execute
# special code that filters the LDIF in order to
# generate a password file for a specific use. For
# example, the "email" tag tells this program to
# generate a password file for only those users that
# have the email privilege.
#
# Returns: None.
#
# Description: Parses the given LDIF file and generates valid
# /etc/passwd file data.
#
#**********************************************************************
sub ldif2pw
{
#
# Get passed in variables...
#
my ($file, $tag) = @_;
#
# Local Variables...
#
my ($i, $line, $lhs, $rhs, $entry, $ldif, $value);
my (@errors, @values);
my (%USERS);
#
# Make sure we can open the given filename.
#
if (! open(INPUT, "<$file"))
{
&logit ("Could not open $file for reading");
exit(1);
}
close (INPUT);
#
# Create an instance of an LDIF object.
#
$ldif = Net::LDAP::LDIF->new($file, "r", onerror => 'undef');
#
# Read and parse the LDIF to build the passwd file.
#
while(! $ldif->eof())
{
#
# Get the entry we are to parse.
#
$entry = $ldif->read_entry();
#
# If nothing was read in successfully, skip it.
#
if (! $entry)
{
next;
}
#
# Was there an error?
#
if ($ldif->error())
{
&logit ("LDIF Error: ", $ldif->error());
next;
}
if (($entry->exists("uid")) and ($entry->exists("gidnumber")))
{
$line = $entry->get_value("uid") . ":";
$line .= "x:";
$line .= $entry->get_value("uidnumber");
$line .= ":";
if ($entry->get_value("gidnumber") eq "10")
{
$line .= 13000;
}
else
{
$line .= $entry->get_value("gidnumber");
}
$line .= ":";
$line .= $entry->get_value("cn");
$line .= ":";
$line .= $entry->get_value("homedirectory");
$line .= ":/bin/false\n";
}
#
# Should we store this user for later or not?
#
if ($tag eq "email")
{
foreach $value
($entry->get_value("edupersonentitlement"))
{
if ($value eq
"urn:mace:mtu.edu:entitlement:campus:email")
{
$USERS{$entry->get_value("uid")} =
$line;
}
}
}
else
{
$USERS{$entry->get_value("uid")} = $line;
}
$line = "";
next;
}
#
# Return the generated hash.
#
return (%USERS);
}
#**********************************************************************
#
# Function: logit
#
# Parameters: $message - A text message to write to the log.
#
# Returns: None.
#
# Description: Prints the given message to the syslog daemon
# with the specified priority and facility.
#
#**********************************************************************
sub logit
{
#
# Get passed in parameters...
#
my ($message) = @_;
#
# Write to the log.
#
openlog("generate_passwd_file", "pid", "local1");
syslog("LOG_DEBUG|local1", "%s", "$message");
closelog();
}
#**********************************************************************
#
# Function: usage
#
# Parameters: None.
#
# Returns: None.
#
# Description: Prints a help message that describes how to use
# this program.
#
#**********************************************************************
sub usage
{
print "\nUsage: generate_password_file.pl [-h] [-t tag]\n\n";
print "\t-h Prints this help message\n\n";
print "\t-t A tag that tells this program to execute special\n";
print "\t code that filters the LDIF in order to generate a\n";
print "\t password file for a specific use. For example,\n";
print "\t the \"email\" tag tells this program to generate\n";
print "\t a password file for only those users that have\n";
print "\t the email privilege.\n";
print "\n";
print "\t-m The minimum number of users that should be in the\n";
print "\t password file upon completion.\n";
print "\n";
}
