Package: openldap2.2 Severity: normal Tags: patch Hi,
quanah Gibson-mount has updated his quicktool patch to fix a few issues: - in libraries/liblutil/ldif.c: correct argumements of ber_pvt_log_printf() calls. This impacts the cases when slapadd -q fails and helps admins to find out where the problem is. - in servers/slapd/slapcommon.c: do not change allowed options for slapindexr; only do it for slapadd - servers/slapd/entry.c: slightly different solution for reordering attributes to add (use less variables. The attached patch is the updated version that should apply to 2.2.26 cleanly. CU Peter PS: Quanah also has an updated version for 2.2.27 and a few other patches on his info page at http://www.stanford.edu/services/directory/openldap/configuration/patches/openldap/ -- System Information: Debian Release: testing/unstable APT prefers testing APT policy: (990, 'testing'), (500, 'unstable') Architecture: i386 (i686) Shell: /bin/sh linked to /bin/bash Kernel: Linux 2.6.11-1-k7 Locale: [EMAIL PROTECTED], [EMAIL PROTECTED] (charmap=ISO-8859-15)
--- servers/slapd/slap.h +++ servers/slapd/slap.h 2005-05-14 14:58:19.000000000 +0200 @@ -1281,6 +1281,7 @@ #define SLAP_TRUNCATE_MODE 0x0100 #define SLAP_TOOL_READMAIN 0x0200 #define SLAP_TOOL_READONLY 0x0400 +#define SLAP_TOOL_QUICK 0x0800 struct slap_replica_info { char *ri_host; /* supersedes be_replica */ --- servers/slapd/slapcommon.c +++ servers/slapd/slapcommon.c 2005-05-14 14:58:19.000000000 +0200 @@ -55,7 +55,7 @@ switch( tool ) { case SLAPADD: options = "\n\t[-n databasenumber | -b suffix]\n" - "\t[-l ldiffile] [-u] [-p [-w] | -r [-i syncreplidlist] [-w]]\n"; + "\t[-l ldiffile] [-q] [-u] [-p [-w] | -r [-i syncreplidlist] [-w]]\n"; break; case SLAPCAT: @@ -116,7 +116,7 @@ switch( tool ) { case SLAPADD: - options = "b:cd:f:i:l:n:prtuvWw"; + options = "b:cd:f:i:l:n:qprtuvWw"; break; case SLAPCAT: @@ -201,6 +201,10 @@ dbnum = atoi( optarg ) - 1; break; + case 'q': /* turn on quick */ + mode |= SLAP_TOOL_QUICK; + break; + case 'p': /* replica promotion */ replica_promotion = 1; break; --- servers/slapd/back-bdb/dbcache.c +++ servers/slapd/back-bdb/dbcache.c 2005-05-14 14:58:19.000000000 +0200 @@ -63,7 +63,7 @@ const char *name, DB **dbout ) { - int i; + int i, flags; int rc; struct bdb_info *bdb = (struct bdb_info *) be->be_private; struct bdb_db_info *db; @@ -126,9 +126,14 @@ #ifdef HAVE_EBCDIC __atoe( file ); #endif + flags = DB_CREATE | DB_THREAD; +#ifdef DB_AUTO_COMMIT + if ( !( slapMode & SLAP_TOOL_QUICK )) + flags |= DB_AUTO_COMMIT; +#endif rc = DB_OPEN( db->bdi_db, file, NULL /* name */, - BDB_INDEXTYPE, bdb->bi_db_opflags | DB_CREATE | DB_THREAD, + BDB_INDEXTYPE, bdb->bi_db_opflags | flags, bdb->bi_dbenv_mode ); ch_free( file ); --- servers/slapd/proto-slap.h +++ servers/slapd/proto-slap.h 2005-05-14 14:58:19.000000000 +0200 @@ -124,6 +124,7 @@ /* * at.c */ +LDAP_SLAPD_V(int) at_oc_cache; LDAP_SLAPD_F (void) at_config LDAP_P(( const char *fname, int lineno, int argc, char **argv )); @@ -502,6 +503,7 @@ LDAP_SLAPD_F (int) entry_destroy LDAP_P((void)); LDAP_SLAPD_F (Entry *) str2entry LDAP_P(( char *s )); +LDAP_SLAPD_F (Entry *) str2entry2 LDAP_P(( char *s, int checkvals )); LDAP_SLAPD_F (char *) entry2str LDAP_P(( Entry *e, int *len )); LDAP_SLAPD_F (void) entry_flatsize LDAP_P(( --- servers/slapd/slapadd.c +++ servers/slapd/slapadd.c 2005-05-14 14:58:19.000000000 +0200 @@ -78,7 +78,7 @@ struct berval ctxcsn_ndn = BER_BVNULL; int ret; struct berval bvtext; - int i; + int i, checkvals; struct berval mc; struct sync_cookie sc; #ifdef NEW_LOGGING @@ -99,6 +99,8 @@ } } + checkvals = (slapMode & SLAP_TOOL_QUICK) ? 0 : 1; + lmax = 0; lineno = 0; @@ -109,7 +111,7 @@ } while( ldif_read_record( ldiffp, &lineno, &buf, &lmax ) ) { - Entry *e = str2entry( buf ); + Entry *e = str2entry2( buf, checkvals ); /* * Initialize text buffer --- include/ldif.h +++ include/ldif.h 2005-05-14 14:58:19.000000000 +0200 @@ -60,6 +60,13 @@ ber_len_t *vlen )); LDAP_LDIF_F( int ) +ldif_parse_line2 LDAP_P(( + char *line, + struct berval *type, + struct berval *value, + int *freeval )); + +LDAP_LDIF_F( int ) ldif_fetch_url LDAP_P(( LDAP_CONST char *line, char **value, @@ -69,6 +76,9 @@ ldif_getline LDAP_P(( char **next )); LDAP_LDIF_F( int ) +ldif_countlines LDAP_P(( LDAP_CONST char *line )); + +LDAP_LDIF_F( int ) ldif_read_record LDAP_P(( FILE *fp, int *lineno, --- servers/slapd/back-ldbm/id2entry.c +++ servers/slapd/back-ldbm/id2entry.c 2005-05-14 14:58:19.000000000 +0200 @@ -241,7 +241,7 @@ return( NULL ); } - e = str2entry( data.dptr ); + e = str2entry2( data.dptr, 0 ); ldbm_datum_free( db->dbc_db, data ); ldbm_cache_close( be, db ); --- servers/slapd/back-ldbm/tools.c +++ servers/slapd/back-ldbm/tools.c 2005-05-14 14:58:19.000000000 +0200 @@ -163,7 +163,7 @@ return NULL; } - e = str2entry( data.dptr ); + e = str2entry2( data.dptr,0 ); ldbm_datum_free( id2entry->dbc_db, data ); if( e != NULL ) { --- servers/slapd/at.c +++ servers/slapd/at.c 2005-05-14 14:58:19.000000000 +0200 @@ -58,9 +58,12 @@ }; static Avlnode *attr_index = NULL; +static Avlnode *attr_cache = NULL; static LDAP_SLIST_HEAD(ATList, slap_attribute_type) attr_list = LDAP_SLIST_HEAD_INITIALIZER(&attr_list); +int at_oc_cache; + static int attr_index_cmp( const void *v_air1, @@ -110,8 +113,18 @@ { struct aindexrec *air; + if ( attr_cache ) { + air = avl_find( attr_cache, name, attr_index_name_cmp ); + if ( air ) return air->air_at; + } + air = avl_find( attr_index, name, attr_index_name_cmp ); + if ( air && ( slapMode & SLAP_TOOL_MODE ) && at_oc_cache ) { + avl_insert( &attr_cache, (caddr_t) air, + attr_index_cmp, avl_dup_error ); + } + return air != NULL ? air->air_at : NULL; } --- servers/slapd/attr.c +++ servers/slapd/attr.c 2005-05-14 14:58:19.000000000 +0200 @@ -156,7 +156,7 @@ Attribute **a; for ( a = &e->e_attrs; *a != NULL; a = &(*a)->a_next ) { - if ( ad_cmp( (*a)->a_desc, desc ) == 0 ) { + if ( (*a)->a_desc == desc ) { break; } } @@ -237,7 +237,7 @@ Attribute **a; for ( a = &e->e_attrs; *a != NULL; a = &(*a)->a_next ) { - if ( ad_cmp( (*a)->a_desc, desc ) == 0 ) { + if ( (*a)->a_desc == desc ) { break; } } @@ -326,7 +326,7 @@ AttributeDescription *desc ) { for ( ; a != NULL; a = a->a_next ) { - if ( ad_cmp( a->a_desc, desc ) == 0 ) { + if ( a->a_desc == desc ) { return( a ); } } @@ -349,7 +349,7 @@ Attribute **a; for ( a = attrs; *a != NULL; a = &(*a)->a_next ) { - if ( ad_cmp( (*a)->a_desc, desc ) == 0 ) { + if ( (*a)->a_desc == desc ) { Attribute *save = *a; *a = (*a)->a_next; attr_free( save ); --- servers/slapd/oc.c +++ servers/slapd/oc.c 2005-05-14 14:58:19.000000000 +0200 @@ -133,6 +133,8 @@ }; static Avlnode *oc_index = NULL; +static Avlnode *oc_cache = NULL; + static LDAP_SLIST_HEAD(OCList, slap_object_class) oc_list = LDAP_SLIST_HEAD_INITIALIZER(&oc_list); @@ -175,9 +177,18 @@ { struct oindexrec *oir; + if ( oc_cache ) { + oir = avl_find( oc_cache, ocname, oc_index_name_cmp ); + if ( oir ) return oir->oir_oc; + } + oir = avl_find( oc_index, ocname, oc_index_name_cmp ); if ( oir != NULL ) { + if ( at_oc_cache ) { + avl_insert( &oc_cache, (caddr_t) oir, + oc_index_cmp, avl_dup_error ); + } return( oir->oir_oc ); } --- servers/slapd/sl_malloc.c +++ servers/slapd/sl_malloc.c 2005-05-14 14:58:19.000000000 +0200 @@ -229,6 +229,8 @@ struct slab_heap *sh = NULL; void *ctx; + if ( slapMode & SLAP_TOOL_MODE ) return NULL; + #ifdef NO_THREADS sh = slheap; #else --- libraries/liblutil/ldif.c +++ libraries/liblutil/ldif.c 2005-05-14 15:04:22.000000000 +0200 @@ -81,7 +81,14 @@ * ldif_parse_line - takes a line of the form "type:[:] value" and splits it * into components "type" and "value". if a double colon separates type from * value, then value is encoded in base 64, and parse_line un-decodes it - * (in place) before returning. + * (in place) before returning. The type and value are stored in malloc'd + * memory which must be freed by the caller. + * + * ldif_parse_line2 - operates in-place on input buffer, returning type + * in-place. Will return value in-place if possible, (must malloc for + * fetched URLs). If freeval is NULL, all return data will be malloc'd + * and the input line will be unmodified. Otherwise freeval is set to + * True if the value was malloc'd. */ int @@ -92,46 +99,65 @@ ber_len_t *vlenp ) { + struct berval type, value; + int rc = ldif_parse_line2( (char *)line, &type, &value, NULL ); + + *typep = type.bv_val; + *valuep = value.bv_val; + *vlenp = value.bv_len; + return rc; +} + +int +ldif_parse_line2( + char *line, + struct berval *type, + struct berval *value, + int *freeval +) +{ char *s, *p, *d; char nib; int b64, url; - char *freeme, *type, *value; - ber_len_t vlen; - *typep = NULL; - *valuep = NULL; - *vlenp = 0; + BER_BVZERO( type ); + BER_BVZERO( value ); /* skip any leading space */ while ( isspace( (unsigned char) *line ) ) { line++; } - freeme = ber_strdup( line ); + if ( freeval ) { + *freeval = 0; + } else { + line = ber_strdup( line ); - if( freeme == NULL ) { - ber_pvt_log_printf( LDAP_DEBUG_ANY, ldif_debug, - _("ldif_parse_line: line malloc failed\n")); - return( -1 ); + if( line == NULL ) { + ber_pvt_log_printf( LDAP_DEBUG_ANY, ldif_debug, + _("ldif_parse_line: line malloc failed\n")); + return( -1 ); + } } - type = freeme; + type->bv_val = line; - s = strchr( type, ':' ); + s = strchr( type->bv_val, ':' ); if ( s == NULL ) { ber_pvt_log_printf( LDAP_DEBUG_PARSE, ldif_debug, _("ldif_parse_line: missing ':' after %s\n"), - type ); - ber_memfree( freeme ); + type->bv_val ); + if ( !freeval ) ber_memfree( line ); return( -1 ); } /* trim any space between type and : */ - for ( p = &s[-1]; p > type && isspace( * (unsigned char *) p ); p-- ) { + for ( p = &s[-1]; p > type->bv_val && isspace( * (unsigned char *) p ); p-- ) { *p = '\0'; } *s++ = '\0'; + type->bv_len = s - type->bv_val - 1; url = 0; b64 = 0; @@ -164,14 +190,15 @@ if ( *s == '\0' ) { /* no value is present, error out */ ber_pvt_log_printf( LDAP_DEBUG_PARSE, ldif_debug, - _("ldif_parse_line: %s missing base64 value\n"), type ); - ber_memfree( freeme ); + _("ldif_parse_line: %s missing base64 value\n"), + type->bv_val ); + if ( !freeval ) ber_memfree( line ); return( -1 ); } - byte = value = s; + byte = value->bv_val = s; - for ( p = s, vlen = 0; p < d; p += 4, vlen += 3 ) { + for ( p = s, value->bv_len = 0; p < d; p += 4, value->bv_len += 3 ) { int i; for ( i = 0; i < 4; i++ ) { if ( p[i] != '=' && (p[i] & 0x80 || @@ -179,8 +206,8 @@ ber_pvt_log_printf( LDAP_DEBUG_ANY, ldif_debug, _("ldif_parse_line: %s: invalid base64 encoding" " char (%c) 0x%x\n"), - type, p[i], p[i] ); - ber_memfree( freeme ); + type->bv_val, p[i], p[i] ); + if ( !freeval ) ber_memfree( line ); return( -1 ); } } @@ -194,7 +221,7 @@ byte[1] = (nib & RIGHT4) << 4; /* third digit */ if ( p[2] == '=' ) { - vlen += 1; + value->bv_len += 1; break; } nib = b642nib[ p[2] & 0x7f ]; @@ -202,7 +229,7 @@ byte[2] = (nib & RIGHT2) << 6; /* fourth digit */ if ( p[3] == '=' ) { - vlen += 2; + value->bv_len += 2; break; } nib = b642nib[ p[3] & 0x7f ]; @@ -210,59 +237,59 @@ byte += 3; } - s[ vlen ] = '\0'; + s[ value->bv_len ] = '\0'; } else if ( url ) { if ( *s == '\0' ) { /* no value is present, error out */ ber_pvt_log_printf( LDAP_DEBUG_PARSE, ldif_debug, - _("ldif_parse_line: %s missing URL value\n"), type ); - ber_memfree( freeme ); + _("ldif_parse_line: %s missing URL value\n"), + type->bv_val ); + if ( !freeval ) ber_memfree( line ); return( -1 ); } - if( ldif_fetch_url( s, &value, &vlen ) ) { + if( ldif_fetch_url( s, &value->bv_val, &value->bv_len ) ) { ber_pvt_log_printf( LDAP_DEBUG_ANY, ldif_debug, _("ldif_parse_line: %s: URL \"%s\" fetch failed\n"), - type, s ); - ber_memfree( freeme ); + type->bv_val, s ); + if ( !freeval ) ber_memfree( line ); return( -1 ); } + if ( freeval ) *freeval = 1; } else { - value = s; - vlen = (int) (d - s); + value->bv_val = s; + value->bv_len = (int) (d - s); } - type = ber_strdup( type ); + if ( !freeval ) { + struct berval bv = *type; + + ber_dupbv( type, &bv ); - if( type == NULL ) { + if( BER_BVISNULL( type )) { ber_pvt_log_printf( LDAP_DEBUG_ANY, ldif_debug, _("ldif_parse_line: type malloc failed\n")); - if( url ) ber_memfree( value ); - ber_memfree( freeme ); + if( url ) ber_memfree( value->bv_val ); + ber_memfree( line ); return( -1 ); } if( !url ) { - p = ber_memalloc( vlen + 1 ); - if( p == NULL ) { + bv = *value; + ber_dupbv( value, &bv ); + if( BER_BVISNULL( value )) { ber_pvt_log_printf( LDAP_DEBUG_ANY, ldif_debug, _("ldif_parse_line: value malloc failed\n")); - ber_memfree( type ); - ber_memfree( freeme ); + ber_memfree( type->bv_val ); + ber_memfree( line ); return( -1 ); } - AC_MEMCPY( p, value, vlen ); - p[vlen] = '\0'; - value = p; } - ber_memfree( freeme ); - - *typep = type; - *valuep = value; - *vlenp = vlen; + ber_memfree( line ); + } return( 0 ); } @@ -283,6 +310,21 @@ * which it updates and must be supplied on subsequent calls. */ +int +ldif_countlines( LDAP_CONST char *buf ) +{ + char *nl; + int ret = 0; + + if ( !buf ) return ret; + + for ( nl = strchr(buf, '\n'); nl; nl = strchr(nl, '\n') ) { + nl++; + if ( *nl != ' ' ) ret++; + } + return ret; +} + char * ldif_getline( char **next ) { --- servers/slapd/back-bdb/back-bdb.h +++ servers/slapd/back-bdb/back-bdb.h 2005-05-14 14:58:19.000000000 +0200 @@ -236,7 +236,7 @@ #if DB_VERSION_FULL >= 0x04010011 #undef DB_OPEN #define DB_OPEN(db, file, name, type, flags, mode) \ - (db)->open(db, NULL, file, name, type, (flags)|DB_AUTO_COMMIT, mode) + (db)->open(db, NULL, file, name, type, flags, mode) #endif #endif --- servers/slapd/back-bdb/dn2id.c +++ servers/slapd/back-bdb/dn2id.c 2005-05-14 14:58:19.000000000 +0200 @@ -710,7 +710,7 @@ data.data = d; /* Delete our ID from the parent's list */ - rc = cursor->c_get( cursor, &key, &data, DB_GET_BOTH | DB_RMW ); + rc = cursor->c_get( cursor, &key, &data, DB_GET_BOTH_RANGE ); if ( rc == 0 ) rc = cursor->c_del( cursor, 0 ); @@ -720,7 +720,7 @@ */ if ( rc == 0 ) { key.data = &e->e_id; - rc = cursor->c_get( cursor, &key, &data, DB_SET | DB_RMW ); + rc = cursor->c_get( cursor, &key, &data, DB_SET ); if ( rc == 0 ) rc = cursor->c_del( cursor, 0 ); } --- servers/slapd/back-bdb/init.c +++ servers/slapd/back-bdb/init.c 2005-05-14 14:58:19.000000000 +0200 @@ -155,8 +155,9 @@ return rc; } - flags = DB_INIT_MPOOL | DB_THREAD | DB_CREATE - | DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_TXN; + flags = DB_INIT_MPOOL | DB_THREAD | DB_CREATE; + if ( !( slapMode & SLAP_TOOL_QUICK )) + flags |= DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_TXN; #if 0 /* Never do automatic recovery, must perform it manually. @@ -280,6 +281,11 @@ flags = DB_THREAD | bdb->bi_db_opflags; +#ifdef DB_AUTO_COMMIT + if ( !( slapMode & SLAP_TOOL_QUICK )) + flags |= DB_AUTO_COMMIT; +#endif + bdb->bi_databases = (struct bdb_db_info **) ch_malloc( BDB_INDICES * sizeof(struct bdb_db_info *) ); @@ -375,7 +381,9 @@ return rc; } - XLOCK_ID(bdb->bi_dbenv, &bdb->bi_cache.c_locker); + if ( !( slapMode & SLAP_TOOL_QUICK )) { + XLOCK_ID(bdb->bi_dbenv, &bdb->bi_cache.c_locker); + } /* <insert> open (and create) index databases */ return 0; @@ -416,8 +424,9 @@ } ldap_pvt_thread_rdwr_wunlock ( &bdb->bi_idl_tree_rwlock ); } - - XLOCK_ID_FREE(bdb->bi_dbenv, bdb->bi_cache.c_locker); + if ( !( slapMode & SLAP_TOOL_QUICK )) { + XLOCK_ID_FREE(bdb->bi_dbenv, bdb->bi_cache.c_locker); + } return 0; } @@ -434,11 +443,13 @@ /* close db environment */ if( bdb->bi_dbenv ) { /* force a checkpoint */ - rc = TXN_CHECKPOINT( bdb->bi_dbenv, 0, 0, DB_FORCE ); - if( rc != 0 ) { - Debug( LDAP_DEBUG_ANY, - "bdb_db_destroy: txn_checkpoint failed: %s (%d)\n", - db_strerror(rc), rc, 0 ); + if ( !( slapMode & SLAP_TOOL_QUICK )) { + rc = TXN_CHECKPOINT( bdb->bi_dbenv, 0, 0, DB_FORCE ); + if( rc != 0 ) { + Debug( LDAP_DEBUG_ANY, + "bdb_db_destroy: txn_checkpoint failed: %s (%d)\n", + db_strerror(rc), rc, 0 ); + } } rc = bdb->bi_dbenv->close( bdb->bi_dbenv, 0 ); --- servers/slapd/back-bdb/tools.c +++ servers/slapd/back-bdb/tools.c 2005-05-14 14:58:19.000000000 +0200 @@ -302,16 +302,18 @@ Debug( LDAP_DEBUG_TRACE, "=> " LDAP_XSTRING(bdb_tool_entry_put) "( %ld, \"%s\" )\n", (long) e->e_id, e->e_dn, 0 ); - rc = TXN_BEGIN( bdb->bi_dbenv, NULL, &tid, - bdb->bi_db_opflags ); - if( rc != 0 ) { - snprintf( text->bv_val, text->bv_len, - "txn_begin failed: %s (%d)", - db_strerror(rc), rc ); - Debug( LDAP_DEBUG_ANY, - "=> " LDAP_XSTRING(bdb_tool_entry_put) ": %s\n", - text->bv_val, 0, 0 ); - return NOID; + if (! (slapMode & SLAP_TOOL_QUICK)) { + rc = TXN_BEGIN( bdb->bi_dbenv, NULL, &tid, + bdb->bi_db_opflags ); + if( rc != 0 ) { + snprintf( text->bv_val, text->bv_len, + "txn_begin failed: %s (%d)", + db_strerror(rc), rc ); + Debug( LDAP_DEBUG_ANY, + "=> " LDAP_XSTRING(bdb_tool_entry_put) ": %s\n", + text->bv_val, 0, 0 ); + return NOID; + } } op.o_bd = be; @@ -349,25 +351,29 @@ done: if( rc == 0 ) { - rc = TXN_COMMIT( tid, 0 ); - if( rc != 0 ) { + if ( !( slapMode & SLAP_TOOL_QUICK )) { + rc = TXN_COMMIT( tid, 0 ); + if( rc != 0 ) { + snprintf( text->bv_val, text->bv_len, + "txn_commit failed: %s (%d)", + db_strerror(rc), rc ); + Debug( LDAP_DEBUG_ANY, + "=> " LDAP_XSTRING(bdb_tool_entry_put) ": %s\n", + text->bv_val, 0, 0 ); + e->e_id = NOID; + } + } + + } else { + if ( !( slapMode & SLAP_TOOL_QUICK )) { + TXN_ABORT( tid ); snprintf( text->bv_val, text->bv_len, - "txn_commit failed: %s (%d)", - db_strerror(rc), rc ); + "txn_aborted! %s (%d)", + db_strerror(rc), rc ); Debug( LDAP_DEBUG_ANY, "=> " LDAP_XSTRING(bdb_tool_entry_put) ": %s\n", text->bv_val, 0, 0 ); - e->e_id = NOID; } - - } else { - TXN_ABORT( tid ); - snprintf( text->bv_val, text->bv_len, - "txn_aborted! %s (%d)", - db_strerror(rc), rc ); - Debug( LDAP_DEBUG_ANY, - "=> " LDAP_XSTRING(bdb_tool_entry_put) ": %s\n", - text->bv_val, 0, 0 ); e->e_id = NOID; } @@ -398,13 +404,15 @@ return -1; } + if (! (slapMode & SLAP_TOOL_QUICK)) { rc = TXN_BEGIN( bi->bi_dbenv, NULL, &tid, bi->bi_db_opflags ); - if( rc != 0 ) { - Debug( LDAP_DEBUG_ANY, - "=> " LDAP_XSTRING(bdb_tool_entry_reindex) ": " - "txn_begin failed: %s (%d)\n", - db_strerror(rc), rc, 0 ); - goto done; + if( rc != 0 ) { + Debug( LDAP_DEBUG_ANY, + "=> " LDAP_XSTRING(bdb_tool_entry_reindex) ": " + "txn_begin failed: %s (%d)\n", + db_strerror(rc), rc, 0 ); + goto done; + } } /* @@ -422,37 +430,29 @@ op.o_tmpmemctx = NULL; op.o_tmpmfuncs = &ch_mfuncs; -#ifndef BDB_HIER - /* add dn2id indices */ - rc = bdb_dn2id_add( &op, tid, NULL, e ); - if( rc != 0 && rc != DB_KEYEXIST ) { - Debug( LDAP_DEBUG_ANY, - "=> " LDAP_XSTRING(bdb_tool_entry_reindex) - ": dn2id_add failed: %s (%d)\n", - db_strerror(rc), rc, 0 ); - goto done; - } -#endif - rc = bdb_index_entry_add( &op, tid, e ); done: if( rc == 0 ) { - rc = TXN_COMMIT( tid, 0 ); - if( rc != 0 ) { + if (! (slapMode & SLAP_TOOL_QUICK)) { + rc = TXN_COMMIT( tid, 0 ); + if( rc != 0 ) { + Debug( LDAP_DEBUG_ANY, + "=> " LDAP_XSTRING(bdb_tool_entry_reindex) + ": txn_commit failed: %s (%d)\n", + db_strerror(rc), rc, 0 ); + e->e_id = NOID; + } + } + + } else { + if (! (slapMode & SLAP_TOOL_QUICK)) { + TXN_ABORT( tid ); Debug( LDAP_DEBUG_ANY, "=> " LDAP_XSTRING(bdb_tool_entry_reindex) - ": txn_commit failed: %s (%d)\n", + ": txn_aborted! %s (%d)\n", db_strerror(rc), rc, 0 ); - e->e_id = NOID; } - - } else { - TXN_ABORT( tid ); - Debug( LDAP_DEBUG_ANY, - "=> " LDAP_XSTRING(bdb_tool_entry_reindex) - ": txn_aborted! %s (%d)\n", - db_strerror(rc), rc, 0 ); e->e_id = NOID; } bdb_entry_release( &op, e, 0 ); @@ -484,16 +484,18 @@ "=> " LDAP_XSTRING(bdb_tool_entry_modify) "( %ld, \"%s\" )\n", (long) e->e_id, e->e_dn, 0 ); - rc = TXN_BEGIN( bdb->bi_dbenv, NULL, &tid, - bdb->bi_db_opflags ); - if( rc != 0 ) { - snprintf( text->bv_val, text->bv_len, - "txn_begin failed: %s (%d)", - db_strerror(rc), rc ); - Debug( LDAP_DEBUG_ANY, - "=> " LDAP_XSTRING(bdb_tool_entry_modify) ": %s\n", - text->bv_val, 0, 0 ); - return NOID; + if (! (slapMode & SLAP_TOOL_QUICK)) { + rc = TXN_BEGIN( bdb->bi_dbenv, NULL, &tid, + bdb->bi_db_opflags ); + if( rc != 0 ) { + snprintf( text->bv_val, text->bv_len, + "txn_begin failed: %s (%d)", + db_strerror(rc), rc ); + Debug( LDAP_DEBUG_ANY, + "=> " LDAP_XSTRING(bdb_tool_entry_modify) ": %s\n", + text->bv_val, 0, 0 ); + return NOID; + } } op.o_bd = be; @@ -536,25 +538,28 @@ done: if( rc == 0 ) { - rc = TXN_COMMIT( tid, 0 ); - if( rc != 0 ) { + if (! (slapMode & SLAP_TOOL_QUICK)) { + rc = TXN_COMMIT( tid, 0 ); + if( rc != 0 ) { + snprintf( text->bv_val, text->bv_len, + "txn_commit failed: %s (%d)", + db_strerror(rc), rc ); + Debug( LDAP_DEBUG_ANY, + "=> " LDAP_XSTRING(bdb_tool_entry_modify) ": " + "%s\n", text->bv_val, 0, 0 ); + e->e_id = NOID; + } + } + } else { + if (! (slapMode & SLAP_TOOL_QUICK)) { + TXN_ABORT( tid ); snprintf( text->bv_val, text->bv_len, - "txn_commit failed: %s (%d)", - db_strerror(rc), rc ); + "txn_aborted! %s (%d)", + db_strerror(rc), rc ); Debug( LDAP_DEBUG_ANY, - "=> " LDAP_XSTRING(bdb_tool_entry_modify) ": " - "%s\n", text->bv_val, 0, 0 ); - e->e_id = NOID; + "=> " LDAP_XSTRING(bdb_tool_entry_modify) ": %s\n", + text->bv_val, 0, 0 ); } - - } else { - TXN_ABORT( tid ); - snprintf( text->bv_val, text->bv_len, - "txn_aborted! %s (%d)", - db_strerror(rc), rc ); - Debug( LDAP_DEBUG_ANY, - "=> " LDAP_XSTRING(bdb_tool_entry_modify) ": %s\n", - text->bv_val, 0, 0 ); e->e_id = NOID; } --- servers/slapd/back-bdb/idl.c +++ servers/slapd/back-bdb/idl.c 2005-05-14 14:58:19.000000000 +0200 @@ -472,8 +472,15 @@ #if (DB_VERSION_MAJOR == 4) && (DB_VERSION_MINOR == 0) # define BDB_ENOUGH 5 #else + /* We sometimes test with tiny IDLs, and BDB always wants buffers + * that are at least one page in size. + */ +# if BDB_IDL_DB_SIZE < 4096 +# define BDB_ENOUGH 2048 +# else # define BDB_ENOUGH 1 #endif +#endif ID buf[BDB_IDL_DB_SIZE*BDB_ENOUGH]; char keybuf[16]; @@ -501,8 +508,6 @@ data.ulen = sizeof(buf); data.flags = DB_DBT_USERMEM; - if ( tid ) flags |= DB_RMW; - rc = db->cursor( db, tid, &cursor, bdb->bi_db_opflags ); if( rc != 0 ) { #ifdef NEW_LOGGING @@ -673,7 +678,7 @@ /* Fetch the first data item for this key, to see if it * exists and if it's a range. */ - rc = cursor->c_get( cursor, key, &data, DB_SET | DB_RMW ); + rc = cursor->c_get( cursor, key, &data, DB_SET ); err = "c_get"; if ( rc == 0 ) { if ( tmp != 0 ) { @@ -687,6 +692,7 @@ if ( count >= BDB_IDL_DB_MAX ) { /* No room, convert to a range */ DBT key2 = *key; + db_recno_t i; key2.dlen = key2.ulen; key2.flags |= DB_DBT_PARTIAL; @@ -711,20 +717,48 @@ goto fail; } } - if ( id < lo ) + /* Update hi/lo if needed, then delete all the items + * between lo and hi + */ + if ( id < lo ) { lo = id; - else if ( id > hi ) + } else if ( id > hi ) { hi = id; - rc = db->del( db, tid, key, 0 ); + } + data.data = &id; + /* Don't fetch anything, just position cursor */ + data.flags = DB_DBT_USERMEM | DB_DBT_PARTIAL; + data.dlen = data.ulen = 0; + rc = cursor->c_get( cursor, key, &data, DB_SET ); if ( rc != 0 ) { - err = "del"; + err = "c_get 2"; goto fail; } - data.data = &id; + rc = cursor->c_del( cursor, 0 ); + if ( rc != 0 ) { + err = "c_del range1"; + goto fail; + } + /* Delete all the records */ + for ( i=1; i<count; i++ ) { + rc = cursor->c_get( cursor, &key2, &data, DB_NEXT_DUP ); + if ( rc != 0 ) { + err = "c_get next_dup"; + goto fail; + } + rc = cursor->c_del( cursor, 0 ); + if ( rc != 0 ) { + err = "c_del range"; + goto fail; + } + } + /* Store the range marker */ + data.size = data.ulen = sizeof(ID); + data.flags = DB_DBT_USERMEM; id = 0; rc = cursor->c_put( cursor, key, &data, DB_KEYFIRST ); if ( rc != 0 ) { - err = "c_put 0"; + err = "c_put range"; goto fail; } id = lo; @@ -868,7 +902,7 @@ /* Fetch the first data item for this key, to see if it * exists and if it's a range. */ - rc = cursor->c_get( cursor, key, &data, DB_SET | DB_RMW ); + rc = cursor->c_get( cursor, key, &data, DB_SET ); err = "c_get"; if ( rc == 0 ) { if ( tmp != 0 ) { @@ -876,8 +910,7 @@ if (tmp != id) { /* position to correct item */ tmp = id; - rc = cursor->c_get( cursor, key, &data, - DB_GET_BOTH | DB_RMW ); + rc = cursor->c_get( cursor, key, &data, DB_GET_BOTH ); if ( rc != 0 ) { err = "c_get id"; goto fail; --- servers/slapd/entry.c +++ servers/slapd/entry.c 2005-05-14 14:58:19.000000000 +0200 @@ -47,6 +47,8 @@ NOID, { 0, "" }, { 0, "" }, NULL, 0, { 0, "" }, NULL }; +static const struct berval dn_bv = BER_BVC("dn"); + int entry_destroy(void) { if ( ebuf ) free( ebuf ); @@ -56,18 +58,26 @@ return 0; } - Entry * str2entry( char *s ) { + return str2entry2( s, 1 ); +} + +Entry * +str2entry2( char *s, int checkvals ) +{ int rc; Entry *e; - char *type; - struct berval vals[2]; - struct berval nvals[2], *nvalsp; - AttributeDescription *ad; + struct berval *nvalsp; + struct berval *type, *vals, *nvals; + char *freeval; + AttributeDescription *ad, *ad_prev; const char *text; char *next; + int attr_cnt; + int i, lines; + Attribute ahead, *atail; /* * LDIF is used as the string format. @@ -84,25 +94,16 @@ * or newline. */ -#ifdef NEW_LOGGING - LDAP_LOG( OPERATION, DETAIL1, "str2entry: \"%s\"\n", - s ? s : "NULL", 0, 0 ); -#else Debug( LDAP_DEBUG_TRACE, "=> str2entry: \"%s\"\n", s ? s : "NULL", 0, 0 ); -#endif /* initialize reader/writer lock */ e = (Entry *) ch_calloc( 1, sizeof(Entry) ); if( e == NULL ) { -#ifdef NEW_LOGGING - LDAP_LOG( OPERATION, ERR, "str2entry: entry allocation failed.\n", 0, 0, 0 ); -#else Debug( LDAP_DEBUG_ANY, - "<= str2entry NULL (entry allocation failed)\n", - 0, 0, 0 ); -#endif + "<= str2entry NULL (entry allocation failed)\n", + 0, 0, 0 ); return( NULL ); } @@ -110,98 +111,154 @@ e->e_id = NOID; /* dn + attributes */ - vals[1].bv_len = 0; - vals[1].bv_val = NULL; - + atail = &ahead; + ahead.a_next = NULL; + ad = NULL; + ad_prev = NULL; + attr_cnt = 0; next = s; + + lines = ldif_countlines( s ); + type = ch_calloc( 1, (lines+1)*3*sizeof(struct berval)+lines ); + vals = type+lines+1; + nvals = vals+lines+1; + freeval = (char *)(nvals+lines+1); + i = -1; + + /* parse into individual values, record DN */ while ( (s = ldif_getline( &next )) != NULL ) { + int freev; if ( *s == '\n' || *s == '\0' ) { break; } + i++; - if ( ldif_parse_line( s, &type, &vals[0].bv_val, &vals[0].bv_len ) != 0 ) { -#ifdef NEW_LOGGING - LDAP_LOG( OPERATION, DETAIL1, "str2entry: NULL (parse_line)\n",0, 0, 0 ); -#else + rc = ldif_parse_line2( s, type+i, vals+i, &freev ); + freeval[i] = freev; + if ( rc ) { Debug( LDAP_DEBUG_TRACE, - "<= str2entry NULL (parse_line)\n", 0, 0, 0 ); -#endif + "<= str2entry NULL (parse_line)\n", 0, 0, 0 ); continue; } - if ( strcasecmp( type, "dn" ) == 0 ) { - free( type ); + if ( type[i].bv_len == dn_bv.bv_len && + strcasecmp( type[i].bv_val, dn_bv.bv_val ) == 0 ) { if ( e->e_dn != NULL ) { -#ifdef NEW_LOGGING - LDAP_LOG( OPERATION, DETAIL1, "str2entry: " - "entry %ld has multiple DNs \"%s\" and \"%s\"\n", - (long) e->e_id, e->e_dn, vals[0].bv_val ); -#else Debug( LDAP_DEBUG_ANY, "str2entry: " "entry %ld has multiple DNs \"%s\" and \"%s\"\n", - (long) e->e_id, e->e_dn, vals[0].bv_val ); -#endif - free( vals[0].bv_val ); - entry_free( e ); - return NULL; + (long) e->e_id, e->e_dn, vals[i].bv_val ); + goto fail; } - rc = dnPrettyNormal( NULL, &vals[0], &e->e_name, &e->e_nname, NULL ); + rc = dnPrettyNormal( NULL, &vals[i], &e->e_name, &e->e_nname, NULL ); if( rc != LDAP_SUCCESS ) { -#ifdef NEW_LOGGING - LDAP_LOG( OPERATION, DETAIL1, - "str2entry: entry %ld has invalid DN \"%s\"\n", - (long) e->e_id, vals[0].bv_val, 0 ); -#else Debug( LDAP_DEBUG_ANY, "str2entry: " "entry %ld has invalid DN \"%s\"\n", - (long) e->e_id, vals[0].bv_val, 0 ); -#endif - entry_free( e ); - free( vals[0].bv_val ); - return NULL; + (long) e->e_id, vals[i].bv_val, 0 ); + goto fail; } - free( vals[0].bv_val ); + if ( freeval[i] ) free( vals[i].bv_val ); + vals[i].bv_val = NULL; + i--; continue; } + } + lines = i+1; - ad = NULL; - rc = slap_str2ad( type, &ad, &text ); + /* check to make sure there was a dn: line */ + if ( BER_BVISNULL( &e->e_name )) { + Debug( LDAP_DEBUG_ANY, "str2entry: entry %ld has no dn\n", + (long) e->e_id, 0, 0 ); + goto fail; + } - if( rc != LDAP_SUCCESS ) { -#ifdef NEW_LOGGING - LDAP_LOG( OPERATION, DETAIL1, - "str2entry: str2ad(%s): %s\n", type, text, 0 ); -#else - Debug( slapMode & SLAP_TOOL_MODE - ? LDAP_DEBUG_ANY : LDAP_DEBUG_TRACE, - "<= str2entry: str2ad(%s): %s\n", type, text, 0 ); -#endif - if( slapMode & SLAP_TOOL_MODE ) { - entry_free( e ); - free( vals[0].bv_val ); - free( type ); - return NULL; + /* Make sure all attributes with multiple values are contiguous */ + if ( checkvals ) { + int j, k; + struct berval bv; + int fv; + + for (i=0; i<lines; i++) { + for ( j=i+1; j<lines; j++ ) { + if ( bvmatch( type+i, type+j )) { + /* out of order, move intervening attributes down */ + if ( j != i+1 ) { + bv = vals[j]; + fv = freeval[j]; + for ( k=j; k>i; k-- ) { + type[k] = type[k-1]; + vals[k] = vals[k-1]; + freeval[k] = freeval[k-1]; + } + type[k] = type[i]; + vals[k] = bv; + freeval[k] = fv; + } + i++; + } } + } + } + + for ( i=0; i<=lines; i++ ) { + ad_prev = ad; + if ( !ad || ( i<lines && !bvmatch( type+i, &ad->ad_cname ))) { + ad = NULL; + rc = slap_bv2ad( type+i, &ad, &text ); - rc = slap_str2undef_ad( type, &ad, &text ); if( rc != LDAP_SUCCESS ) { -#ifdef NEW_LOGGING - LDAP_LOG( OPERATION, DETAIL1, - "str2entry: str2undef_ad(%s): %s\n", type, text, 0 ); -#else - Debug( LDAP_DEBUG_ANY, - "<= str2entry: str2undef_ad(%s): %s\n", - type, text, 0 ); -#endif - entry_free( e ); - free( vals[0].bv_val ); - free( type ); - return NULL; + Debug( slapMode & SLAP_TOOL_MODE + ? LDAP_DEBUG_ANY : LDAP_DEBUG_TRACE, + "<= str2entry: str2ad(%s): %s\n", type[i].bv_val, text, 0 ); + if( slapMode & SLAP_TOOL_MODE ) { + goto fail; + } + + rc = slap_bv2undef_ad( type+i, &ad, &text ); + if( rc != LDAP_SUCCESS ) { + Debug( LDAP_DEBUG_ANY, + "<= str2entry: str2undef_ad(%s): %s\n", + type[i].bv_val, text, 0 ); + goto fail; + } } } + if (( ad_prev && ad != ad_prev ) || ( i == lines )) { + int j, k; + atail->a_next = (Attribute *) ch_malloc( sizeof(Attribute) ); + atail = atail->a_next; + atail->a_desc = ad_prev; + atail->a_vals = ch_malloc( (attr_cnt + 1) * sizeof(struct berval)); + if( ad_prev->ad_type->sat_equality && + ad_prev->ad_type->sat_equality->smr_normalize ) + atail->a_nvals = ch_malloc( (attr_cnt + 1) * sizeof(struct berval)); + else + atail->a_nvals = NULL; + k = i - attr_cnt; + for ( j=0; j<attr_cnt; j++ ) { + if ( freeval[k] ) + atail->a_vals[j] = vals[k]; + else + ber_dupbv( atail->a_vals+j, &vals[k] ); + vals[k].bv_val = NULL; + if ( atail->a_nvals ) { + atail->a_nvals[j] = nvals[k]; + nvals[k].bv_val = NULL; + } + k++; + } + BER_BVZERO( &atail->a_vals[j] ); + if ( atail->a_nvals ) { + BER_BVZERO( &atail->a_nvals[j] ); + } else { + atail->a_nvals = atail->a_vals; + } + attr_cnt = 0; + if ( i == lines ) break; + } + if( slapMode & SLAP_TOOL_MODE ) { struct berval pval; slap_syntax_validate_func *validate = @@ -211,55 +268,39 @@ if( pretty ) { rc = pretty( ad->ad_type->sat_syntax, - &vals[0], &pval, NULL ); + &vals[i], &pval, NULL ); } else if( validate ) { /* * validate value per syntax */ - rc = validate( ad->ad_type->sat_syntax, &vals[0] ); + rc = validate( ad->ad_type->sat_syntax, &vals[i] ); } else { -#ifdef NEW_LOGGING - LDAP_LOG( OPERATION, INFO, - "str2entry: no validator for syntax %s\n", - ad->ad_type->sat_syntax->ssyn_oid, 0, 0 ); -#else Debug( LDAP_DEBUG_ANY, - "str2entry: no validator for syntax %s\n", - ad->ad_type->sat_syntax->ssyn_oid, 0, 0 ); -#endif - entry_free( e ); - free( vals[0].bv_val ); - free( type ); - return NULL; + "str2entry: attributeType %s #%d: " + "no validator for syntax %s\n", + ad->ad_cname.bv_val, attr_cnt, + ad->ad_type->sat_syntax->ssyn_oid ); + goto fail; } if( rc != 0 ) { -#ifdef NEW_LOGGING - LDAP_LOG( OPERATION, ERR, - "str2entry: invalid value for attribute %s (syntax %s)\n", - ad->ad_cname.bv_val, ad->ad_type->sat_syntax->ssyn_oid, 0 ); -#else Debug( LDAP_DEBUG_ANY, - "str2entry: invalid value for attribute %s (syntax %s)\n", - ad->ad_cname.bv_val, ad->ad_type->sat_syntax->ssyn_oid, 0 ); -#endif - entry_free( e ); - free( vals[0].bv_val ); - free( type ); - return NULL; + "str2entry: invalid value " + "for attributeType %s #%d (syntax %s)\n", + ad->ad_cname.bv_val, attr_cnt, + ad->ad_type->sat_syntax->ssyn_oid ); + goto fail; } if( pretty ) { - free( vals[0].bv_val ); - vals[0] = pval; + if ( freeval[i] ) free( vals[i].bv_val ); + vals[i] = pval; + freeval[i] = 1; } } - nvalsp = NULL; - nvals[0].bv_val = NULL; - if( ad->ad_type->sat_equality && ad->ad_type->sat_equality->smr_normalize ) { @@ -267,72 +308,35 @@ SLAP_MR_VALUE_OF_ATTRIBUTE_SYNTAX, ad->ad_type->sat_syntax, ad->ad_type->sat_equality, - &vals[0], &nvals[0], NULL ); + &vals[i], &nvals[i], NULL ); if( rc ) { -#ifdef NEW_LOGGING - LDAP_LOG( OPERATION, DETAIL1, - "str2entry: NULL (smr_normalize %d)\n" , rc, 0, 0 ); -#else Debug( LDAP_DEBUG_ANY, "<= str2entry NULL (smr_normalize %d)\n", rc, 0, 0 ); -#endif - - entry_free( e ); - free( vals[0].bv_val ); - free( type ); - return NULL; + goto fail; } - - nvals[1].bv_len = 0; - nvals[1].bv_val = NULL; - - nvalsp = &nvals[0]; - } - - rc = attr_merge( e, ad, vals, nvalsp ); - if( rc != 0 ) { -#ifdef NEW_LOGGING - LDAP_LOG( OPERATION, DETAIL1, - "str2entry: NULL (attr_merge)\n" , 0, 0, 0 ); -#else - Debug( LDAP_DEBUG_ANY, - "<= str2entry NULL (attr_merge)\n", 0, 0, 0 ); -#endif - entry_free( e ); - free( vals[0].bv_val ); - free( type ); - return( NULL ); } - free( type ); - free( vals[0].bv_val ); - free( nvals[0].bv_val ); + attr_cnt++; } - /* check to make sure there was a dn: line */ - if ( e->e_dn == NULL ) { -#ifdef NEW_LOGGING - LDAP_LOG( OPERATION, INFO, - "str2entry: entry %ld has no dn.\n", (long) e->e_id, 0, 0 ); -#else - Debug( LDAP_DEBUG_ANY, "str2entry: entry %ld has no dn\n", - (long) e->e_id, 0, 0 ); -#endif - entry_free( e ); - return NULL; - } + free( type ); + atail->a_next = NULL; + e->e_attrs = ahead.a_next; -#ifdef NEW_LOGGING - LDAP_LOG( OPERATION, DETAIL2, - "str2entry(%s) -> 0x%lx\n", e->e_dn, (unsigned long)e, 0 ); -#else Debug(LDAP_DEBUG_TRACE, "<= str2entry(%s) -> 0x%lx\n", e->e_dn, (unsigned long) e, 0 ); -#endif return( e ); -} +fail: + for ( i=0; i<lines; i++ ) { + if ( freeval[i] ) free( vals[i].bv_val ); + free( nvals[i].bv_val ); + } + free( type ); + entry_free( e ); + return NULL; +} #define GRABSIZE BUFSIZ --- doc/man/man8/slapadd.8 +++ doc/man/man8/slapadd.8 2005-03-21 16:54:06.000000000 -0800 @@ -9,6 +9,7 @@ .B [\-v] .B [\-c] .B [\-u] +.B [\-q] .B [\-p] .B [\-r] .B [\-w] @@ -49,6 +50,12 @@ .B \-u enable dry-run (don't write to backend) mode. .TP +.B \-q +enable quick (fewer integrity checks) mode. Does fewer consistency checks +on the input data, and no consistency checks when writing the database. +Improves the load time but if any errors or interruptions occur the resulting +database will be unusable. +.TP .BI \-p If the LDIF input represents a syncrepl consumer replica, promote it to the syncrepl provider's content after adding