The branch, master has been updated via c40531d ldb: Release ldb 1.3.0 via 303739e ldb_tdb: Treat distinguishedName and objectGUID (in Samba) as unique via 644b3ff ldb: Ensure we do not run out of File descriptors in autobuild via a0b46ec ldb: Do not make an ldb file for API tests via 8565ca1 ldb: Add test showing a search against the index is not possible via f0624d3 ldb_tdb: Also ban a (indexed) search against like @IDXDN=foo via f4f9acf ldb_tdb: Update comments for the delete_index() pass of ltdb_reindex() via 61b66b8 ldb_tdb: Print progress messages on re-index via c71ddab ldb_tdb: Remove incorrect early return from re-index via 0503f09 ldb: Add more tests covering in-transaction GUID index and unique index behaviour via 00410ba ldb_tdb: Add UNIQUE_INDEX as a possible flag via 561ef20 ldb_tdb: Remove LTDB_FLAG_HIDDEN and ignore "HIDDEN" in @ATTRIBUTES via 29c303f ldb_tdb: Clean up index records on ltdb_index_add_new() failure. via 83bc607 ldb_tdb: Describe index format and control points via 3113d87 ldb_tdb: Give a good error message on add without an objectGUID via b9d561b ldb_tdb: Avoid canonicalise and base64 work for DN values, these are already OK via 3e60f02 ldb_tdb: Clean up list in after use in ltdb_key_dn_from_idx() via d198a43 ldb_tdb: Avoid allocation of a DN between the GUID index and the DB lookup via 4e99785 ldb_tdb: Move constants into ldb_tdb.h via b0a6b5a ldb_tdb: Optimise ltdb_search_and_return_base() to re-use casefolding via f14370d ldb: Add tests for base DN of a different case via dfe85ec ldb_tdb: Use the DN extracted from the DB to filter the message later via 23e0a55 ldb_tdb: Add tests for add/modify with the GUID index via 24c71ed ldb: Also test the new GUID index mode via 64b4fbc ldb: Add an unused objectGUID to each record in SearchTests via a2ada58 ldb_tdb: Duplicate values are no longer permitted in the index via 3c6977b ldb_tdb: Reduce memory consumption in list_intersect() via 97b026a ldb_tdb: Use the binary search more efficiently in list_intersect() via b6bf7e7 ldb_tdb: Use a binary search to speed up ltdb_dn_list_find_val() via fdff9a7 ldb_tdb: Rework list_union to not return duplicates, and keep sort order via b86a46d ldb_tdb: Sort inputs to list_union() via 1390e55 ldb_tdb: sort GUID index list at add time by inserting in sorted order via 9c9d9ef binsearch.h: Re-licence under LGPLv3 per agreement of the copyright holders via cf85370 ldb_tdb: Change error code on unique index violation via e8fb451 ldb_tdb: Re-add of both existing DN and GUID must gives LDB_ERR_ENTRY_ALREADY_EXISTS via e16c8aa ldb_tdb: Add unique index checking for @IDXDN via b97d556 ldb_tdb: Improve debugging in ltdb_modify_index_dn() on casefold failure via 1f42ded ldb_tdb: Add improved error strings on ltdb_key_dn_from_idx() failure via 50f36e4 ldb_tdb: Read from @INDEXLIST or an override if we are using a GUID index via 2f8a8c7 ldb_tdb: Optionally use GUID index values a direct TDB keys via 93b1898 ldb_tdb: Trust the BASE and ONELEVEL index via e4562e5 ldb_tdb: Add ltdb_idx_to_key() and use it in ltdb_index_filter() via ba762fa ldb_tdb: Do not add an index for GUID_index_attribute via c4f3546 ldb_tdb: Do not query an index on the GUID_index_attribute via 6008382 ldb_tdb: Optionally use GUID index in ltdb_search_dn1() via fa44c52 ldb_tdb: Use the objectGUID (or similar) as the TDB key in ltdb_key_msg() via f253dcf ldb_tdb: Use ltdb_key_msg() in ltdb_delete_noindex() via 6ad4bdc ldb_tdb: Add mem_ctx to ltdb_key_dn() and ltdb_key_msg() via fb77ad4 ldb_tdb: Check version number on index via e394b9a ldb_tdb: Add an index shortcut for a <GUID= DN via ccb9443 ldb_tdb: Add a function to get the GUID key for a DN via f26d1a8 ldb_tdb: Add a function to take a GUID and make the TDB_DATA key via aa68957 ldb_tdb: Load the syntax of the GUID index attr during ltdb_cache_load() via 8555f6c ldb_tdb: add control points for the new GUID index mode via 3b0698e ldb_tdb: Do not directly assign DN into the index result list via 19ee043 ldb_tdb: Pass ltdb_private to ltdb_dn_list_load() via 2bac00a ldb_tdb: Give LDB_ERR_CONSTRAINT_VIOLATION when a duplicate GUID index is detected via 06e67c5 ldb_tdb: Give LDB_ERR_ENTRY_ALREADY_EXISTS when a duplicate DN index is detected via ec30439 ldb_tdb: Add/remove a GUID index of the DN during ltdb_index_add_all()/ltdb_index_delete() via 98e8211 ldb_tdb: Split ltdb_index_onelevel() into a helper function via b2aff12 ldb_tdb: Prepare to handle rename with GUID index by using ltdb_search_base() via c24df8e ldb_tdb: Implement ltdb_search_base() for a GUID index via 754329a ldb_tdb: Optionally store a GUID as the index record via b154acb ldb_tdb: Optionally use a GUID index key in ltdb_dn_list_find_msg() via 9d9ae1c ldb_tdb: Pass the full ldb_message to ldb index funtions via 4c01ccb ldb_tdb: Delete a successful tdb_store on index add fail in ltdb_add_internal() via e8cdacc ldb_tdb: modify ltdb_delete_noindex() to take a struct ldb_message via de641db ldb_tdb: Write GUID index values as version 3 via 91e6028 ldb_tdb: Refuse to load a GUID index that is not a multiple of 16 bytes via ad26b4e ldb_tdb: Read GUID index values as one packed LDB attribute via f6e953d ldb_tdb: Store GUID index values in one packed ldb attribute via 367b7ab ldb_tdb: Move constants into ldb_tdb.h via c174048 ldb_tdb: replace strange dn_list_cmp() in index code via a44d3ae ldb_tdb: Do not allow a modification of the GUID_index_attribute (objectGUID) via 26dd6f6 ldb_tdb: Add GUID_index_attribute to ltdb_private via 3d95215 ldb_tdb: Provide struct ltdb_private to index routines via dded015 ldb_tdb: Use a more complete error mapping in ltdb_search_key() via 6db8095 ldb_tdb: Add ltdb_search_key() via d4c1a60 ldb_tdb: Use ltdb_key_msg() in re_index() via 1b310ad ldb_tdb: provide ldb_key_dn() and ldb_key_msg() via cd8ecb3 ldb_tdb: Add helper function ltdb_search_and_return_base() via d8a2288 ldb_tdb: Make ldb_match_message() available to ldb_tdb from 4df3dcd ctdb-tests: Add tests for client with multiple connections
https://git.samba.org/?p=samba.git;a=shortlog;h=master - Log ----------------------------------------------------------------- commit c40531d9b964588f96d2ab5545b328f8dc787ad9 Author: Andrew Bartlett <abart...@samba.org> Date: Tue Aug 15 15:54:00 2017 +1200 ldb: Release ldb 1.3.0 * GUID Index support. NOTE: When activated by setting @IDXGUID in the @INDEXLIST dn, all entries in the DB are re-keyed in a way that is NOT visible to ldb 1.2.2 and earlier. To re-key back to the previous format, remove the @IDXGUID attribute from @INDEXLIST using ldb 1.2.2 or later. (ldb 1.2.2 can re-key, but not otherwise read, the new DB format). * Give LDB_ERR_CONSTRAINT_VIOLATION, not LDB_ERR_ENTRY_ALREADY_EXISTS when a duplicate value is detected in a unique index * Print status information during a > 10,000 entry re-index (as this can be slow) Signed-off-by: Andrew Bartlett <abart...@samba.org> Reviewed-by: Garming Sam <garm...@catalyst.net.nz> Autobuild-User(master): Andrew Bartlett <abart...@samba.org> Autobuild-Date(master): Sat Sep 23 01:24:19 CEST 2017 on sn-devel-144 commit 303739e8017a0fac957ea97df35655d26b9a481e Author: Andrew Bartlett <abart...@samba.org> Date: Thu Sep 21 21:11:54 2017 +1200 ldb_tdb: Treat distinguishedName and objectGUID (in Samba) as unique This avoids loading any second index for these cases. Signed-off-by: Andrew Bartlett <abart...@samba.org> Reviewed-by: Douglas Bagnall <douglas.bagn...@catalyst.net.nz> commit 644b3ffb14252f2a2dfe282b5d748b6ea86325e1 Author: Andrew Bartlett <abart...@samba.org> Date: Thu Sep 21 14:15:32 2017 +1200 ldb: Ensure we do not run out of File descriptors in autobuild The python TestCase API will keep a reference to the test object until the end of the tests, long after we need the actual LDB or the fd. Signed-off-by: Andrew Bartlett <abart...@samba.org> Reviewed-by: Douglas Bagnall <douglas.bagn...@catalyst.net.nz> commit a0b46ecdd2481699ce91c2636d4eac9067f8f69e Author: Andrew Bartlett <abart...@samba.org> Date: Thu Sep 21 13:47:49 2017 +1200 ldb: Do not make an ldb file for API tests An ldb context is valid without a backing file for tests of ldb.Message and ldb.MessageElement Signed-off-by: Andrew Bartlett <abart...@samba.org> Reviewed-by: Douglas Bagnall <douglas.bagn...@catalyst.net.nz> commit 8565ca1c6d9d80a326525b1c66ee7e93dab08ead Author: Andrew Bartlett <abart...@samba.org> Date: Wed Sep 13 15:49:43 2017 +1200 ldb: Add test showing a search against the index is not possible This is not actually a great test, as the filter would fail to match these anyway, but it at least checks the codepath is safe. Signed-off-by: Andrew Bartlett <abart...@samba.org> Reviewed-by: Garming Sam <garm...@catalyst.net.nz> commit f0624d389010bd3ac760f8f07a7174fd708496be Author: Andrew Bartlett <abart...@samba.org> Date: Wed Sep 13 13:00:08 2017 +1200 ldb_tdb: Also ban a (indexed) search against like @IDXDN=foo Signed-off-by: Andrew Bartlett <abart...@samba.org> Reviewed-by: Garming Sam <garm...@catalyst.net.nz> commit f4f9acf7811f4df31e0a675f2c23198161f3bcf9 Author: Andrew Bartlett <abart...@samba.org> Date: Mon Sep 11 22:00:40 2017 +1200 ldb_tdb: Update comments for the delete_index() pass of ltdb_reindex() Signed-off-by: Andrew Bartlett <abart...@samba.org> Reviewed-by: Garming Sam <garm...@catalyst.net.nz> commit 61b66b8d0a2a29496ad96774011e158e5f089bf0 Author: Andrew Bartlett <abart...@samba.org> Date: Mon Sep 11 13:16:31 2017 +1200 ldb_tdb: Print progress messages on re-index A re-index of 10,000 entries is slow enough and rare enought that we can justify the message being at LDB_DEBUG_WARNING as otherwise the administrator will be sure the "lockup" was one. The default for ldb is to print LDB_DEBUG_WARNING in comand-line tools and the default for Samba is to log it at level 2. Signed-off-by: Andrew Bartlett <abart...@samba.org> Reviewed-by: Garming Sam <garm...@catalyst.net.nz> commit c71ddab974eae65a8b1120de0a62d67bc62223c9 Author: Andrew Bartlett <abart...@samba.org> Date: Mon Sep 11 11:49:02 2017 +1200 ldb_tdb: Remove incorrect early return from re-index The ltdb->cache->attribute_indexes test is not correct with the GUID index mode so for consistency remove it. This will make re-index on a large un-indexed database slower, but that is better than making the wrong choice on a large GUID-indexed database. Signed-off-by: Andrew Bartlett <abart...@samba.org> Reviewed-by: Garming Sam <garm...@catalyst.net.nz> commit 0503f0984afa1490b7b328d5f1ee933e0eaaba0b Author: Andrew Bartlett <abart...@samba.org> Date: Wed Sep 13 11:28:23 2017 +1200 ldb: Add more tests covering in-transaction GUID index and unique index behaviour Signed-off-by: Andrew Bartlett <abart...@samba.org> Reviewed-by: Garming Sam <garm...@catalyst.net.nz> commit 00410ba99f55f317eae188aebe3af40a6b43f71d Author: Andrew Bartlett <abart...@samba.org> Date: Wed Sep 13 12:36:43 2017 +1200 ldb_tdb: Add UNIQUE_INDEX as a possible flag This allows easy testing of our unique index code and behaivour from python Signed-off-by: Andrew Bartlett <abart...@samba.org> Reviewed-by: Garming Sam <garm...@catalyst.net.nz> commit 561ef206344f1b829d419ad5c024cc81a09645e3 Author: Andrew Bartlett <abart...@samba.org> Date: Mon Sep 11 10:02:16 2017 +1200 ldb_tdb: Remove LTDB_FLAG_HIDDEN and ignore "HIDDEN" in @ATTRIBUTES This was (unintentionally) disabled by 6ef61825541131e16a03975cdbd344e2bbebf810 in 2006. Signed-off-by: Andrew Bartlett <abart...@samba.org> Reviewed-by: Garming Sam <garm...@catalyst.net.nz> commit 29c303f21b40b905375fb2b971ef6cdcdcb9eb36 Author: Andrew Bartlett <abart...@samba.org> Date: Fri Sep 8 18:07:51 2017 +1200 ldb_tdb: Clean up index records on ltdb_index_add_new() failure. Signed-off-by: Andrew Bartlett <abart...@samba.org> Reviewed-by: Garming Sam <garm...@catalyst.net.nz> commit 83bc60746310e47855d21bef745298ffe31aa5b3 Author: Andrew Bartlett <abart...@samba.org> Date: Fri Sep 8 15:30:08 2017 +1200 ldb_tdb: Describe index format and control points Signed-off-by: Andrew Bartlett <abart...@samba.org> Reviewed-by: Garming Sam <garm...@catalyst.net.nz> commit 3113d871bdb7bc9daf6616337ef9d83712af8f9d Author: Andrew Bartlett <abart...@samba.org> Date: Sat Sep 2 16:19:39 2017 +1200 ldb_tdb: Give a good error message on add without an objectGUID (or whatever the @IDX_GUID value is) Signed-off-by: Andrew Bartlett <abart...@samba.org> Reviewed-by: Garming Sam <garm...@catalyst.net.nz> commit b9d561b023cefa8e64abc4c5772f6a5d704f19ab Author: Andrew Bartlett <abart...@samba.org> Date: Fri Sep 1 20:06:50 2017 +1200 ldb_tdb: Avoid canonicalise and base64 work for DN values, these are already OK This is important with the GUID index, as a DN lookup is much more common now. Signed-off-by: Andrew Bartlett <abart...@samba.org> Reviewed-by: Garming Sam <garm...@catalyst.net.nz> commit 3e60f027535cff737a431adf4658f0f760ba9d0c Author: Andrew Bartlett <abart...@samba.org> Date: Fri Sep 1 22:25:43 2017 +1200 ldb_tdb: Clean up list in after use in ltdb_key_dn_from_idx() Signed-off-by: Andrew Bartlett <abart...@samba.org> Reviewed-by: Garming Sam <garm...@catalyst.net.nz> commit d198a43b553692060a0630fea0c8e3b9a2890d76 Author: Andrew Bartlett <abart...@samba.org> Date: Fri Sep 1 20:06:15 2017 +1200 ldb_tdb: Avoid allocation of a DN between the GUID index and the DB lookup Signed-off-by: Andrew Bartlett <abart...@samba.org> Reviewed-by: Garming Sam <garm...@catalyst.net.nz> commit 4e9978506aa6f09db8fe874457965fbe85fafd17 Author: Andrew Bartlett <abart...@samba.org> Date: Fri Sep 1 20:04:43 2017 +1200 ldb_tdb: Move constants into ldb_tdb.h This helps ensure we keep these all in sync. Signed-off-by: Andrew Bartlett <abart...@samba.org> Reviewed-by: Garming Sam <garm...@catalyst.net.nz> commit b0a6b5ab515661a7aa227e329e5a506fa802c198 Author: Andrew Bartlett <abart...@samba.org> Date: Wed Aug 30 13:18:20 2017 +1200 ldb_tdb: Optimise ltdb_search_and_return_base() to re-use casefolding The casefolding of a DN is one of the more expensive and pointless things in LDB operation. The ldb_dn abstraction works hard to avoid duplicating this work, but we can work harder to save that information. Here we copy in the DN, that has been casefolded already for the index, and keep that as the returned DN, after stripping any extended components. Signed-off-by: Andrew Bartlett <abart...@samba.org> Reviewed-by: Garming Sam <garm...@catalyst.net.nz> commit f14370d4710d170a10d530a7bff9cd6cdd61dea0 Author: Andrew Bartlett <abart...@samba.org> Date: Tue Aug 29 10:40:22 2017 +1200 ldb: Add tests for base DN of a different case This ensures we cover the case where the DN does not match the DB exactly Signed-off-by: Andrew Bartlett <abart...@samba.org> Reviewed-by: Garming Sam <garm...@catalyst.net.nz> commit dfe85ecacab2ef6ce7efa3dd42734bdfafbd3096 Author: Andrew Bartlett <abart...@samba.org> Date: Mon Aug 28 15:37:28 2017 +1200 ldb_tdb: Use the DN extracted from the DB to filter the message later This should ensure that the upper or lower case the user chooses does not impact on the filtering, at least for database that have checkBaseOnSearch set. Signed-off-by: Andrew Bartlett <abart...@samba.org> Reviewed-by: Garming Sam <garm...@catalyst.net.nz> commit 23e0a553d436015c11d20b502d47d89341a91bb1 Author: Andrew Bartlett <abart...@samba.org> Date: Fri Aug 25 22:22:27 2017 +1200 ldb_tdb: Add tests for add/modify with the GUID index Signed-off-by: Andrew Bartlett <abart...@samba.org> Reviewed-by: Garming Sam <garm...@catalyst.net.nz> commit 24c71ed2116b28c303fc26449c92c8c2fe307f33 Author: Andrew Bartlett <abart...@samba.org> Date: Fri Aug 25 15:34:31 2017 +1200 ldb: Also test the new GUID index mode Signed-off-by: Andrew Bartlett <abart...@samba.org> Reviewed-by: Garming Sam <garm...@catalyst.net.nz> commit 64b4fbc613d632299f62dbda4595e7ff49d63e63 Author: Andrew Bartlett <abart...@samba.org> Date: Fri Aug 25 12:06:48 2017 +1200 ldb: Add an unused objectGUID to each record in SearchTests This will then be used by the GUID index tests. Signed-off-by: Andrew Bartlett <abart...@samba.org> Reviewed-by: Garming Sam <garm...@catalyst.net.nz> commit a2ada5844d1f3b3e9cb24aeaa68a6a921363e139 Author: Andrew Bartlett <abart...@samba.org> Date: Tue Aug 22 11:55:01 2017 +1200 ldb_tdb: Duplicate values are no longer permitted in the index By removing the qsort() we avoid work. Signed-off-by: Andrew Bartlett <abart...@samba.org> Reviewed-by: Garming Sam <garm...@catalyst.net.nz> commit 3c6977b17c66e1bc8dbab9df5cf2338868f1a180 Author: Andrew Bartlett <abart...@samba.org> Date: Tue Aug 22 11:17:56 2017 +1200 ldb_tdb: Reduce memory consumption in list_intersect() We will never have more results than is in either list or list2. Signed-off-by: Andrew Bartlett <abart...@samba.org> Reviewed-by: Garming Sam <garm...@catalyst.net.nz> commit 97b026a73fafa0d8b54d3b69fcce604d8c44ebcc Author: Andrew Bartlett <abart...@samba.org> Date: Tue Aug 22 11:16:56 2017 +1200 ldb_tdb: Use the binary search more efficiently in list_intersect() This change ensures we walk the short list and look up into the longer of the two lists. ltdb_dn_list_find_val() will do a binary search for the GUID case. Before GUID indexes this was O(n*m), now it is O(n*log(m)). Signed-off-by: Andrew Bartlett <abart...@samba.org> Reviewed-by: Garming Sam <garm...@catalyst.net.nz> commit b6bf7e7b0b8bd11d2c804125140b2a8077c11d85 Author: Andrew Bartlett <abart...@samba.org> Date: Mon Aug 21 15:51:19 2017 +1200 ldb_tdb: Use a binary search to speed up ltdb_dn_list_find_val() This only works if we have the GUID index format, as otherwise these are unsorted. Signed-off-by: Andrew Bartlett <abart...@samba.org> Reviewed-by: Garming Sam <garm...@catalyst.net.nz> commit fdff9a70876ed6ff06a4cc0076f8e85838f14b06 Author: Andrew Bartlett <abart...@samba.org> Date: Tue Aug 22 11:07:45 2017 +1200 ldb_tdb: Rework list_union to not return duplicates, and keep sort order This allows the binary search to still operate on the list, even after a or operator in the search expression Signed-off-by: Andrew Bartlett <abart...@samba.org> Reviewed-by: Garming Sam <garm...@catalyst.net.nz> commit b86a46df1df60e9362b9c3cc7f41e8315329f3aa Author: Andrew Bartlett <abart...@samba.org> Date: Tue Aug 22 11:51:30 2017 +1200 ldb_tdb: Sort inputs to list_union() This allows us to merge the lists finding common values. Signed-off-by: Andrew Bartlett <abart...@samba.org> Reviewed-by: Garming Sam <garm...@catalyst.net.nz> commit 1390e55bb5e0cb601b2aa216a8cd49736fdd6fa0 Author: Andrew Bartlett <abart...@samba.org> Date: Mon Aug 21 15:35:32 2017 +1200 ldb_tdb: sort GUID index list at add time by inserting in sorted order Signed-off-by: Andrew Bartlett <abart...@samba.org> Reviewed-by: Garming Sam <garm...@catalyst.net.nz> commit 9c9d9ef69e3dc0ad6e84428ff50b3b3d8ad405fc Author: Andrew Bartlett <abart...@samba.org> Date: Thu Sep 21 12:35:45 2017 +1200 binsearch.h: Re-licence under LGPLv3 per agreement of the copyright holders Documented in mails to contribut...@samba.org. Signed-off-by: Andrew Bartlett <abart...@samba.org> commit cf8537068fc71d27dcb9994f73099a3664ba7c31 Author: Andrew Bartlett <abart...@samba.org> Date: Mon Aug 21 14:25:46 2017 +1200 ldb_tdb: Change error code on unique index violation The LDB_ERR_ENTRY_ALREADY_EXISTS error code is detected in repl_meta_data as indicating that the DN exists, and that a conflict record should be created. This is really a constraint violation, not a duplicate record. Signed-off-by: Andrew Bartlett <abart...@samba.org> Reviewed-by: Garming Sam <garm...@catalyst.net.nz> commit e8fb45125e6a279b918694668e0d4fbddac10aee Author: Andrew Bartlett <abart...@samba.org> Date: Wed Sep 13 12:37:51 2017 +1200 ldb_tdb: Re-add of both existing DN and GUID must gives LDB_ERR_ENTRY_ALREADY_EXISTS Signed-off-by: Andrew Bartlett <abart...@samba.org> Reviewed-by: Garming Sam <garm...@catalyst.net.nz> commit e16c8aa5daf58f1d281697af8412c8df94e67755 Author: Andrew Bartlett <abart...@samba.org> Date: Mon Aug 21 14:24:44 2017 +1200 ldb_tdb: Add unique index checking for @IDXDN This will give us errors earlier if the index code becomes broken Signed-off-by: Andrew Bartlett <abart...@samba.org> Reviewed-by: Garming Sam <garm...@catalyst.net.nz> commit b97d5567318f7a116494d393a05a8e42c824fd0e Author: Andrew Bartlett <abart...@samba.org> Date: Mon Aug 21 12:59:50 2017 +1200 ldb_tdb: Improve debugging in ltdb_modify_index_dn() on casefold failure This is unlikely, but when it happens it will be really painful to debug. Signed-off-by: Andrew Bartlett <abart...@samba.org> Reviewed-by: Garming Sam <garm...@catalyst.net.nz> commit 1f42ded463ac91a8b5431f280f745d7f3a1de770 Author: Andrew Bartlett <abart...@samba.org> Date: Mon Aug 21 11:17:59 2017 +1200 ldb_tdb: Add improved error strings on ltdb_key_dn_from_idx() failure Signed-off-by: Andrew Bartlett <abart...@samba.org> Reviewed-by: Garming Sam <garm...@catalyst.net.nz> commit 50f36e4fb5e251042d2c35e804f0bc477e2b790e Author: Andrew Bartlett <abart...@samba.org> Date: Tue Aug 15 15:57:57 2017 +1200 ldb_tdb: Read from @INDEXLIST or an override if we are using a GUID index This allows all the previous patches to be enabled. Signed-off-by: Andrew Bartlett <abart...@samba.org> Reviewed-by: Garming Sam <garm...@catalyst.net.nz> commit 2f8a8c765f652dfb438f3d92d51e9af863aefdaa Author: Andrew Bartlett <abart...@samba.org> Date: Thu Aug 10 17:05:37 2017 +1200 ldb_tdb: Optionally use GUID index values a direct TDB keys This connects the GUID based index records to GUID based TDB keys. Signed-off-by: Andrew Bartlett <abart...@samba.org> Reviewed-by: Garming Sam <garm...@catalyst.net.nz> commit 93b18984bd4603ad14a3ed71f4e10b43443803e0 Author: Andrew Bartlett <abart...@samba.org> Date: Tue Aug 29 09:59:54 2017 +1200 ldb_tdb: Trust the BASE and ONELEVEL index This avoids re-checking the fetched DN against the scope Signed-off-by: Andrew Bartlett <abart...@samba.org> Reviewed-by: Garming Sam <garm...@catalyst.net.nz> commit e4562e58f9018198600b5a303d7d832cf8a475f0 Author: Andrew Bartlett <abart...@samba.org> Date: Thu Aug 17 07:15:50 2017 +1200 ldb_tdb: Add ltdb_idx_to_key() and use it in ltdb_index_filter() This will allow a common point to parse index records into a TDB key, allowing them to be a GUID or DN in the future Signed-off-by: Andrew Bartlett <abart...@samba.org> Reviewed-by: Garming Sam <garm...@catalyst.net.nz> commit ba762fac819eaefd980bd795de5227a04d2fbb2e Author: Andrew Bartlett <abart...@samba.org> Date: Thu Aug 17 12:44:34 2017 +1200 ldb_tdb: Do not add an index for GUID_index_attribute This would be pointless and we no longer query for it. Signed-off-by: Andrew Bartlett <abart...@samba.org> Reviewed-by: Garming Sam <garm...@catalyst.net.nz> commit c4f35462e9f2c6b6d202aa5d9bf407b5fd9a0883 Author: Andrew Bartlett <abart...@samba.org> Date: Thu Sep 21 22:21:36 2017 +1200 ldb_tdb: Do not query an index on the GUID_index_attribute The objectGUID (or similar) is already the record key, there is no need to index it to itself. Signed-off-by: Andrew Bartlett <abart...@samba.org> Reviewed-by: Garming Sam <garm...@catalyst.net.nz> commit 6008382191f5115539906f264050b80b61b0693b Author: Andrew Bartlett <abart...@samba.org> Date: Thu Aug 10 17:12:30 2017 +1200 ldb_tdb: Optionally use GUID index in ltdb_search_dn1() Signed-off-by: Andrew Bartlett <abart...@samba.org> Reviewed-by: Garming Sam <garm...@catalyst.net.nz> commit fa44c5262b5485273e39b52dc821a74ce6878515 Author: Andrew Bartlett <abart...@samba.org> Date: Thu Aug 10 17:11:13 2017 +1200 ldb_tdb: Use the objectGUID (or similar) as the TDB key in ltdb_key_msg() When we have the full ldb_message we can read the objectGUID as the TDB key Signed-off-by: Andrew Bartlett <abart...@samba.org> Reviewed-by: Garming Sam <garm...@catalyst.net.nz> commit f253dcf8b99ea7a9afc57496d685ed0ee110a3c0 Author: Andrew Bartlett <abart...@samba.org> Date: Thu Aug 17 12:53:34 2017 +1200 ldb_tdb: Use ltdb_key_msg() in ltdb_delete_noindex() This allows the optional use of GUID based TDB key. Signed-off-by: Andrew Bartlett <abart...@samba.org> Reviewed-by: Garming Sam <garm...@catalyst.net.nz> commit 6ad4bdc9de7dd5a5cf6807b66d046d04a4aea4b9 Author: Andrew Bartlett <abart...@samba.org> Date: Mon Aug 14 15:47:15 2017 +1200 ldb_tdb: Add mem_ctx to ltdb_key_dn() and ltdb_key_msg() This follows modern Samba coding style where memory returned is allocated on a supplied memory context. Signed-off-by: Andrew Bartlett <abart...@samba.org> Reviewed-by: Garming Sam <garm...@catalyst.net.nz> commit fb77ad42447e2e190026c6edf84cdbe4297ebdfe Author: Andrew Bartlett <abart...@samba.org> Date: Thu Aug 17 17:27:23 2017 +1200 ldb_tdb: Check version number on index Signed-off-by: Andrew Bartlett <abart...@samba.org> Reviewed-by: Garming Sam <garm...@catalyst.net.nz> commit e394b9a8568a52a6afa0bb29685d6bdf1f3d8354 Author: Andrew Bartlett <abart...@samba.org> Date: Mon Aug 14 16:13:42 2017 +1200 ldb_tdb: Add an index shortcut for a <GUID= DN Signed-off-by: Andrew Bartlett <abart...@samba.org> Reviewed-by: Garming Sam <garm...@catalyst.net.nz> commit ccb94436642c8254049e251d954e05605c66f591 Author: Andrew Bartlett <abart...@samba.org> Date: Fri Aug 11 18:09:01 2017 +1200 ldb_tdb: Add a function to get the GUID key for a DN Signed-off-by: Andrew Bartlett <abart...@samba.org> Reviewed-by: Garming Sam <garm...@catalyst.net.nz> commit f26d1a8c5de931685b5de7a52a58ebcbf2ea5f88 Author: Andrew Bartlett <abart...@samba.org> Date: Wed Aug 16 10:44:34 2017 +1200 ldb_tdb: Add a function to take a GUID and make the TDB_DATA key This allows us to format the TDB key as DN=GUID=f7c953ee-cf9c-433f-b423-21ce04d09591 and so be compatible with an un-indexed search and a re-index with an old ldb. Signed-off-by: Andrew Bartlett <abart...@samba.org> Reviewed-by: Garming Sam <garm...@catalyst.net.nz> commit aa68957f7f086fcedc9c0d5df7f29cbc7c002a26 Author: Andrew Bartlett <abart...@samba.org> Date: Wed Aug 16 10:42:40 2017 +1200 ldb_tdb: Load the syntax of the GUID index attr during ltdb_cache_load() This allows us to use the ldif_write function later to create a string GUID for the TDB key. Signed-off-by: Andrew Bartlett <abart...@samba.org> Reviewed-by: Garming Sam <garm...@catalyst.net.nz> commit 8555f6ccc244543818727a2186a643cd54cc9aed Author: Andrew Bartlett <abart...@samba.org> Date: Tue Aug 15 15:33:24 2017 +1200 ldb_tdb: add control points for the new GUID index mode The @IDXGUID attribute in the @INDEXLIST will be objectGUID in Samba. The @IDX_DN_GUID attribute in the @INDEXLIST will be GUID in Samba. Signed-off-by: Andrew Bartlett <abart...@samba.org> Reviewed-by: Garming Sam <garm...@catalyst.net.nz> commit 3b0698e0b672a8bb67454183031ecb2f91740177 Author: Andrew Bartlett <abart...@samba.org> Date: Fri Aug 11 17:51:40 2017 +1200 ldb_tdb: Do not directly assign DN into the index result list With the GUID index option, the values in the index result list may not be a DN but the objectGUID. We look up the @IDXDN index with the case-folded DN to get that if required. We re-use the code from the SCOPE_BASE search do avoid duplication and for reliablity. Signed-off-by: Andrew Bartlett <abart...@samba.org> Reviewed-by: Douglas Bagnall <douglas.bagn...@catalyst.net.nz> commit 19ee0438ffffc357c7d04e1f36cf56d2f169bea8 Author: Andrew Bartlett <abart...@samba.org> Date: Fri Aug 11 13:26:30 2017 +1200 ldb_tdb: Pass ltdb_private to ltdb_dn_list_load() Signed-off-by: Andrew Bartlett <abart...@samba.org> Reviewed-by: Garming Sam <garm...@catalyst.net.nz> commit 2bac00a7b9da064932644647f037e1a03703be9d Author: Andrew Bartlett <abart...@samba.org> Date: Fri Aug 25 22:21:26 2017 +1200 ldb_tdb: Give LDB_ERR_CONSTRAINT_VIOLATION when a duplicate GUID index is detected Signed-off-by: Andrew Bartlett <abart...@samba.org> Reviewed-by: Garming Sam <garm...@catalyst.net.nz> commit 06e67c5894d705bc7c72c2585436af7cedbbb839 Author: Andrew Bartlett <abart...@samba.org> Date: Fri Aug 25 22:21:00 2017 +1200 ldb_tdb: Give LDB_ERR_ENTRY_ALREADY_EXISTS when a duplicate DN index is detected Signed-off-by: Andrew Bartlett <abart...@samba.org> Reviewed-by: Garming Sam <garm...@catalyst.net.nz> commit ec30439c43f40d7f20df64c56394c9e023dcd914 Author: Andrew Bartlett <abart...@samba.org> Date: Fri Aug 11 12:25:42 2017 +1200 ldb_tdb: Add/remove a GUID index of the DN during ltdb_index_add_all()/ltdb_index_delete() Signed-off-by: Andrew Bartlett <abart...@samba.org> Reviewed-by: Garming Sam <garm...@catalyst.net.nz> commit 98e82113ed050ad1658a446b15cc30c6894cd78f Author: Andrew Bartlett <abart...@samba.org> Date: Mon Aug 21 12:58:58 2017 +1200 ldb_tdb: Split ltdb_index_onelevel() into a helper function This will allow the code to be re-used for storing the DN->GUID index Signed-off-by: Andrew Bartlett <abart...@samba.org> Reviewed-by: Garming Sam <garm...@catalyst.net.nz> commit b2aff1236643652a738d64488943d6d1565fcac2 Author: Andrew Bartlett <abart...@samba.org> Date: Fri Aug 11 11:33:17 2017 +1200 ldb_tdb: Prepare to handle rename with GUID index by using ltdb_search_base() This will allow use of a GUID TDB key in the future. While ltdb_search_base() might be marginally slower than tdb_exists(), no allocation is done for the attributes or DN, and renmaes are not a very common operation. This allows a check if the target DN exists even when the direct DN -> key link is broken. Signed-off-by: Andrew Bartlett <abart...@samba.org> Reviewed-by: Garming Sam <garm...@catalyst.net.nz> commit c24df8e43127b627f7cd70bf55f2bd5da8db61df Author: Andrew Bartlett <abart...@samba.org> Date: Tue Aug 15 14:39:08 2017 +1200 ldb_tdb: Implement ltdb_search_base() for a GUID index The GUID index case can not directly use ltdb_key_dn() and tdb_exists() to show that a records exists. Signed-off-by: Andrew Bartlett <abart...@samba.org> Reviewed-by: Garming Sam <garm...@catalyst.net.nz> commit 754329a9d9812032dc16dcbfb427739b26a101d2 Author: Andrew Bartlett <abart...@samba.org> Date: Tue Aug 15 14:18:19 2017 +1200 ldb_tdb: Optionally store a GUID as the index record This allows, when enabled, the index record to contain (say) the objectGUID, not the DN of the record. Signed-off-by: Andrew Bartlett <abart...@samba.org> Reviewed-by: Garming Sam <garm...@catalyst.net.nz> commit b154acb0c7917aaef0f7bc750a4a0ce2690b1c44 Author: Andrew Bartlett <abart...@samba.org> Date: Tue Aug 15 14:13:32 2017 +1200 ldb_tdb: Optionally use a GUID index key in ltdb_dn_list_find_msg() This function is used to find an existing index value and this change allows it to find the value by GUID rather than by DN once the GUID index is enabled. Signed-off-by: Andrew Bartlett <abart...@samba.org> Reviewed-by: Garming Sam <garm...@catalyst.net.nz> commit 9d9ae1c0051d83bd3873d03642ad86002513e395 Author: Andrew Bartlett <abart...@samba.org> Date: Tue Aug 15 13:08:42 2017 +1200 ldb_tdb: Pass the full ldb_message to ldb index funtions This allows the objectGUID, rather than the DN, to be the index key Signed-off-by: Andrew Bartlett <abart...@samba.org> Reviewed-by: Garming Sam <garm...@catalyst.net.nz> commit 4c01ccb47e2ccbb89a75687d421bcf4f3e0357b5 Author: Andrew Bartlett <abart...@samba.org> Date: Fri Aug 25 22:33:15 2017 +1200 ldb_tdb: Delete a successful tdb_store on index add fail in ltdb_add_internal() Signed-off-by: Andrew Bartlett <abart...@samba.org> Reviewed-by: Garming Sam <garm...@catalyst.net.nz> commit e8cdacc509016d9273d63faf334d9f827585c3eb Author: Andrew Bartlett <abart...@samba.org> Date: Thu Aug 17 09:08:34 2017 +1200 ldb_tdb: modify ltdb_delete_noindex() to take a struct ldb_message This will make it easier to delete records with the GUID TDB key Signed-off-by: Andrew Bartlett <abart...@samba.org> Reviewed-by: Garming Sam <garm...@catalyst.net.nz> commit de641db8b76c4c3bbb701167cee13d9b19a9fc64 Author: Andrew Bartlett <abart...@samba.org> Date: Thu Aug 10 17:04:23 2017 +1200 ldb_tdb: Write GUID index values as version 3 Nothing reads these currently, but we should refuse to load a mixed up index in the future Signed-off-by: Andrew Bartlett <abart...@samba.org> Reviewed-by: Garming Sam <garm...@catalyst.net.nz> commit 91e6028c08ddb2ce6bfa8d5fb08a8083a4cc3e29 Author: Andrew Bartlett <abart...@samba.org> Date: Fri Aug 25 15:33:14 2017 +1200 ldb_tdb: Refuse to load a GUID index that is not a multiple of 16 bytes Signed-off-by: Andrew Bartlett <abart...@samba.org> Reviewed-by: Garming Sam <garm...@catalyst.net.nz> commit ad26b4e2923ad2468eef8e48d065b2ca2ca11d36 Author: Andrew Bartlett <abart...@samba.org> Date: Thu Aug 17 17:26:35 2017 +1200 ldb_tdb: Read GUID index values as one packed LDB attribute This packing should be more efficient to read than the ldb_pack format. Signed-off-by: Andrew Bartlett <abart...@samba.org> Reviewed-by: Garming Sam <garm...@catalyst.net.nz> commit f6e953d316cdd5a5fcb4d8b83a71744363272a4a Author: Andrew Bartlett <abart...@samba.org> Date: Thu Aug 10 17:02:14 2017 +1200 ldb_tdb: Store GUID index values in one packed ldb attribute This should make them more memory efficient Signed-off-by: Andrew Bartlett <abart...@samba.org> Reviewed-by: Garming Sam <garm...@catalyst.net.nz> commit 367b7ab7483ac36e1c810ea86bc3a7d99c37ad38 Author: Andrew Bartlett <abart...@samba.org> Date: Fri Sep 1 20:04:43 2017 +1200 ldb_tdb: Move constants into ldb_tdb.h This helps ensure we keep these all in sync. Signed-off-by: Andrew Bartlett <abart...@samba.org> Reviewed-by: Garming Sam <garm...@catalyst.net.nz> commit c17404826e46bb9f5cd7dc36f7c90c961bd2a0f4 Author: Andrew Bartlett <abart...@samba.org> Date: Thu Aug 10 17:00:48 2017 +1200 ldb_tdb: replace strange dn_list_cmp() in index code This replaces dn_list_cmp() with functions that do not attempt to to care about string termination. All index values are case sensitive and correctly length-bound already, even for a DN index so just use a length check and memcmp() Signed-off-by: Andrew Bartlett <abart...@samba.org> Reviewed-by: Garming Sam <garm...@catalyst.net.nz> commit a44d3aedba15f96c84f9d83b0cace7e0d8141f13 Author: Andrew Bartlett <abart...@samba.org> Date: Thu Aug 17 12:42:25 2017 +1200 ldb_tdb: Do not allow a modification of the GUID_index_attribute (objectGUID) This would totally break our index scheme if this could be modified. Signed-off-by: Andrew Bartlett <abart...@samba.org> Reviewed-by: Garming Sam <garm...@catalyst.net.nz> commit 26dd6f63654baa238822c0ec3c51b32b37b0c254 Author: Andrew Bartlett <abart...@samba.org> Date: Thu Aug 10 17:13:48 2017 +1200 ldb_tdb: Add GUID_index_attribute to ltdb_private This will be used to determine if we are in GUID index mode Signed-off-by: Andrew Bartlett <abart...@samba.org> Reviewed-by: Garming Sam <garm...@catalyst.net.nz> commit 3d952157d72b3a4635f3942449c1727c438c97c6 Author: Andrew Bartlett <abart...@samba.org> Date: Thu Aug 10 16:23:33 2017 +1200 ldb_tdb: Provide struct ltdb_private to index routines This will make it easier to switch the GUID index mode on and off Signed-off-by: Andrew Bartlett <abart...@samba.org> Reviewed-by: Garming Sam <garm...@catalyst.net.nz> commit dded01598e5123898d91fbab24827758f0719783 Author: Andrew Bartlett <abart...@samba.org> Date: Wed Aug 16 12:46:57 2017 +1200 ldb_tdb: Use a more complete error mapping in ltdb_search_key() Signed-off-by: Andrew Bartlett <abart...@samba.org> Reviewed-by: Garming Sam <garm...@catalyst.net.nz> commit 6db809583563652edba377cae395fbdca9e3ebb3 Author: Andrew Bartlett <abart...@samba.org> Date: Thu Aug 10 16:06:08 2017 +1200 ldb_tdb: Add ltdb_search_key() This allows us to slowly split out the tdb key in the DB from being the DN Signed-off-by: Andrew Bartlett <abart...@samba.org> Reviewed-by: Garming Sam <garm...@catalyst.net.nz> commit d4c1a600b1901991ffc989859e523155044d3242 Author: Andrew Bartlett <abart...@samba.org> Date: Thu Aug 10 16:01:12 2017 +1200 ldb_tdb: Use ltdb_key_msg() in re_index() This will allow changing to a GUID tdb key in the future Signed-off-by: Andrew Bartlett <abart...@samba.org> Reviewed-by: Garming Sam <garm...@catalyst.net.nz> commit 1b310ad99cf7aa4e9f598b755b342a64632cff63 Author: Andrew Bartlett <abart...@samba.org> Date: Thu Aug 10 14:31:18 2017 +1200 ldb_tdb: provide ldb_key_dn() and ldb_key_msg() This will in time allow us to generate a TDB key from the msg, eg from an objectGUID. Signed-off-by: Andrew Bartlett <abart...@samba.org> Reviewed-by: Garming Sam <garm...@catalyst.net.nz> commit cd8ecb3332b1939a07c3455ec3a66ca3d23bb32d Author: Andrew Bartlett <abart...@samba.org> Date: Tue Aug 15 17:21:28 2017 +1200 ldb_tdb: Add helper function ltdb_search_and_return_base() This avoids an extra DB lookup for the base, when that is the only record we will return, and avoids going into the index code for a base search, as that won't work for special DNs once the GUID index mode is enabled. Signed-off-by: Andrew Bartlett <abart...@samba.org> Reviewed-by: Garming Sam <garm...@catalyst.net.nz> commit d8a22884c1089f35e6df111c1f3f65b0848a1438 Author: Andrew Bartlett <abart...@samba.org> Date: Mon Aug 28 14:55:00 2017 +1200 ldb_tdb: Make ldb_match_message() available to ldb_tdb Signed-off-by: Andrew Bartlett <abart...@samba.org> Reviewed-by: Garming Sam <garm...@catalyst.net.nz> ----------------------------------------------------------------------- Summary of changes: lib/ldb/ABI/{ldb-1.2.3.sigs => ldb-1.3.0.sigs} | 2 + ...yldb-util-1.1.10.sigs => pyldb-util-1.3.0.sigs} | 0 ...-util-1.1.10.sigs => pyldb-util.py3-1.3.0.sigs} | 0 lib/ldb/common/ldb_attributes.c | 14 + lib/ldb/common/ldb_match.c | 22 +- lib/ldb/include/ldb_module.h | 16 + lib/ldb/include/ldb_private.h | 22 + lib/ldb/ldb_tdb/ldb_cache.c | 54 +- lib/ldb/ldb_tdb/ldb_index.c | 1150 ++++++++++++++++---- lib/ldb/ldb_tdb/ldb_search.c | 287 ++++- lib/ldb/ldb_tdb/ldb_tdb.c | 292 ++++- lib/ldb/ldb_tdb/ldb_tdb.h | 58 +- lib/ldb/tests/python/api.py | 426 +++++++- lib/ldb/wscript | 3 +- lib/util/binsearch.h | 22 +- 15 files changed, 2015 insertions(+), 353 deletions(-) copy lib/ldb/ABI/{ldb-1.2.3.sigs => ldb-1.3.0.sigs} (98%) copy lib/ldb/ABI/{pyldb-util-1.1.10.sigs => pyldb-util-1.3.0.sigs} (100%) copy lib/ldb/ABI/{pyldb-util-1.1.10.sigs => pyldb-util.py3-1.3.0.sigs} (100%) Changeset truncated at 500 lines: diff --git a/lib/ldb/ABI/ldb-1.2.3.sigs b/lib/ldb/ABI/ldb-1.3.0.sigs similarity index 98% copy from lib/ldb/ABI/ldb-1.2.3.sigs copy to lib/ldb/ABI/ldb-1.3.0.sigs index 9dc61cd..a31b84e 100644 --- a/lib/ldb/ABI/ldb-1.2.3.sigs +++ b/lib/ldb/ABI/ldb-1.3.0.sigs @@ -119,6 +119,7 @@ ldb_map_init: int (struct ldb_module *, const struct ldb_map_attribute *, const ldb_map_modify: int (struct ldb_module *, struct ldb_request *) ldb_map_rename: int (struct ldb_module *, struct ldb_request *) ldb_map_search: int (struct ldb_module *, struct ldb_request *) +ldb_match_message: int (struct ldb_context *, const struct ldb_message *, const struct ldb_parse_tree *, enum ldb_scope, bool *) ldb_match_msg: int (struct ldb_context *, const struct ldb_message *, const struct ldb_parse_tree *, struct ldb_dn *, enum ldb_scope) ldb_match_msg_error: int (struct ldb_context *, const struct ldb_message *, const struct ldb_parse_tree *, struct ldb_dn *, enum ldb_scope, bool *) ldb_match_msg_objectclass: int (const struct ldb_message *, const char *) @@ -232,6 +233,7 @@ ldb_schema_attribute_fill_with_syntax: int (struct ldb_context *, TALLOC_CTX *, ldb_schema_attribute_remove: void (struct ldb_context *, const char *) ldb_schema_attribute_remove_flagged: void (struct ldb_context *, unsigned int) ldb_schema_attribute_set_override_handler: void (struct ldb_context *, ldb_attribute_handler_override_fn_t, void *) +ldb_schema_set_override_GUID_index: void (struct ldb_context *, const char *, const char *) ldb_schema_set_override_indexlist: void (struct ldb_context *, bool) ldb_search: int (struct ldb_context *, TALLOC_CTX *, struct ldb_result **, struct ldb_dn *, enum ldb_scope, const char * const *, const char *, ...) ldb_search_default_callback: int (struct ldb_request *, struct ldb_reply *) diff --git a/lib/ldb/ABI/pyldb-util-1.1.10.sigs b/lib/ldb/ABI/pyldb-util-1.3.0.sigs similarity index 100% copy from lib/ldb/ABI/pyldb-util-1.1.10.sigs copy to lib/ldb/ABI/pyldb-util-1.3.0.sigs diff --git a/lib/ldb/ABI/pyldb-util-1.1.10.sigs b/lib/ldb/ABI/pyldb-util.py3-1.3.0.sigs similarity index 100% copy from lib/ldb/ABI/pyldb-util-1.1.10.sigs copy to lib/ldb/ABI/pyldb-util.py3-1.3.0.sigs diff --git a/lib/ldb/common/ldb_attributes.c b/lib/ldb/common/ldb_attributes.c index 98ec5a4..32f25fd 100644 --- a/lib/ldb/common/ldb_attributes.c +++ b/lib/ldb/common/ldb_attributes.c @@ -395,3 +395,17 @@ void ldb_schema_set_override_indexlist(struct ldb_context *ldb, ldb->schema.index_handler_override = true; ldb->schema.one_level_indexes = one_level_indexes; } + +/* + * set that the GUID index mode is in operation + * + * The caller must ensure the supplied strings do not go out of + * scope (they are typically constant memory). + */ +void ldb_schema_set_override_GUID_index(struct ldb_context *ldb, + const char *GUID_index_attribute, + const char *GUID_index_dn_component) +{ + ldb->schema.GUID_index_attribute = GUID_index_attribute; + ldb->schema.GUID_index_dn_component = GUID_index_dn_component; +} diff --git a/lib/ldb/common/ldb_match.c b/lib/ldb/common/ldb_match.c index 1415fac..25fe3f9 100644 --- a/lib/ldb/common/ldb_match.c +++ b/lib/ldb/common/ldb_match.c @@ -515,17 +515,18 @@ static int ldb_match_extended(struct ldb_context *ldb, } /* - return 0 if the given parse tree matches the given message. Assumes - the message is in sorted order + Check if a particular message will match the given filter - return 1 if it matches, and 0 if it doesn't match + set *matched to true if it matches, false otherwise + + returns LDB_SUCCESS or an error this is a recursive function, and does short-circuit evaluation */ -static int ldb_match_message(struct ldb_context *ldb, - const struct ldb_message *msg, - const struct ldb_parse_tree *tree, - enum ldb_scope scope, bool *matched) +int ldb_match_message(struct ldb_context *ldb, + const struct ldb_message *msg, + const struct ldb_parse_tree *tree, + enum ldb_scope scope, bool *matched) { unsigned int i; int ret; @@ -587,6 +588,13 @@ static int ldb_match_message(struct ldb_context *ldb, return LDB_ERR_INAPPROPRIATE_MATCHING; } +/* + return 0 if the given parse tree matches the given message. Assumes + the message is in sorted order + + return 1 if it matches, and 0 if it doesn't match +*/ + int ldb_match_msg(struct ldb_context *ldb, const struct ldb_message *msg, const struct ldb_parse_tree *tree, diff --git a/lib/ldb/include/ldb_module.h b/lib/ldb/include/ldb_module.h index 71b4074..ffa6c2e 100644 --- a/lib/ldb/include/ldb_module.h +++ b/lib/ldb/include/ldb_module.h @@ -183,6 +183,22 @@ void ldb_schema_attribute_set_override_handler(struct ldb_context *ldb, void ldb_schema_set_override_indexlist(struct ldb_context *ldb, bool one_level_indexes); +/** + + \param ldb The ldb context + \param GUID_index_attribute The globally attribute (eg objectGUID) + on each entry + \param GUID_index_attribute The DN component matching the + globally attribute on each entry (eg GUID) + + The caller must ensure the supplied strings do not go out of + scope (they are typically constant memory). + +*/ +void ldb_schema_set_override_GUID_index(struct ldb_context *ldb, + const char *GUID_index_attribute, + const char *GUID_index_dn_component); + /* A useful function to build comparison functions with */ int ldb_any_comparison(struct ldb_context *ldb, void *mem_ctx, ldb_attr_handler_t canonicalise_fn, diff --git a/lib/ldb/include/ldb_private.h b/lib/ldb/include/ldb_private.h index ae7ec3c..f999f75 100644 --- a/lib/ldb/include/ldb_private.h +++ b/lib/ldb/include/ldb_private.h @@ -98,6 +98,9 @@ struct ldb_schema { */ bool index_handler_override; bool one_level_indexes; + + const char *GUID_index_attribute; + const char *GUID_index_dn_component; }; /* @@ -287,5 +290,24 @@ int ldb_msg_find_duplicate_val(struct ldb_context *ldb, const struct ldb_message_element *el, struct ldb_val **duplicate, uint32_t options); +/** + Check if a particular message will match the given filter + + \param ldb an ldb context + \param msg the message to be checked + \param tree the filter tree to check against + \param scope the scope to match against + (to avoid matching special DNs except on a base search) + \param matched a pointer to a boolean set true if it matches, + false otherwise + + returns LDB_SUCCESS or an error + + \note this is a recursive function, and does short-circuit evaluation + */ +int ldb_match_message(struct ldb_context *ldb, + const struct ldb_message *msg, + const struct ldb_parse_tree *tree, + enum ldb_scope scope, bool *matched); #endif diff --git a/lib/ldb/ldb_tdb/ldb_cache.c b/lib/ldb/ldb_tdb/ldb_cache.c index f08e073..5b90bd9 100644 --- a/lib/ldb/ldb_tdb/ldb_cache.c +++ b/lib/ldb/ldb_tdb/ldb_cache.c @@ -36,7 +36,7 @@ #define LTDB_FLAG_CASE_INSENSITIVE (1<<0) #define LTDB_FLAG_INTEGER (1<<1) -#define LTDB_FLAG_HIDDEN (1<<2) +#define LTDB_FLAG_UNIQUE_INDEX (1<<2) /* valid attribute flags */ static const struct { @@ -45,7 +45,8 @@ static const struct { } ltdb_valid_attr_flags[] = { { "CASE_INSENSITIVE", LTDB_FLAG_CASE_INSENSITIVE }, { "INTEGER", LTDB_FLAG_INTEGER }, - { "HIDDEN", LTDB_FLAG_HIDDEN }, + { "HIDDEN", 0 }, + { "UNIQUE_INDEX", LTDB_FLAG_UNIQUE_INDEX}, { "NONE", 0 }, { NULL, 0 } }; @@ -150,7 +151,7 @@ static int ltdb_attributes_load(struct ldb_module *module) /* mapping these flags onto ldap 'syntaxes' isn't strictly correct, but its close enough for now */ for (i=0;i<attrs_msg->num_elements;i++) { - unsigned flags; + unsigned flags = 0, attr_flags = 0; const char *syntax; const struct ldb_schema_syntax *s; const struct ldb_schema_attribute *a = @@ -167,17 +168,20 @@ static int ltdb_attributes_load(struct ldb_module *module) attrs_msg->elements[i].name); goto failed; } - switch (flags & ~LTDB_FLAG_HIDDEN) { - case 0: - syntax = LDB_SYNTAX_OCTET_STRING; - break; - case LTDB_FLAG_CASE_INSENSITIVE: + + if (flags & LTDB_FLAG_UNIQUE_INDEX) { + attr_flags = LDB_ATTR_FLAG_UNIQUE_INDEX; + } + flags &= ~LTDB_FLAG_UNIQUE_INDEX; + + /* These are not currently flags, each is exclusive */ + if (flags == LTDB_FLAG_CASE_INSENSITIVE) { syntax = LDB_SYNTAX_DIRECTORY_STRING; - break; - case LTDB_FLAG_INTEGER: + } else if (flags == LTDB_FLAG_INTEGER) { syntax = LDB_SYNTAX_INTEGER; - break; - default: + } else if (flags == 0) { + syntax = LDB_SYNTAX_OCTET_STRING; + } else { ldb_debug(ldb, LDB_DEBUG_ERROR, "Invalid flag combination 0x%x for '%s' " "in @ATTRIBUTES", @@ -194,12 +198,12 @@ static int ltdb_attributes_load(struct ldb_module *module) goto failed; } - flags |= LDB_ATTR_FLAG_ALLOCATED | LDB_ATTR_FLAG_FROM_DB; + attr_flags |= LDB_ATTR_FLAG_ALLOCATED | LDB_ATTR_FLAG_FROM_DB; r = ldb_schema_attribute_fill_with_syntax(ldb, attrs, attrs_msg->elements[i].name, - flags, s, + attr_flags, s, &attrs[num_loaded_attrs + ldb->schema.num_attributes]); if (r != 0) { goto failed; @@ -243,6 +247,10 @@ static int ltdb_index_load(struct ldb_module *module, */ ltdb->cache->attribute_indexes = true; ltdb->cache->one_level_indexes = ldb->schema.one_level_indexes; + ltdb->cache->GUID_index_attribute + = ldb->schema.GUID_index_attribute; + ltdb->cache->GUID_index_dn_component + = ldb->schema.GUID_index_dn_component; return 0; } @@ -276,6 +284,12 @@ static int ltdb_index_load(struct ldb_module *module, if (ldb_msg_find_element(ltdb->cache->indexlist, LTDB_IDXATTR) != NULL) { ltdb->cache->attribute_indexes = true; } + ltdb->cache->GUID_index_attribute + = ldb_msg_find_attr_as_string(ltdb->cache->indexlist, + LTDB_IDXGUID, NULL); + ltdb->cache->GUID_index_dn_component + = ldb_msg_find_attr_as_string(ltdb->cache->indexlist, + LTDB_IDX_DN_GUID, NULL); return 0; } @@ -371,6 +385,7 @@ int ltdb_cache_load(struct ldb_module *module) struct ldb_dn *baseinfo_dn = NULL, *options_dn = NULL; uint64_t seq; struct ldb_message *baseinfo = NULL, *options = NULL; + const struct ldb_schema_attribute *a; int r; ldb = ldb_module_get_ctx(module); @@ -474,6 +489,17 @@ int ltdb_cache_load(struct ldb_module *module) goto failed; } + ltdb->GUID_index_syntax = NULL; + if (ltdb->cache->GUID_index_attribute != NULL) { + /* + * Now the attributes are loaded, set the guid_index_syntax. + * This can't fail, it will return a default at worst + */ + a = ldb_schema_attribute_by_name(ldb, + ltdb->cache->GUID_index_attribute); + ltdb->GUID_index_syntax = a->syntax; + } + done: talloc_free(options); talloc_free(baseinfo); diff --git a/lib/ldb/ldb_tdb/ldb_index.c b/lib/ldb/ldb_tdb/ldb_index.c index 7b36ac4..c71e866 100644 --- a/lib/ldb/ldb_tdb/ldb_index.c +++ b/lib/ldb/ldb_tdb/ldb_index.c @@ -31,8 +31,122 @@ * Author: Andrew Tridgell */ +/* + +LDB Index design and choice of TDB key: +======================================= + +LDB has index records held as LDB objects with a special record like: + +dn: @INDEX:attr:value + +value may be base64 encoded, if it is deemed not printable: + +dn: @INDEX:attr::base64-value + +In each record, there is two possible formats: + +The original format is: +----------------------- + +dn: @INDEX:NAME:DNSUPDATEPROXY +@IDXVERSION: 2 +@IDX: CN=DnsUpdateProxy,CN=Users,DC=addom,DC=samba,DC=example,DC=com + +In this format, @IDX is multi-valued, one entry for each match + +The corrosponding entry is stored in a TDB record with key: + +DN=CN=DNSUPDATEPROXY,CN=USERS,DC=ADDOM,DC=SAMBA,DC=EXAMPLE,DC=COM + +(This allows a scope BASE search to directly find the record via +a simple casefold of the DN). + +The original mixed-case DN is stored in the entry iself. + + +The new 'GUID index' format is: +------------------------------- + +dn: @INDEX:NAME:DNSUPDATEPROXY +@IDXVERSION: 3 +@IDX: <binary GUID>[<binary GUID>[...]] + +The binary guid is 16 bytes, as bytes and not expanded as hexidecimal +or pretty-printed. The GUID is chosen from the message to be stored +by the @IDXGUID attribute on @INDEXLIST. + +If there are multiple values the @IDX value simply becomes longer, +in multiples of 16. + +The corrosponding entry is stored in a TDB record with key: + +GUID=<binary GUID> + +This allows a very quick translation between the fixed-length index +values and the TDB key, while seperating entries from other data +in the TDB, should they be unlucky enough to start with the bytes of +the 'DN=' prefix. + +Additionally, this allows a scope BASE search to directly find the +record via a simple match on a GUID= extended DN, controlled via +@IDX_DN_GUID on @INDEXLIST + +Exception for special @ DNs: + +@BASEINFO, @INDEXLIST and all other special DNs are stored as per the +original format, as they are never referenced in an index and are used +to bootstrap the database. + + +Control points for choice of index mode +--------------------------------------- + +The choice of index and TDB key mode is made based (for example, from +Samba) on entries in the @INDEXLIST DN: + +dn: @INDEXLIST +@IDXGUID: objectGUID +@IDX_DN_GUID: GUID + +By default, the original DN format is used. + + +Control points for choosing indexed attributes +---------------------------------------------- + +@IDXATTR controls if an attribute is indexed + +dn: @INDEXLIST +@IDXATTR: samAccountName +@IDXATTR: nETBIOSName + + +C Override functions +-------------------- + +void ldb_schema_set_override_GUID_index(struct ldb_context *ldb, + const char *GUID_index_attribute, + const char *GUID_index_dn_component) + +This is used, particularly in combination with the below, instead of +the @IDXGUID and @IDX_DN_GUID values in @INDEXLIST. + +void ldb_schema_set_override_indexlist(struct ldb_context *ldb, + bool one_level_indexes); +void ldb_schema_attribute_set_override_handler(struct ldb_context *ldb, + ldb_attribute_handler_override_fn_t override, + void *private_data); + +When the above two functions are called in combination, the @INDEXLIST +values are not read from the DB, so +ldb_schema_set_override_GUID_index() must be called. + +*/ + #include "ldb_tdb.h" #include "ldb_private.h" +#include "lib/util/binsearch.h" struct dn_list { unsigned int count; @@ -44,11 +158,24 @@ struct ltdb_idxptr { int error; }; +static int ltdb_write_index_dn_guid(struct ldb_module *module, + const struct ldb_message *msg, + int add); +static int ltdb_index_dn_base_dn(struct ldb_module *module, + struct ltdb_private *ltdb, + struct ldb_dn *base_dn, + struct dn_list *dn_list); + +static void ltdb_dn_list_sort(struct ltdb_private *ltdb, + struct dn_list *list); + /* we put a @IDXVERSION attribute on index entries. This allows us to tell if it was written by an older version */ #define LTDB_INDEXING_VERSION 2 +#define LTDB_GUID_INDEXING_VERSION 3 + /* enable the idxptr mode when transactions start */ int ltdb_index_transaction_start(struct ldb_module *module) { @@ -61,45 +188,99 @@ int ltdb_index_transaction_start(struct ldb_module *module) return LDB_SUCCESS; } -/* compare two DN entries in a dn_list. Take account of possible - * differences in string termination */ -static int dn_list_cmp(const struct ldb_val *v1, const struct ldb_val *v2) +/* + see if two ldb_val structures contain exactly the same data + return -1 or 1 for a mismatch, 0 for match +*/ +static int ldb_val_equal_exact_for_qsort(const struct ldb_val *v1, + const struct ldb_val *v2) { - if (v1->length > v2->length && v1->data[v2->length] != 0) { + if (v1->length > v2->length) { return -1; } - if (v1->length < v2->length && v2->data[v1->length] != 0) { + if (v1->length < v2->length) { return 1; } - return strncmp((char *)v1->data, (char *)v2->data, v1->length); + return memcmp(v1->data, v2->data, v1->length); +} + +/* + see if two ldb_val structures contain exactly the same data + return -1 or 1 for a mismatch, 0 for match +*/ +static int ldb_val_equal_exact_ordered(const struct ldb_val v1, + const struct ldb_val *v2) +{ + if (v1.length > v2->length) { + return -1; + } + if (v1.length < v2->length) { + return 1; + } + return memcmp(v1.data, v2->data, v1.length); } /* find a entry in a dn_list, using a ldb_val. Uses a case sensitive - comparison with the dn returns -1 if not found + binary-safe comparison for the 'dn' returns -1 if not found + + This is therefore safe when the value is a GUID in the future */ -static int ltdb_dn_list_find_val(const struct dn_list *list, const struct ldb_val *v) +static int ltdb_dn_list_find_val(struct ltdb_private *ltdb, + const struct dn_list *list, + const struct ldb_val *v) { unsigned int i; - for (i=0; i<list->count; i++) { - if (dn_list_cmp(&list->dn[i], v) == 0) { -- Samba Shared Repository