The branch, master has been updated via 06f87f14b37 README.Coding: target Python 3.6+ via 14210c248a9 python tests: drop python 2.6 compatibility functions via 9148f38c203 ndr: avoid excessive reallocing in pull_string_array via 9bf331b46a7 ndr: maintain proper talloc tree in pull_string_array via 326bc84c0d0 oss-fuzz: use uninstrumented dynamic python from 698d20d39d4 smbd: remove get_current_vuid()
https://git.samba.org/?p=samba.git;a=shortlog;h=master - Log ----------------------------------------------------------------- commit 06f87f14b37e08e6f78892b2f768282a742359df Author: Douglas Bagnall <douglas.bagn...@catalyst.net.nz> Date: Wed Jul 29 15:35:12 2020 +1200 README.Coding: target Python 3.6+ Signed-off-by: Douglas Bagnall <douglas.bagn...@catalyst.net.nz> Reviewed-by: Noel Power <npo...@samba.org> Reviewed-by: Andrew Bartlett <abart...@samba.org> Autobuild-User(master): Andrew Bartlett <abart...@samba.org> Autobuild-Date(master): Mon Aug 3 04:13:37 UTC 2020 on sn-devel-184 commit 14210c248a9dd313671dcddf443096c17deb052a Author: Douglas Bagnall <douglas.bagn...@catalyst.net.nz> Date: Sat Jul 4 14:28:40 2020 +1200 python tests: drop python 2.6 compatibility functions Signed-off-by: Douglas Bagnall <douglas.bagn...@catalyst.net.nz> Reviewed-by: Noel Power <npo...@samba.org> Reviewed-by: Andrew Bartlett <abart...@samba.org> commit 9148f38c203c3481a43ef6d39ea9313dfa1c1bea Author: Douglas Bagnall <douglas.bagn...@catalyst.net.nz> Date: Thu Jul 30 12:06:10 2020 +1200 ndr: avoid excessive reallocing in pull_string_array Before, talloc_realloc() was being called n times for an array of length n. This could be very expensive on long string arrays since it is reasonable to assume each realloc moves O(n) bytes. This addresses at least one OSS-Fuzz bug, making a timing out test case 100 times faster. Credit to OSS-Fuzz. REF: https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=19706 Signed-off-by: Douglas Bagnall <douglas.bagn...@catalyst.net.nz> Reviewed-by: Noel Power <npo...@samba.org> Reviewed-by: Andrew Bartlett <abart...@samba.org> commit 9bf331b46a70189f2f63a5223a31eae64a9854db Author: Douglas Bagnall <douglas.bagn...@catalyst.net.nz> Date: Thu Jul 30 10:46:17 2020 +1200 ndr: maintain proper talloc tree in pull_string_array We don't want to leave other parts of the ndr struct hanging off this string array just because LIBNDR_FLAG_REMAINING is used. Signed-off-by: Douglas Bagnall <douglas.bagn...@catalyst.net.nz> Reviewed-by: Noel Power <npo...@samba.org> Reviewed-by: Andrew Bartlett <abart...@samba.org> commit 326bc84c0d0f83b4429e78b6c974204116233330 Author: Douglas Bagnall <douglas.bagn...@catalyst.net.nz> Date: Fri Jul 31 15:36:16 2020 +1200 oss-fuzz: use uninstrumented dynamic python We can't link to the instrumented statically built Python, so instead we use the system Python in the docker image. REF: https://github.com/google/oss-fuzz/issues/4223 REF: https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=22618 BUG: https://bugzilla.samba.org/show_bug.cgi?id=14451 Signed-off-by: Douglas Bagnall <douglas.bagn...@catalyst.net.nz> Reviewed-by: Andrew Bartlett <abart...@samba.org> ----------------------------------------------------------------------- Summary of changes: README.Coding.md | 5 +- lib/fuzzing/oss-fuzz/build_samba.sh | 6 ++ librpc/ndr/ndr_string.c | 73 +++++++++++++++++++++---- python/samba/tests/__init__.py | 106 ------------------------------------ 4 files changed, 69 insertions(+), 121 deletions(-) Changeset truncated at 500 lines: diff --git a/README.Coding.md b/README.Coding.md index 914e3573ad4..b87580f5f85 100644 --- a/README.Coding.md +++ b/README.Coding.md @@ -17,9 +17,8 @@ what most Samba developers use already anyways, with a few exceptions as mentioned below. The coding style for Python code is documented in -[PEP8](https://www.python.org/dev/peps/pep-0008/). New Python code should be compatible -with Python 2.6, 2.7, and Python 3.4 onwards. This means using Python 3 syntax -with the appropriate `from __future__` imports. +[PEP8](https://www.python.org/dev/peps/pep-0008/). New Python code +should be compatible with Python 3.6 onwards. But to save you the trouble of reading the Linux kernel style guide, here are the highlights. diff --git a/lib/fuzzing/oss-fuzz/build_samba.sh b/lib/fuzzing/oss-fuzz/build_samba.sh index 93039e4dbe0..5d3bc3ff6c3 100755 --- a/lib/fuzzing/oss-fuzz/build_samba.sh +++ b/lib/fuzzing/oss-fuzz/build_samba.sh @@ -21,6 +21,12 @@ export CFLAGS LD="$CXX" export LD +# Use the system Python, not the OSS-Fuzz provided statically linked +# and instrumented Python, because we can't statically link. + +PYTHON=/usr/bin/python3 +export PYTHON + # $SANITIZER is provided by the oss-fuzz "compile" command # # We need to add the waf configure option as otherwise when we also diff --git a/librpc/ndr/ndr_string.c b/librpc/ndr/ndr_string.c index 25f211b2ad3..bddab9edd51 100644 --- a/librpc/ndr/ndr_string.c +++ b/librpc/ndr/ndr_string.c @@ -348,6 +348,44 @@ _PUBLIC_ uint32_t ndr_size_string(int ret, const char * const* string, int flags return ret+strlen(*string)+1; } +static uint32_t guess_string_array_size(struct ndr_pull *ndr, int ndr_flags) +{ + /* + * Here we could do something clever like count the number of zeros in + * the ndr data, but it is probably sufficient to pick a lowish number + * (compared to the overhead of the talloc header) and let the + * expontential resizing deal with longer arrays. + */ + return 5; +} + +static enum ndr_err_code extend_string_array(struct ndr_pull *ndr, + const char ***_a, + uint32_t *count) +{ + const char **a = *_a; + uint32_t inc = *count / 4 + 3; + uint32_t alloc_size = *count + inc; + + if (alloc_size < *count) { + /* overflow ! */ + return NDR_ERR_ALLOC; + } + /* + * We allocate and zero two more bytes than we report back, so that + * the string array will always be NULL terminated. + */ + a = talloc_realloc(ndr->current_mem_ctx, a, + const char *, + alloc_size); + NDR_ERR_HAVE_NO_MEMORY(a); + + memset(a + *count, 0, inc * sizeof(a[0])); + *_a = a; + *count = alloc_size - 2; + return NDR_ERR_SUCCESS; +} + /** pull a general string array from the wire */ @@ -357,14 +395,19 @@ _PUBLIC_ enum ndr_err_code ndr_pull_string_array(struct ndr_pull *ndr, int ndr_f uint32_t count; unsigned flags = ndr->flags; unsigned saved_flags = ndr->flags; + uint32_t alloc_size; if (!(ndr_flags & NDR_SCALARS)) { return NDR_ERR_SUCCESS; } + alloc_size = guess_string_array_size(ndr, ndr_flags); + a = talloc_zero_array(ndr->current_mem_ctx, const char *, alloc_size + 2); + NDR_ERR_HAVE_NO_MEMORY(a); + switch (flags & (LIBNDR_FLAG_STR_NULLTERM|LIBNDR_FLAG_STR_NOTERM)) { case LIBNDR_FLAG_STR_NULLTERM: - /* + /* * here the strings are null terminated * but also the array is null terminated if LIBNDR_FLAG_REMAINING * is specified @@ -372,20 +415,21 @@ _PUBLIC_ enum ndr_err_code ndr_pull_string_array(struct ndr_pull *ndr, int ndr_f for (count = 0;; count++) { TALLOC_CTX *tmp_ctx; const char *s = NULL; - a = talloc_realloc(ndr->current_mem_ctx, a, const char *, count + 2); - NDR_ERR_HAVE_NO_MEMORY(a); - a[count] = NULL; - a[count+1] = NULL; + if (count == alloc_size) { + NDR_CHECK(extend_string_array(ndr, + &a, + &alloc_size)); + } tmp_ctx = ndr->current_mem_ctx; ndr->current_mem_ctx = a; NDR_CHECK(ndr_pull_string(ndr, ndr_flags, &s)); + ndr->current_mem_ctx = tmp_ctx; if ((ndr->data_size - ndr->offset) == 0 && ndr->flags & LIBNDR_FLAG_REMAINING) { a[count] = s; break; } - ndr->current_mem_ctx = tmp_ctx; if (strcmp("", s)==0) { a[count] = NULL; break; @@ -393,6 +437,8 @@ _PUBLIC_ enum ndr_err_code ndr_pull_string_array(struct ndr_pull *ndr, int ndr_f a[count] = s; } } + a = talloc_realloc(ndr->current_mem_ctx, a, const char *, count + 1); + NDR_ERR_HAVE_NO_MEMORY(a); *_a =a; break; @@ -404,7 +450,7 @@ _PUBLIC_ enum ndr_err_code ndr_pull_string_array(struct ndr_pull *ndr, int ndr_f } /* * here the strings are not null terminated - * but serarated by a null terminator + * but separated by a null terminator * * which means the same as: * Every string is null terminated exept the last @@ -422,10 +468,11 @@ _PUBLIC_ enum ndr_err_code ndr_pull_string_array(struct ndr_pull *ndr, int ndr_f for (count = 0; ((ndr->data_size - ndr->offset) > 0); count++) { TALLOC_CTX *tmp_ctx; const char *s = NULL; - a = talloc_realloc(ndr->current_mem_ctx, a, const char *, count + 2); - NDR_ERR_HAVE_NO_MEMORY(a); - a[count] = NULL; - a[count+1] = NULL; + if (count == alloc_size) { + NDR_CHECK(extend_string_array(ndr, + &a, + &alloc_size)); + } tmp_ctx = ndr->current_mem_ctx; ndr->current_mem_ctx = a; @@ -434,7 +481,9 @@ _PUBLIC_ enum ndr_err_code ndr_pull_string_array(struct ndr_pull *ndr, int ndr_f a[count] = s; } - *_a =a; + a = talloc_realloc(ndr->current_mem_ctx, a, const char *, count + 1); + NDR_ERR_HAVE_NO_MEMORY(a); + *_a = a; break; default: diff --git a/python/samba/tests/__init__.py b/python/samba/tests/__init__.py index e7979a7e3ab..768231384d8 100644 --- a/python/samba/tests/__init__.py +++ b/python/samba/tests/__init__.py @@ -141,112 +141,6 @@ class TestCase(unittest.TestCase): c.set_kerberos_state(kerberos_state) return c - # These functions didn't exist before Python2.7: - if sys.version_info < (2, 7): - import warnings - - def skipTest(self, reason): - raise SkipTest(reason) - - def assertIn(self, member, container, msg=None): - self.assertTrue(member in container, msg) - - def assertIs(self, a, b, msg=None): - self.assertTrue(a is b, msg) - - def assertIsNot(self, a, b, msg=None): - self.assertTrue(a is not b, msg) - - def assertIsNotNone(self, a, msg=None): - self.assertTrue(a is not None) - - def assertIsInstance(self, a, b, msg=None): - self.assertTrue(isinstance(a, b), msg) - - def assertIsNone(self, a, msg=None): - self.assertTrue(a is None, msg) - - def assertGreater(self, a, b, msg=None): - self.assertTrue(a > b, msg) - - def assertGreaterEqual(self, a, b, msg=None): - self.assertTrue(a >= b, msg) - - def assertLess(self, a, b, msg=None): - self.assertTrue(a < b, msg) - - def assertLessEqual(self, a, b, msg=None): - self.assertTrue(a <= b, msg) - - def addCleanup(self, fn, *args, **kwargs): - self._cleanups = getattr(self, "_cleanups", []) + [ - (fn, args, kwargs)] - - def assertRegexpMatches(self, text, regex, msg=None): - # PY3 note: Python 3 will never see this, but we use - # text_type for the benefit of linters. - if isinstance(regex, (str, text_type)): - regex = re.compile(regex) - if not regex.search(text): - self.fail(msg) - - def _addSkip(self, result, reason): - addSkip = getattr(result, 'addSkip', None) - if addSkip is not None: - addSkip(self, reason) - else: - warnings.warn("TestResult has no addSkip method, skips not reported", - RuntimeWarning, 2) - result.addSuccess(self) - - def run(self, result=None): - if result is None: - result = self.defaultTestResult() - result.startTest(self) - testMethod = getattr(self, self._testMethodName) - try: - try: - self.setUp() - except SkipTest as e: - self._addSkip(result, str(e)) - return - except KeyboardInterrupt: - raise - except: - result.addError(self, self._exc_info()) - return - - ok = False - try: - testMethod() - ok = True - except SkipTest as e: - self._addSkip(result, str(e)) - return - except self.failureException: - result.addFailure(self, self._exc_info()) - except KeyboardInterrupt: - raise - except: - result.addError(self, self._exc_info()) - - try: - self.tearDown() - except SkipTest as e: - self._addSkip(result, str(e)) - except KeyboardInterrupt: - raise - except: - result.addError(self, self._exc_info()) - ok = False - - for (fn, args, kwargs) in reversed(getattr(self, "_cleanups", [])): - fn(*args, **kwargs) - if ok: - result.addSuccess(self) - finally: - result.stopTest(self) - def assertStringsEqual(self, a, b, msg=None, strip=False): """Assert equality between two strings and highlight any differences. If strip is true, leading and trailing whitespace is ignored.""" -- Samba Shared Repository