On Sun, Oct 16, 2016 at 06:05:47PM +0200, Christian Boltz wrote: > Hello, > > this patch adds TestLogToProfile to test-libapparmor-test_multi.py which > "translates" the test_multi log sniplets to a profile, and checks if it > matches the expected profile. > > The expected profile for one log event will obviously contain only one > rule, and gets added as *.profile to the test_multi directory. > > This patch includes 33 test_multi profiles - which means 83 more need to > be created Whenever you have some time, add one or two! (Please write > those test_multi profiles manually, without using the tools.) > > I know some parts of the test code looks complicated. Unfortunately this > is how things work - compare it with do_logprof_pass() in aa.py... > > While on it, set tests = 'invalid' which ensures a failure in case > parse_test_profiles() doesn't set the tests array, and move printing > the test name out of parse_test_profiles() to avoid printing it twice. > > A nice side effect of this patch is increased test coverage: > - 30% -> 40% in aa.py (= 250 more lines) > - 52% -> 78% in aamode.py (= 23 more lines) > - 26% -> 68% in logparser.py (= 120 more lines) > - total coverage increases from 57% to 62% > > > Note: to review this patch, you'll need to compare the *.profile files > to the *.in files. > > > > [ 01-test-log-to-profile.diff ]
This is cool :) nice.
Acked-by: Seth Arnold <seth.arn...@canonical.com>
Thanks
>
> === added file 'libraries/libapparmor/testsuite/test_multi/testcase02.profile'
> --- libraries/libapparmor/testsuite/test_multi/testcase02.profile
> 1970-01-01 00:00:00 +0000
> +++ libraries/libapparmor/testsuite/test_multi/testcase02.profile
> 2016-10-16 15:19:28 +0000
> @@ -0,0 +1,4 @@
> +/bin/ping {
> + capability setuid,
> +
> +}
>
> === added file 'libraries/libapparmor/testsuite/test_multi/testcase03.profile'
> --- libraries/libapparmor/testsuite/test_multi/testcase03.profile
> 1970-01-01 00:00:00 +0000
> +++ libraries/libapparmor/testsuite/test_multi/testcase03.profile
> 2016-10-16 15:19:28 +0000
> @@ -0,0 +1,4 @@
> +/bin/ping {
> + capability setuid,
> +
> +}
>
> === added file 'libraries/libapparmor/testsuite/test_multi/testcase04.profile'
> --- libraries/libapparmor/testsuite/test_multi/testcase04.profile
> 1970-01-01 00:00:00 +0000
> +++ libraries/libapparmor/testsuite/test_multi/testcase04.profile
> 2016-10-16 15:19:28 +0000
> @@ -0,0 +1,4 @@
> +/bin/ping {
> + capability setuid,
> +
> +}
>
> === added file 'libraries/libapparmor/testsuite/test_multi/testcase05.profile'
> --- libraries/libapparmor/testsuite/test_multi/testcase05.profile
> 1970-01-01 00:00:00 +0000
> +++ libraries/libapparmor/testsuite/test_multi/testcase05.profile
> 2016-10-16 15:19:28 +0000
> @@ -0,0 +1,4 @@
> +/bin/ping {
> + capability setuid,
> +
> +}
>
> === added file 'libraries/libapparmor/testsuite/test_multi/testcase06.profile'
> --- libraries/libapparmor/testsuite/test_multi/testcase06.profile
> 1970-01-01 00:00:00 +0000
> +++ libraries/libapparmor/testsuite/test_multi/testcase06.profile
> 2016-10-16 15:19:28 +0000
> @@ -0,0 +1,4 @@
> +/bin/ping {
> + capability setuid,
> +
> +}
>
> === added file 'libraries/libapparmor/testsuite/test_multi/testcase07.profile'
> --- libraries/libapparmor/testsuite/test_multi/testcase07.profile
> 1970-01-01 00:00:00 +0000
> +++ libraries/libapparmor/testsuite/test_multi/testcase07.profile
> 2016-10-16 15:19:28 +0000
> @@ -0,0 +1,4 @@
> +/bin/ping {
> + capability setuid,
> +
> +}
>
> === added file 'libraries/libapparmor/testsuite/test_multi/testcase08.profile'
> --- libraries/libapparmor/testsuite/test_multi/testcase08.profile
> 1970-01-01 00:00:00 +0000
> +++ libraries/libapparmor/testsuite/test_multi/testcase08.profile
> 2016-10-16 15:19:28 +0000
> @@ -0,0 +1,4 @@
> +/bin/ping {
> + capability setuid,
> +
> +}
>
> === added file 'libraries/libapparmor/testsuite/test_multi/testcase09.profile'
> --- libraries/libapparmor/testsuite/test_multi/testcase09.profile
> 1970-01-01 00:00:00 +0000
> +++ libraries/libapparmor/testsuite/test_multi/testcase09.profile
> 2016-10-16 15:19:28 +0000
> @@ -0,0 +1,4 @@
> +/bin/ping {
> + capability setuid,
> +
> +}
>
> === added file 'libraries/libapparmor/testsuite/test_multi/testcase10.profile'
> --- libraries/libapparmor/testsuite/test_multi/testcase10.profile
> 1970-01-01 00:00:00 +0000
> +++ libraries/libapparmor/testsuite/test_multi/testcase10.profile
> 2016-10-16 15:19:28 +0000
> @@ -0,0 +1,4 @@
> +/bin/ping {
> + capability setuid,
> +
> +}
>
> === added file 'libraries/libapparmor/testsuite/test_multi/testcase11.profile'
> --- libraries/libapparmor/testsuite/test_multi/testcase11.profile
> 1970-01-01 00:00:00 +0000
> +++ libraries/libapparmor/testsuite/test_multi/testcase11.profile
> 2016-10-16 15:19:28 +0000
> @@ -0,0 +1,4 @@
> +/bin/ping {
> + capability setuid,
> +
> +}
>
> === added file 'libraries/libapparmor/testsuite/test_multi/testcase14.profile'
> --- libraries/libapparmor/testsuite/test_multi/testcase14.profile
> 1970-01-01 00:00:00 +0000
> +++ libraries/libapparmor/testsuite/test_multi/testcase14.profile
> 2016-10-16 15:19:28 +0000
> @@ -0,0 +1,4 @@
> +/usr/sbin/nmbd {
> + /var/run/samba/unexpected.tdb k,
> +
> +}
>
> === added file 'libraries/libapparmor/testsuite/test_multi/testcase15.profile'
> --- libraries/libapparmor/testsuite/test_multi/testcase15.profile
> 1970-01-01 00:00:00 +0000
> +++ libraries/libapparmor/testsuite/test_multi/testcase15.profile
> 2016-10-16 15:19:28 +0000
> @@ -0,0 +1,4 @@
> +/usr/sbin/nmbd {
> + /var/run/samba/unexpected.tdb k,
> +
> +}
>
> === added file 'libraries/libapparmor/testsuite/test_multi/testcase16.profile'
> --- libraries/libapparmor/testsuite/test_multi/testcase16.profile
> 1970-01-01 00:00:00 +0000
> +++ libraries/libapparmor/testsuite/test_multi/testcase16.profile
> 2016-10-16 15:19:28 +0000
> @@ -0,0 +1,4 @@
> +/usr/sbin/nmbd {
> + /var/run/samba/unexpected.tdb k,
> +
> +}
>
> === added file 'libraries/libapparmor/testsuite/test_multi/testcase17.profile'
> --- libraries/libapparmor/testsuite/test_multi/testcase17.profile
> 1970-01-01 00:00:00 +0000
> +++ libraries/libapparmor/testsuite/test_multi/testcase17.profile
> 2016-10-16 15:19:28 +0000
> @@ -0,0 +1,4 @@
> +/usr/sbin/nmbd {
> + /var/run/samba/unexpected.tdb k,
> +
> +}
>
> === added file 'libraries/libapparmor/testsuite/test_multi/testcase21.profile'
> --- libraries/libapparmor/testsuite/test_multi/testcase21.profile
> 1970-01-01 00:00:00 +0000
> +++ libraries/libapparmor/testsuite/test_multi/testcase21.profile
> 2016-10-16 15:19:28 +0000
> @@ -0,0 +1,4 @@
> +/home/steve/tmp/sh {
> + /etc/passwd r,
> +
> +}
>
> === added file 'libraries/libapparmor/testsuite/test_multi/testcase22.profile'
> --- libraries/libapparmor/testsuite/test_multi/testcase22.profile
> 1970-01-01 00:00:00 +0000
> +++ libraries/libapparmor/testsuite/test_multi/testcase22.profile
> 2016-10-16 15:19:28 +0000
> @@ -0,0 +1,4 @@
> +/home/steve/tmp/sh {
> + "/tmp/does not exist" w,
> +
> +}
>
> === added file 'libraries/libapparmor/testsuite/test_multi/testcase24.profile'
> --- libraries/libapparmor/testsuite/test_multi/testcase24.profile
> 1970-01-01 00:00:00 +0000
> +++ libraries/libapparmor/testsuite/test_multi/testcase24.profile
> 2016-10-16 15:19:28 +0000
> @@ -0,0 +1,4 @@
> +/home/steve/aa-regression-tests/changehat_wrapper//net_raw {
> + network packet raw,
> +
> +}
>
> === added file 'libraries/libapparmor/testsuite/test_multi/testcase33.profile'
> --- libraries/libapparmor/testsuite/test_multi/testcase33.profile
> 1970-01-01 00:00:00 +0000
> +++ libraries/libapparmor/testsuite/test_multi/testcase33.profile
> 2016-10-16 15:19:28 +0000
> @@ -0,0 +1,3 @@
> +/home/steve/aa-regression-tests/changehat_wrapper//net_raw {
> + network packet raw,
> +}
>
> === added file
> 'libraries/libapparmor/testsuite/test_multi/testcase_dbus_01.profile'
> --- libraries/libapparmor/testsuite/test_multi/testcase_dbus_01.profile
> 1970-01-01 00:00:00 +0000
> +++ libraries/libapparmor/testsuite/test_multi/testcase_dbus_01.profile
> 2016-10-16 15:19:28 +0000
> @@ -0,0 +1,4 @@
> +/tmp/apparmor-2.8.0/tests/regression/apparmor/dbus_service {
> + dbus send bus=session path=/org/freedesktop/DBus
> interface=org.freedesktop.DBus member=Hello peer=(label=unconfined),
> +
> +}
>
> === added file
> 'libraries/libapparmor/testsuite/test_multi/testcase_dbus_02.profile'
> --- libraries/libapparmor/testsuite/test_multi/testcase_dbus_02.profile
> 1970-01-01 00:00:00 +0000
> +++ libraries/libapparmor/testsuite/test_multi/testcase_dbus_02.profile
> 2016-10-16 15:19:28 +0000
> @@ -0,0 +1,4 @@
> +/tmp/apparmor-2.8.0/tests/regression/apparmor/dbus_service {
> + dbus bind bus=session name=com.apparmor.Test,
> +
> +}
>
> === added file
> 'libraries/libapparmor/testsuite/test_multi/testcase_dbus_03.profile'
> --- libraries/libapparmor/testsuite/test_multi/testcase_dbus_03.profile
> 1970-01-01 00:00:00 +0000
> +++ libraries/libapparmor/testsuite/test_multi/testcase_dbus_03.profile
> 2016-10-16 15:19:28 +0000
> @@ -0,0 +1,4 @@
> +/tmp/apparmor-2.8.0/tests/regression/apparmor/dbus_service {
> + dbus receive bus=session path=/com/apparmor/Test
> interface=com.apparmor.Test member=Signal peer=(label=unconfined),
> +
> +}
>
> === added file
> 'libraries/libapparmor/testsuite/test_multi/testcase_dbus_04.profile'
> --- libraries/libapparmor/testsuite/test_multi/testcase_dbus_04.profile
> 1970-01-01 00:00:00 +0000
> +++ libraries/libapparmor/testsuite/test_multi/testcase_dbus_04.profile
> 2016-10-16 15:19:28 +0000
> @@ -0,0 +1,4 @@
> +/tmp/apparmor-2.8.0/tests/regression/apparmor/dbus_service {
> + dbus send bus=system path=/org/freedesktop/DBus
> interface=org.freedesktop.DBus member=Hello peer=(label=unconfined),
> +
> +}
>
> === added file
> 'libraries/libapparmor/testsuite/test_multi/testcase_dbus_05.profile'
> --- libraries/libapparmor/testsuite/test_multi/testcase_dbus_05.profile
> 1970-01-01 00:00:00 +0000
> +++ libraries/libapparmor/testsuite/test_multi/testcase_dbus_05.profile
> 2016-10-16 15:19:28 +0000
> @@ -0,0 +1,4 @@
> +/tmp/apparmor-2.8.0/tests/regression/apparmor/dbus_service {
> + dbus bind bus=system name=org.freedesktop.nm_dhcp_client,
> +
> +}
>
> === added file
> 'libraries/libapparmor/testsuite/test_multi/testcase_dbus_06.profile'
> --- libraries/libapparmor/testsuite/test_multi/testcase_dbus_06.profile
> 1970-01-01 00:00:00 +0000
> +++ libraries/libapparmor/testsuite/test_multi/testcase_dbus_06.profile
> 2016-10-16 15:19:28 +0000
> @@ -0,0 +1,4 @@
> +/tmp/apparmor-2.8.0/tests/regression/apparmor/dbus_service {
> + dbus receive bus=system path=/org/freedesktop/nm_dhcp_client
> interface=org.freedesktop.nm_dhcp_client member=Signal
> peer=(label=unconfined),
> +
> +}
>
> === added file
> 'libraries/libapparmor/testsuite/test_multi/testcase_dmesg_capability.profile'
> ---
> libraries/libapparmor/testsuite/test_multi/testcase_dmesg_capability.profile
> 1970-01-01 00:00:00 +0000
> +++
> libraries/libapparmor/testsuite/test_multi/testcase_dmesg_capability.profile
> 2016-10-16 15:19:28 +0000
> @@ -0,0 +1,4 @@
> +/home/ubuntu/bzr/apparmor/tests/regression/apparmor/syscall_setpriority {
> + capability sys_nice,
> +
> +}
>
> === added file
> 'libraries/libapparmor/testsuite/test_multi/testcase_encoded_comm.profile'
> --- libraries/libapparmor/testsuite/test_multi/testcase_encoded_comm.profile
> 1970-01-01 00:00:00 +0000
> +++ libraries/libapparmor/testsuite/test_multi/testcase_encoded_comm.profile
> 2016-10-16 15:19:28 +0000
> @@ -0,0 +1,4 @@
> +"/home/steve/tmp/my prog.sh" {
> + "/home/steve/tmp/my prog.sh" r,
> +
> +}
>
> === added file
> 'libraries/libapparmor/testsuite/test_multi/testcase_encoded_profile.profile'
> ---
> libraries/libapparmor/testsuite/test_multi/testcase_encoded_profile.profile
> 1970-01-01 00:00:00 +0000
> +++
> libraries/libapparmor/testsuite/test_multi/testcase_encoded_profile.profile
> 2016-10-16 15:19:28 +0000
> @@ -0,0 +1,4 @@
> +profile "test space" {
> + /lib/x86_64-linux-gnu/libdl-2.13.so r,
> +
> +}
>
> === added file
> 'libraries/libapparmor/testsuite/test_multi/testcase_network_02.profile'
> --- libraries/libapparmor/testsuite/test_multi/testcase_network_02.profile
> 1970-01-01 00:00:00 +0000
> +++ libraries/libapparmor/testsuite/test_multi/testcase_network_02.profile
> 2016-10-16 15:19:28 +0000
> @@ -0,0 +1,4 @@
> +/usr/bin/evince-thumbnailer {
> + network inet stream,
> +
> +}
>
> === added file
> 'libraries/libapparmor/testsuite/test_multi/testcase_network_03.profile'
> --- libraries/libapparmor/testsuite/test_multi/testcase_network_03.profile
> 1970-01-01 00:00:00 +0000
> +++ libraries/libapparmor/testsuite/test_multi/testcase_network_03.profile
> 2016-10-16 15:19:28 +0000
> @@ -0,0 +1,4 @@
> +/usr/lib/dovecot/imap-login {
> + network inet6 stream,
> +
> +}
>
> === added file
> 'libraries/libapparmor/testsuite/test_multi/testcase_network_04.profile'
> --- libraries/libapparmor/testsuite/test_multi/testcase_network_04.profile
> 1970-01-01 00:00:00 +0000
> +++ libraries/libapparmor/testsuite/test_multi/testcase_network_04.profile
> 2016-10-16 15:19:28 +0000
> @@ -0,0 +1,4 @@
> +/home/ubuntu/tmp/nc {
> + network inet6 stream,
> +
> +}
>
> === added file
> 'libraries/libapparmor/testsuite/test_multi/testcase_network_05.profile'
> --- libraries/libapparmor/testsuite/test_multi/testcase_network_05.profile
> 1970-01-01 00:00:00 +0000
> +++ libraries/libapparmor/testsuite/test_multi/testcase_network_05.profile
> 2016-10-16 15:19:28 +0000
> @@ -0,0 +1,4 @@
> +/home/ubuntu/tmp/nc {
> + network inet6 stream,
> +
> +}
>
> === added file
> 'libraries/libapparmor/testsuite/test_multi/testcase_syslog_capability.profile'
> ---
> libraries/libapparmor/testsuite/test_multi/testcase_syslog_capability.profile
> 1970-01-01 00:00:00 +0000
> +++
> libraries/libapparmor/testsuite/test_multi/testcase_syslog_capability.profile
> 2016-10-16 15:19:28 +0000
> @@ -0,0 +1,4 @@
> +/home/ubuntu/bzr/apparmor/tests/regression/apparmor/syscall_setpriority {
> + capability sys_nice,
> +
> +}
>
> === added file
> 'libraries/libapparmor/testsuite/test_multi/testcase_syslog_read.profile'
> --- libraries/libapparmor/testsuite/test_multi/testcase_syslog_read.profile
> 1970-01-01 00:00:00 +0000
> +++ libraries/libapparmor/testsuite/test_multi/testcase_syslog_read.profile
> 2016-10-16 15:19:28 +0000
> @@ -0,0 +1,4 @@
> +/usr/sbin/vsftpd {
> + /home/bane/foo r,
> +
> +}
>
> === modified file 'utils/test/test-libapparmor-test_multi.py'
> --- utils/test/test-libapparmor-test_multi.py 2016-10-01 18:57:09 +0000
> +++ utils/test/test-libapparmor-test_multi.py 2016-10-16 15:28:20 +0000
> @@ -10,19 +10,18 @@
> # ------------------------------------------------------------------
>
> import unittest
> -from common_test import AATest, setup_all_loops
> +from common_test import AATest, setup_all_loops, read_file
>
> import os
> from apparmor.common import open_file_read
>
> +import apparmor.aa
> from apparmor.logparser import ReadLog
>
> -# This testcase will parse all libraries/libapparmor/testsuite/test_multi
> tests
> -# and compare the result with the *.out files
> -
> -
> class TestLibapparmorTestMulti(AATest):
> - tests = [] # filled by parse_test_profiles()
> + '''Parse all libraries/libapparmor/testsuite/test_multi tests and
> compare the result with the *.out files'''
> +
> + tests = 'invalid' # filled by parse_test_profiles()
>
> def _run_test(self, params, expected):
> # tests[][expected] is a dummy, replace it with the real values
> @@ -138,13 +137,79 @@
> return exresult
>
>
> +log_to_profile_known_failures = [
> + 'testcase31', # XXX AppArmorBug: Log contains unknown mode mrwIxl
> + 'testcase24', # XXX network with operation="socket_create"
> + 'testcase33', # XXX network with operation="socket_create"
> + 'testcase_dmesg_changehat_negative_error', # fails in write_header ->
> quote_if_needed because data is None
> + 'testcase_syslog_changehat_negative_error', # fails in write_header ->
> quote_if_needed because data is None
> + 'testcase_changehat_01', # interactive, asks to add a hat
> +]
> +
> +class TestLogToProfile(AATest):
> + '''Check if the libraries/libapparmor/testsuite/test_multi tests result
> in the expected profile'''
> +
> + tests = 'invalid' # filled by parse_test_profiles()
> +
> + def _run_test(self, params, expected):
> + logfile = '%s.in' % params
> + profile_dummy_file = 'AATest_does_exist'
> +
> + # we need to find out the profile name and aamode (complain vs.
> enforce mode) so that the test can access the correct place in storage
> + parser = ReadLog('', '', '', '', '')
> + parsed_event = parser.parse_event(read_file(logfile))
> +
> + if not parsed_event: # AA_RECORD_INVALID
> + return
> +
> + if params.split('/')[-1] in log_to_profile_known_failures:
> + return
> +
> + aamode = parsed_event['aamode']
> +
> + if aamode in['AUDIT', 'STATUS', 'HINT']: # ignore some event types
> # XXX maybe we shouldn't ignore AUDIT events?
> + return
> +
> + if aamode not in ['PERMITTING', 'REJECTING']:
> + raise Exception('Unexpected aamode %s' % parsed_event['aamode'])
> +
> + # cleanup apparmor.aa storage
> + apparmor.aa.log = dict()
> + apparmor.aa.aa = apparmor.aa.hasher()
> + apparmor.aa.prelog = apparmor.aa.hasher()
> + apparmor.aa.log_dict = apparmor.aa.hasher()
> +
> + profile = parsed_event['profile']
> +
> + apparmor.aa.existing_profiles = {profile: profile_dummy_file}
> +
> + log_reader = ReadLog(dict(), logfile, apparmor.aa.existing_profiles,
> '', [])
> + log = log_reader.read_log('')
> +
> + for root in log:
> + apparmor.aa.handle_children('', '', root) # interactive for
> exec events!
> +
> + apparmor.aa.collapse_log()
> +
> + apparmor.aa.filelist = apparmor.aa.hasher()
> + apparmor.aa.filelist[profile_dummy_file]['profiles'][profile] = True
> +
> + new_profile =
> apparmor.aa.serialize_profile(apparmor.aa.log_dict[aamode][profile], profile,
> None)
> +
> + try:
> + expected_profile = read_file('%s.profile' % params)
> + except FileNotFoundError: # no .profile file
> + # print('%s.profile not found, skipping' % params) # XXX enable
> this line to get a TODO list
> + return # we don't have profiles for all testcases yet
> +
> + self.assertEqual(new_profile, expected_profile)
> +
> +
> def find_test_multi(log_dir):
> '''find all log sniplets in the given log_dir'''
>
> log_dir = os.path.abspath(log_dir)
>
> - print('Testing libapparmor test_multi tests...')
> -
> tests = []
> for root, dirs, files in os.walk(log_dir):
> for file in files:
> @@ -152,7 +217,7 @@
> file_with_path = os.path.join(root, file[:-3]) # filename
> without '.in'
> tests.append([file_with_path, True]) # True is a dummy
> testresult, parsing of the *.out files is done while running the tests
>
> - elif file.endswith('.out') or file.endswith('.err'):
> + elif file.endswith('.out') or file.endswith('.err') or
> file.endswith('.profile'):
> pass
> else:
> raise Exception('Found unknown file %s in libapparmor
> test_multi' % file)
> @@ -160,7 +225,9 @@
> return tests
>
>
> +print('Testing libapparmor test_multi tests...')
> TestLibapparmorTestMulti.tests =
> find_test_multi('../../libraries/libapparmor/testsuite/test_multi/')
> +TestLogToProfile.tests =
> find_test_multi('../../libraries/libapparmor/testsuite/test_multi/')
>
> setup_all_loops(__name__)
> if __name__ == '__main__':
>
>
signature.asc
Description: PGP signature
-- AppArmor mailing list AppArmor@lists.ubuntu.com Modify settings or unsubscribe at: https://lists.ubuntu.com/mailman/listinfo/apparmor
