On Sun, Oct 16, 2016 at 06:05:47PM +0200, Christian Boltz wrote: > Hello, > > this patch adds TestLogToProfile to test-libapparmor-test_multi.py which > "translates" the test_multi log sniplets to a profile, and checks if it > matches the expected profile. > > The expected profile for one log event will obviously contain only one > rule, and gets added as *.profile to the test_multi directory. > > This patch includes 33 test_multi profiles - which means 83 more need to > be created Whenever you have some time, add one or two! (Please write > those test_multi profiles manually, without using the tools.) > > I know some parts of the test code looks complicated. Unfortunately this > is how things work - compare it with do_logprof_pass() in aa.py... > > While on it, set tests = 'invalid' which ensures a failure in case > parse_test_profiles() doesn't set the tests array, and move printing > the test name out of parse_test_profiles() to avoid printing it twice. > > A nice side effect of this patch is increased test coverage: > - 30% -> 40% in aa.py (= 250 more lines) > - 52% -> 78% in aamode.py (= 23 more lines) > - 26% -> 68% in logparser.py (= 120 more lines) > - total coverage increases from 57% to 62% > > > Note: to review this patch, you'll need to compare the *.profile files > to the *.in files. > > > > [ 01-test-log-to-profile.diff ]
This is cool :) nice. Acked-by: Seth Arnold <seth.arn...@canonical.com> Thanks > > === added file 'libraries/libapparmor/testsuite/test_multi/testcase02.profile' > --- libraries/libapparmor/testsuite/test_multi/testcase02.profile > 1970-01-01 00:00:00 +0000 > +++ libraries/libapparmor/testsuite/test_multi/testcase02.profile > 2016-10-16 15:19:28 +0000 > @@ -0,0 +1,4 @@ > +/bin/ping { > + capability setuid, > + > +} > > === added file 'libraries/libapparmor/testsuite/test_multi/testcase03.profile' > --- libraries/libapparmor/testsuite/test_multi/testcase03.profile > 1970-01-01 00:00:00 +0000 > +++ libraries/libapparmor/testsuite/test_multi/testcase03.profile > 2016-10-16 15:19:28 +0000 > @@ -0,0 +1,4 @@ > +/bin/ping { > + capability setuid, > + > +} > > === added file 'libraries/libapparmor/testsuite/test_multi/testcase04.profile' > --- libraries/libapparmor/testsuite/test_multi/testcase04.profile > 1970-01-01 00:00:00 +0000 > +++ libraries/libapparmor/testsuite/test_multi/testcase04.profile > 2016-10-16 15:19:28 +0000 > @@ -0,0 +1,4 @@ > +/bin/ping { > + capability setuid, > + > +} > > === added file 'libraries/libapparmor/testsuite/test_multi/testcase05.profile' > --- libraries/libapparmor/testsuite/test_multi/testcase05.profile > 1970-01-01 00:00:00 +0000 > +++ libraries/libapparmor/testsuite/test_multi/testcase05.profile > 2016-10-16 15:19:28 +0000 > @@ -0,0 +1,4 @@ > +/bin/ping { > + capability setuid, > + > +} > > === added file 'libraries/libapparmor/testsuite/test_multi/testcase06.profile' > --- libraries/libapparmor/testsuite/test_multi/testcase06.profile > 1970-01-01 00:00:00 +0000 > +++ libraries/libapparmor/testsuite/test_multi/testcase06.profile > 2016-10-16 15:19:28 +0000 > @@ -0,0 +1,4 @@ > +/bin/ping { > + capability setuid, > + > +} > > === added file 'libraries/libapparmor/testsuite/test_multi/testcase07.profile' > --- libraries/libapparmor/testsuite/test_multi/testcase07.profile > 1970-01-01 00:00:00 +0000 > +++ libraries/libapparmor/testsuite/test_multi/testcase07.profile > 2016-10-16 15:19:28 +0000 > @@ -0,0 +1,4 @@ > +/bin/ping { > + capability setuid, > + > +} > > === added file 'libraries/libapparmor/testsuite/test_multi/testcase08.profile' > --- libraries/libapparmor/testsuite/test_multi/testcase08.profile > 1970-01-01 00:00:00 +0000 > +++ libraries/libapparmor/testsuite/test_multi/testcase08.profile > 2016-10-16 15:19:28 +0000 > @@ -0,0 +1,4 @@ > +/bin/ping { > + capability setuid, > + > +} > > === added file 'libraries/libapparmor/testsuite/test_multi/testcase09.profile' > --- libraries/libapparmor/testsuite/test_multi/testcase09.profile > 1970-01-01 00:00:00 +0000 > +++ libraries/libapparmor/testsuite/test_multi/testcase09.profile > 2016-10-16 15:19:28 +0000 > @@ -0,0 +1,4 @@ > +/bin/ping { > + capability setuid, > + > +} > > === added file 'libraries/libapparmor/testsuite/test_multi/testcase10.profile' > --- libraries/libapparmor/testsuite/test_multi/testcase10.profile > 1970-01-01 00:00:00 +0000 > +++ libraries/libapparmor/testsuite/test_multi/testcase10.profile > 2016-10-16 15:19:28 +0000 > @@ -0,0 +1,4 @@ > +/bin/ping { > + capability setuid, > + > +} > > === added file 'libraries/libapparmor/testsuite/test_multi/testcase11.profile' > --- libraries/libapparmor/testsuite/test_multi/testcase11.profile > 1970-01-01 00:00:00 +0000 > +++ libraries/libapparmor/testsuite/test_multi/testcase11.profile > 2016-10-16 15:19:28 +0000 > @@ -0,0 +1,4 @@ > +/bin/ping { > + capability setuid, > + > +} > > === added file 'libraries/libapparmor/testsuite/test_multi/testcase14.profile' > --- libraries/libapparmor/testsuite/test_multi/testcase14.profile > 1970-01-01 00:00:00 +0000 > +++ libraries/libapparmor/testsuite/test_multi/testcase14.profile > 2016-10-16 15:19:28 +0000 > @@ -0,0 +1,4 @@ > +/usr/sbin/nmbd { > + /var/run/samba/unexpected.tdb k, > + > +} > > === added file 'libraries/libapparmor/testsuite/test_multi/testcase15.profile' > --- libraries/libapparmor/testsuite/test_multi/testcase15.profile > 1970-01-01 00:00:00 +0000 > +++ libraries/libapparmor/testsuite/test_multi/testcase15.profile > 2016-10-16 15:19:28 +0000 > @@ -0,0 +1,4 @@ > +/usr/sbin/nmbd { > + /var/run/samba/unexpected.tdb k, > + > +} > > === added file 'libraries/libapparmor/testsuite/test_multi/testcase16.profile' > --- libraries/libapparmor/testsuite/test_multi/testcase16.profile > 1970-01-01 00:00:00 +0000 > +++ libraries/libapparmor/testsuite/test_multi/testcase16.profile > 2016-10-16 15:19:28 +0000 > @@ -0,0 +1,4 @@ > +/usr/sbin/nmbd { > + /var/run/samba/unexpected.tdb k, > + > +} > > === added file 'libraries/libapparmor/testsuite/test_multi/testcase17.profile' > --- libraries/libapparmor/testsuite/test_multi/testcase17.profile > 1970-01-01 00:00:00 +0000 > +++ libraries/libapparmor/testsuite/test_multi/testcase17.profile > 2016-10-16 15:19:28 +0000 > @@ -0,0 +1,4 @@ > +/usr/sbin/nmbd { > + /var/run/samba/unexpected.tdb k, > + > +} > > === added file 'libraries/libapparmor/testsuite/test_multi/testcase21.profile' > --- libraries/libapparmor/testsuite/test_multi/testcase21.profile > 1970-01-01 00:00:00 +0000 > +++ libraries/libapparmor/testsuite/test_multi/testcase21.profile > 2016-10-16 15:19:28 +0000 > @@ -0,0 +1,4 @@ > +/home/steve/tmp/sh { > + /etc/passwd r, > + > +} > > === added file 'libraries/libapparmor/testsuite/test_multi/testcase22.profile' > --- libraries/libapparmor/testsuite/test_multi/testcase22.profile > 1970-01-01 00:00:00 +0000 > +++ libraries/libapparmor/testsuite/test_multi/testcase22.profile > 2016-10-16 15:19:28 +0000 > @@ -0,0 +1,4 @@ > +/home/steve/tmp/sh { > + "/tmp/does not exist" w, > + > +} > > === added file 'libraries/libapparmor/testsuite/test_multi/testcase24.profile' > --- libraries/libapparmor/testsuite/test_multi/testcase24.profile > 1970-01-01 00:00:00 +0000 > +++ libraries/libapparmor/testsuite/test_multi/testcase24.profile > 2016-10-16 15:19:28 +0000 > @@ -0,0 +1,4 @@ > +/home/steve/aa-regression-tests/changehat_wrapper//net_raw { > + network packet raw, > + > +} > > === added file 'libraries/libapparmor/testsuite/test_multi/testcase33.profile' > --- libraries/libapparmor/testsuite/test_multi/testcase33.profile > 1970-01-01 00:00:00 +0000 > +++ libraries/libapparmor/testsuite/test_multi/testcase33.profile > 2016-10-16 15:19:28 +0000 > @@ -0,0 +1,3 @@ > +/home/steve/aa-regression-tests/changehat_wrapper//net_raw { > + network packet raw, > +} > > === added file > 'libraries/libapparmor/testsuite/test_multi/testcase_dbus_01.profile' > --- libraries/libapparmor/testsuite/test_multi/testcase_dbus_01.profile > 1970-01-01 00:00:00 +0000 > +++ libraries/libapparmor/testsuite/test_multi/testcase_dbus_01.profile > 2016-10-16 15:19:28 +0000 > @@ -0,0 +1,4 @@ > +/tmp/apparmor-2.8.0/tests/regression/apparmor/dbus_service { > + dbus send bus=session path=/org/freedesktop/DBus > interface=org.freedesktop.DBus member=Hello peer=(label=unconfined), > + > +} > > === added file > 'libraries/libapparmor/testsuite/test_multi/testcase_dbus_02.profile' > --- libraries/libapparmor/testsuite/test_multi/testcase_dbus_02.profile > 1970-01-01 00:00:00 +0000 > +++ libraries/libapparmor/testsuite/test_multi/testcase_dbus_02.profile > 2016-10-16 15:19:28 +0000 > @@ -0,0 +1,4 @@ > +/tmp/apparmor-2.8.0/tests/regression/apparmor/dbus_service { > + dbus bind bus=session name=com.apparmor.Test, > + > +} > > === added file > 'libraries/libapparmor/testsuite/test_multi/testcase_dbus_03.profile' > --- libraries/libapparmor/testsuite/test_multi/testcase_dbus_03.profile > 1970-01-01 00:00:00 +0000 > +++ libraries/libapparmor/testsuite/test_multi/testcase_dbus_03.profile > 2016-10-16 15:19:28 +0000 > @@ -0,0 +1,4 @@ > +/tmp/apparmor-2.8.0/tests/regression/apparmor/dbus_service { > + dbus receive bus=session path=/com/apparmor/Test > interface=com.apparmor.Test member=Signal peer=(label=unconfined), > + > +} > > === added file > 'libraries/libapparmor/testsuite/test_multi/testcase_dbus_04.profile' > --- libraries/libapparmor/testsuite/test_multi/testcase_dbus_04.profile > 1970-01-01 00:00:00 +0000 > +++ libraries/libapparmor/testsuite/test_multi/testcase_dbus_04.profile > 2016-10-16 15:19:28 +0000 > @@ -0,0 +1,4 @@ > +/tmp/apparmor-2.8.0/tests/regression/apparmor/dbus_service { > + dbus send bus=system path=/org/freedesktop/DBus > interface=org.freedesktop.DBus member=Hello peer=(label=unconfined), > + > +} > > === added file > 'libraries/libapparmor/testsuite/test_multi/testcase_dbus_05.profile' > --- libraries/libapparmor/testsuite/test_multi/testcase_dbus_05.profile > 1970-01-01 00:00:00 +0000 > +++ libraries/libapparmor/testsuite/test_multi/testcase_dbus_05.profile > 2016-10-16 15:19:28 +0000 > @@ -0,0 +1,4 @@ > +/tmp/apparmor-2.8.0/tests/regression/apparmor/dbus_service { > + dbus bind bus=system name=org.freedesktop.nm_dhcp_client, > + > +} > > === added file > 'libraries/libapparmor/testsuite/test_multi/testcase_dbus_06.profile' > --- libraries/libapparmor/testsuite/test_multi/testcase_dbus_06.profile > 1970-01-01 00:00:00 +0000 > +++ libraries/libapparmor/testsuite/test_multi/testcase_dbus_06.profile > 2016-10-16 15:19:28 +0000 > @@ -0,0 +1,4 @@ > +/tmp/apparmor-2.8.0/tests/regression/apparmor/dbus_service { > + dbus receive bus=system path=/org/freedesktop/nm_dhcp_client > interface=org.freedesktop.nm_dhcp_client member=Signal > peer=(label=unconfined), > + > +} > > === added file > 'libraries/libapparmor/testsuite/test_multi/testcase_dmesg_capability.profile' > --- > libraries/libapparmor/testsuite/test_multi/testcase_dmesg_capability.profile > 1970-01-01 00:00:00 +0000 > +++ > libraries/libapparmor/testsuite/test_multi/testcase_dmesg_capability.profile > 2016-10-16 15:19:28 +0000 > @@ -0,0 +1,4 @@ > +/home/ubuntu/bzr/apparmor/tests/regression/apparmor/syscall_setpriority { > + capability sys_nice, > + > +} > > === added file > 'libraries/libapparmor/testsuite/test_multi/testcase_encoded_comm.profile' > --- libraries/libapparmor/testsuite/test_multi/testcase_encoded_comm.profile > 1970-01-01 00:00:00 +0000 > +++ libraries/libapparmor/testsuite/test_multi/testcase_encoded_comm.profile > 2016-10-16 15:19:28 +0000 > @@ -0,0 +1,4 @@ > +"/home/steve/tmp/my prog.sh" { > + "/home/steve/tmp/my prog.sh" r, > + > +} > > === added file > 'libraries/libapparmor/testsuite/test_multi/testcase_encoded_profile.profile' > --- > libraries/libapparmor/testsuite/test_multi/testcase_encoded_profile.profile > 1970-01-01 00:00:00 +0000 > +++ > libraries/libapparmor/testsuite/test_multi/testcase_encoded_profile.profile > 2016-10-16 15:19:28 +0000 > @@ -0,0 +1,4 @@ > +profile "test space" { > + /lib/x86_64-linux-gnu/libdl-2.13.so r, > + > +} > > === added file > 'libraries/libapparmor/testsuite/test_multi/testcase_network_02.profile' > --- libraries/libapparmor/testsuite/test_multi/testcase_network_02.profile > 1970-01-01 00:00:00 +0000 > +++ libraries/libapparmor/testsuite/test_multi/testcase_network_02.profile > 2016-10-16 15:19:28 +0000 > @@ -0,0 +1,4 @@ > +/usr/bin/evince-thumbnailer { > + network inet stream, > + > +} > > === added file > 'libraries/libapparmor/testsuite/test_multi/testcase_network_03.profile' > --- libraries/libapparmor/testsuite/test_multi/testcase_network_03.profile > 1970-01-01 00:00:00 +0000 > +++ libraries/libapparmor/testsuite/test_multi/testcase_network_03.profile > 2016-10-16 15:19:28 +0000 > @@ -0,0 +1,4 @@ > +/usr/lib/dovecot/imap-login { > + network inet6 stream, > + > +} > > === added file > 'libraries/libapparmor/testsuite/test_multi/testcase_network_04.profile' > --- libraries/libapparmor/testsuite/test_multi/testcase_network_04.profile > 1970-01-01 00:00:00 +0000 > +++ libraries/libapparmor/testsuite/test_multi/testcase_network_04.profile > 2016-10-16 15:19:28 +0000 > @@ -0,0 +1,4 @@ > +/home/ubuntu/tmp/nc { > + network inet6 stream, > + > +} > > === added file > 'libraries/libapparmor/testsuite/test_multi/testcase_network_05.profile' > --- libraries/libapparmor/testsuite/test_multi/testcase_network_05.profile > 1970-01-01 00:00:00 +0000 > +++ libraries/libapparmor/testsuite/test_multi/testcase_network_05.profile > 2016-10-16 15:19:28 +0000 > @@ -0,0 +1,4 @@ > +/home/ubuntu/tmp/nc { > + network inet6 stream, > + > +} > > === added file > 'libraries/libapparmor/testsuite/test_multi/testcase_syslog_capability.profile' > --- > libraries/libapparmor/testsuite/test_multi/testcase_syslog_capability.profile > 1970-01-01 00:00:00 +0000 > +++ > libraries/libapparmor/testsuite/test_multi/testcase_syslog_capability.profile > 2016-10-16 15:19:28 +0000 > @@ -0,0 +1,4 @@ > +/home/ubuntu/bzr/apparmor/tests/regression/apparmor/syscall_setpriority { > + capability sys_nice, > + > +} > > === added file > 'libraries/libapparmor/testsuite/test_multi/testcase_syslog_read.profile' > --- libraries/libapparmor/testsuite/test_multi/testcase_syslog_read.profile > 1970-01-01 00:00:00 +0000 > +++ libraries/libapparmor/testsuite/test_multi/testcase_syslog_read.profile > 2016-10-16 15:19:28 +0000 > @@ -0,0 +1,4 @@ > +/usr/sbin/vsftpd { > + /home/bane/foo r, > + > +} > > === modified file 'utils/test/test-libapparmor-test_multi.py' > --- utils/test/test-libapparmor-test_multi.py 2016-10-01 18:57:09 +0000 > +++ utils/test/test-libapparmor-test_multi.py 2016-10-16 15:28:20 +0000 > @@ -10,19 +10,18 @@ > # ------------------------------------------------------------------ > > import unittest > -from common_test import AATest, setup_all_loops > +from common_test import AATest, setup_all_loops, read_file > > import os > from apparmor.common import open_file_read > > +import apparmor.aa > from apparmor.logparser import ReadLog > > -# This testcase will parse all libraries/libapparmor/testsuite/test_multi > tests > -# and compare the result with the *.out files > - > - > class TestLibapparmorTestMulti(AATest): > - tests = [] # filled by parse_test_profiles() > + '''Parse all libraries/libapparmor/testsuite/test_multi tests and > compare the result with the *.out files''' > + > + tests = 'invalid' # filled by parse_test_profiles() > > def _run_test(self, params, expected): > # tests[][expected] is a dummy, replace it with the real values > @@ -138,13 +137,79 @@ > return exresult > > > +log_to_profile_known_failures = [ > + 'testcase31', # XXX AppArmorBug: Log contains unknown mode mrwIxl > + 'testcase24', # XXX network with operation="socket_create" > + 'testcase33', # XXX network with operation="socket_create" > + 'testcase_dmesg_changehat_negative_error', # fails in write_header -> > quote_if_needed because data is None > + 'testcase_syslog_changehat_negative_error', # fails in write_header -> > quote_if_needed because data is None > + 'testcase_changehat_01', # interactive, asks to add a hat > +] > + > +class TestLogToProfile(AATest): > + '''Check if the libraries/libapparmor/testsuite/test_multi tests result > in the expected profile''' > + > + tests = 'invalid' # filled by parse_test_profiles() > + > + def _run_test(self, params, expected): > + logfile = '%s.in' % params > + profile_dummy_file = 'AATest_does_exist' > + > + # we need to find out the profile name and aamode (complain vs. > enforce mode) so that the test can access the correct place in storage > + parser = ReadLog('', '', '', '', '') > + parsed_event = parser.parse_event(read_file(logfile)) > + > + if not parsed_event: # AA_RECORD_INVALID > + return > + > + if params.split('/')[-1] in log_to_profile_known_failures: > + return > + > + aamode = parsed_event['aamode'] > + > + if aamode in['AUDIT', 'STATUS', 'HINT']: # ignore some event types > # XXX maybe we shouldn't ignore AUDIT events? > + return > + > + if aamode not in ['PERMITTING', 'REJECTING']: > + raise Exception('Unexpected aamode %s' % parsed_event['aamode']) > + > + # cleanup apparmor.aa storage > + apparmor.aa.log = dict() > + apparmor.aa.aa = apparmor.aa.hasher() > + apparmor.aa.prelog = apparmor.aa.hasher() > + apparmor.aa.log_dict = apparmor.aa.hasher() > + > + profile = parsed_event['profile'] > + > + apparmor.aa.existing_profiles = {profile: profile_dummy_file} > + > + log_reader = ReadLog(dict(), logfile, apparmor.aa.existing_profiles, > '', []) > + log = log_reader.read_log('') > + > + for root in log: > + apparmor.aa.handle_children('', '', root) # interactive for > exec events! > + > + apparmor.aa.collapse_log() > + > + apparmor.aa.filelist = apparmor.aa.hasher() > + apparmor.aa.filelist[profile_dummy_file]['profiles'][profile] = True > + > + new_profile = > apparmor.aa.serialize_profile(apparmor.aa.log_dict[aamode][profile], profile, > None) > + > + try: > + expected_profile = read_file('%s.profile' % params) > + except FileNotFoundError: # no .profile file > + # print('%s.profile not found, skipping' % params) # XXX enable > this line to get a TODO list > + return # we don't have profiles for all testcases yet > + > + self.assertEqual(new_profile, expected_profile) > + > + > def find_test_multi(log_dir): > '''find all log sniplets in the given log_dir''' > > log_dir = os.path.abspath(log_dir) > > - print('Testing libapparmor test_multi tests...') > - > tests = [] > for root, dirs, files in os.walk(log_dir): > for file in files: > @@ -152,7 +217,7 @@ > file_with_path = os.path.join(root, file[:-3]) # filename > without '.in' > tests.append([file_with_path, True]) # True is a dummy > testresult, parsing of the *.out files is done while running the tests > > - elif file.endswith('.out') or file.endswith('.err'): > + elif file.endswith('.out') or file.endswith('.err') or > file.endswith('.profile'): > pass > else: > raise Exception('Found unknown file %s in libapparmor > test_multi' % file) > @@ -160,7 +225,9 @@ > return tests > > > +print('Testing libapparmor test_multi tests...') > TestLibapparmorTestMulti.tests = > find_test_multi('../../libraries/libapparmor/testsuite/test_multi/') > +TestLogToProfile.tests = > find_test_multi('../../libraries/libapparmor/testsuite/test_multi/') > > setup_all_loops(__name__) > if __name__ == '__main__': > >
signature.asc
Description: PGP signature
-- AppArmor mailing list AppArmor@lists.ubuntu.com Modify settings or unsubscribe at: https://lists.ubuntu.com/mailman/listinfo/apparmor