Forum: CFEngine Help
Subject: Re: Managing Splunk with CFEngine?
Author: msvob...@linkedin.com
Link to topic: https://cfengine.com/forum/read.php?3,24377,24391#msg-24391
This is the policy that I use to deploy and manage the Splunk forwarders.
"all_splunk" here is a global class that contains the search heads and
indexers, so, this is affecting all machines that Cfengine is deployed to,
other than those..
I fetch the binaries over HTTP, compute a checksum, extract a tarball, then
deploy into the root file system. I use Cfengine to put deploymentclient.conf
in place.
Once deploymentclient.conf is in place, I'm still using Splunk's deployment
server to push Splunk config changes out. You dont have to. I was going to
use Cfengine to do this, but, we're heading in a different direction within our
organization.
Anyways, I hope this helps you out. I suppose it at least demonstrates how to
put the code down, and to manage the Splunk processes via Linux or Solaris SMF.
Some of this looks overly complex, but, I'm having to deal with Solaris not
having a network package manager. I'm also upgrading clients here from the
Splunk "fat forwarder" which used to be a full / complete install of the Splunk
binaries, to the Splunk "universal forwarder" which is a stripped down C/C++
based agent.
Thanks
Mike
bundle agent splunk_universal_forwarder
{
vars:
!all_splunk::
"splunk_version" string =>
execresult("/export/apps/splunk/i002/splunkforwarder/bin/splunk --version",
"noshell");
!all_splunk.sunos_5_10.i86pc::
"tarball_hash" string =>
"564c23a454bdd23c4265ea3b7c92c9de";
!all_splunk.sunos_5_10.sparc::
"tarball_hash" string =>
"45ac90c4dc10cb1b09d1f7c882dca6db";
!all_splunk::
"http_server" string =>
"hostname1";
classes:
!all_splunk.sunos_5_10::
"leave_splunk_disabled" or =>
{"esv4_build01_corp", "esv4_build02_corp", };
!all_splunk::
"forwarder_already_installed" expression =>
fileexists("/export/apps/splunk/i002/splunkforwarder/etc/splunk-launch.conf");
!all_splunk::
"is_splunk_current" expression =>
regcmp("Splunk Universal Forwarder 4.2.4 (build 110225)","$(splunk_version)");
!all_splunk.sunos_5_10::
"fat_forwarder" expression =>
isdir("/export/apps/splunk/i002/splunk");
!all_splunk.linux::
"fat_forwarder" expression =>
isdir("/opt/splunk");
!all_splunk.sunos_5_10.i86pc::
"media_accessible" expression =>
fileexists("/var/cfengine/media/splunkforwarder-4.2.4-110225-SunOS-x86_64.tar");
!all_splunk.sunos_5_10.sparc::
"media_accessible" expression =>
fileexists("/var/cfengine/media/splunkforwarder-4.2.4-110225-SunOS-sparc.tar");
!all_splunk.media_accessible.i86pc::
"tarball_hash_match" expression =>
hashmatch("/var/cfengine/media/splunkforwarder-4.2.4-110225-SunOS-x86_64.tar",
"md5", "$(tarball_hash)");
!all_splunk.media_accessible.sparc::
"tarball_hash_match" expression =>
hashmatch("/var/cfengine/media/splunkforwarder-4.2.4-110225-SunOS-sparc.tar",
"md5", "$(tarball_hash)");
!all_splunk.sunos_5_10::
"fat_forwarder_smf_service" expression =>
returnszero("/bin/svcs 'splunk:default' > /dev/null 2>&1","useshell");
!all_splunk.sunos_5_10::
"forwarder_maintenance_status" expression =>
returnszero("/bin/svcs splunkforwarder 2>&1 | /bin/grep maintenance > /dev/null
2>&1","useshell");
!all_splunk.sunos_5_10::
"forwarder_disabled_status" expression =>
returnszero("/bin/svcs splunkforwarder 2>&1 | /bin/grep disabled > /dev/null
2>&1","useshell");
!all_splunk.sunos_5_10::
"forwarder_smf_service" expression =>
returnszero("/bin/svcs splunkforwarder > /dev/null 2>&1","useshell");
files:
!all_splunk::
"/etc/passwd"
edit_line =>
set_user_field("splunk",6,"/export/apps/splunk/i002/splunkforwarder"),
classes =>
if_repaired("splunk_home_directory_modified");
!all_splunk::
"/etc/passwd"
edit_line =>
set_user_field("splunk",7,"/bin/bash"),
classes =>
if_repaired("splunk_shell_modified");
# The correct /etc/shadow string for the splunk user is set in
manage_system_accounts.cf
!all_splunk.!forwarder_already_installed::
"/etc/shadow"
edit_line =>
delete_lines_matching("splunk:.LK.*"),
classes =>
if_repaired("splunk_etc_shadow_modified");
!all_splunk::
"/export/apps/splunk/i002/."
create => "true",
perms =>
mog("0755","splunk","splunk");
!all_splunk.linux::
"/export/apps/splunk/i002/splunkforwarder"
link_from =>
ln_s("/opt/splunkforwarder");
!all_splunk.linux::
"/opt/splunkforwarder/."
perms => og("splunk","splunk"),
depth_search => recurse("inf");
!all_splunk::
"/export/apps/splunk/i002/splunkforwarder/.ssh/."
create => "true",
perms =>
mog("0700","splunk","splunk"),
classes =>
if_repaired("splunk_modified");
!all_splunk::
"/export/apps/splunk/i002/splunkforwarder/.ssh/authorized_keys"
create => "true",
perms =>
mog("0600","splunk","splunk"),
copy_from =>
backup_cp_md5_compare("/var/cfengine/inputs/config-general/splunk_universal_forwarder/splunk-authorized-keys"),
classes =>
if_repaired("splunk_modified");
!all_splunk::
"/export/apps/splunk/i002/splunkforwarder/.bash_profile"
create => "true",
perms =>
mog("0755","splunk","splunk"),
copy_from =>
backup_cp_md5_compare("/var/cfengine/inputs/config-general/splunk_universal_forwarder/splunkforwarder_bash_profile"),
classes =>
if_repaired("splunk_modified");
!all_splunk::
"/export/apps/splunk/i002/splunkforwarder/etc/apps/SplunkForwarder/default/."
perms =>
mog("0755","splunk","splunk"),
classes =>
if_repaired("splunk_permissions_incorrect");
!all_splunk.splunk_permissions_incorrect::
"/export/apps/splunk/i002/splunkforwarder/etc/apps/."
perms =>
mog("0755","splunk","splunk"),
depth_search => recurse("inf"),
action => immediate;
!all_splunk::
"/export/apps/splunk/i002/splunkforwarder/etc/system/local/deploymentclient.conf"
perms =>
mog("0644","splunk","splunk"),
copy_from =>
backup_cp_md5_compare("/var/cfengine/inputs/config-general/splunk_universal_forwarder/deploymentclient.conf-PROD-ELA4"),
classes =>
if_repaired("splunk_restart_needed");
!all_splunk.linux::
"/etc/init.d/splunk"
perms =>
mog("0700","root","root"),
copy_from =>
backup_cp_md5_compare("/var/cfengine/inputs/config-linux/splunk_universal_forwarder/splunkforwarder-init-script"),
classes =>
if_repaired("splunk_init_script_modified");
!all_splunk.sunos_5_10.fat_forwarder::
"/export/apps/splunk/i002/splunk"
handle =>
"remove_solaris_fat_forwarder_data",
delete => tidy,
depth_search => recurse("inf"),
file_select => prune_all,
classes =>
if_repaired("splunk_fat_forwarder_data_purged");
!all_splunk.sunos_5_10.fat_forwarder::
"/export/apps/splunk/i002/splunk/."
handle =>
"remove_solaris_fat_forwarder_base_dir",
delete => tidy,
classes =>
if_repaired("splunk_fat_forwarder_data_purged");
!all_splunk.linux.fat_forwarder::
"/opt/splunk"
handle =>
"remove_linux_fat_forwarder_data",
delete => tidy,
depth_search => recurse("inf"),
file_select => prune_all,
classes =>
if_repaired("splunk_fat_forwarder_data_purged");
!all_splunk.linux.fat_forwarder::
"/opt/splunk/."
handle =>
"remove_linux_fat_forwarder_base_dir",
delete => tidy,
classes =>
if_repaired("splunk_fat_forwarder_data_purged");
# Remove previous media.
!all_splunk.sunos_5_10.i86pc::
"/var/cfengine/media/splunkforwarder-4.2.3-105575-SunOS-x86_64.tar"
delete => tidy;
!all_splunk.sunos_5_10.sparc::
"/var/cfengine/media/splunkforwarder-4.2.3-105575-SunOS-sparc.tar"
delete => tidy;
packages:
# We define this exact version of the RPM for Centos 5.3 and 5.5 only.
!all_splunk.linux::
"splunkforwarder"
package_policy => "addupdate",
package_method => rpm,
package_architectures => { "x86_64" },
package_version => "4.2.4-110225";
!all_splunk.fat_forwarder.sunos_5_10::
"LINKsplunkconf"
package_policy => "delete",
package_method =>
solaris("LINKsplunkconf","/var/cfengine/inputs/config-solaris/pkg.spoolfile","/var/cfengine/inputs/config-solaris/pkg.admin"),
classes =>
if_repaired("fat_forwader_removed");
processes:
!all_splunk.linux::
"splunkd" # This is what appears in the output of ps -ef
in the process table.
restart_class => "linux_splunkd_offline";
commands:
!all_splunk.(!media_accessible|!tarball_hash_match).sunos_5_10.i86pc::
"/usr/sfw/bin/wget -q -O
/var/cfengine/media/splunkforwarder-4.2.4-110225-SunOS-x86_64.tar
$(http_server)/repo/mrepo/www/LINK/nonrepo/splunkforwarder-4.2.4-110225-SunOS-x86_64.tar";
!all_splunk.(!media_accessible|!tarball_hash_match).sunos_5_10.sparc::
"/usr/sfw/bin/wget -q -O
/var/cfengine/media/splunkforwarder-4.2.4-110225-SunOS-sparc.tar
$(http_server)/repo/mrepo/www/LINK/nonrepo/splunkforwarder-4.2.4-110225-SunOS-sparc.tar";
!all_splunk.forwarder_maintenance_status.sunos_5_10::
"/export/apps/splunk/i002/splunkforwarder/bin/splunk stop"
contain => execute_as_splunk;
!all_splunk.splunkd_please_stop_and_svcadm_clear.sunos_5_10::
"/usr/sbin/svcadm clear splunkforwarder";
!all_splunk.!is_splunk_current.linux::
"/export/apps/splunk/i002/splunkforwarder/bin/splunk stop"
contain => execute_as_splunk;
!all_splunk.!is_splunk_current.sunos_5_10::
"/usr/sbin/svcadm disable splunkforwarder";
!all_splunk.sunos_5_10.(fat_forwarder_removed|!is_splunk_current)::
"/bin/pkill -u splunk";
!all_splunk.linux.(fat_forwarder_removed|!is_splunk_current)::
"/usr/bin/pkill -u splunk";
!all_splunk.sunos_5_10.fat_forwarder_smf_service::
"/usr/sbin/svccfg delete splunk:default";
!all_splunk.sunos_5_10.forwarder_maintenance_status::
"/usr/sbin/svcadm clear splunkforwarder";
!all_splunk.(!forwarder_already_installed|!is_splunk_current).!forwarder_maintenance_status::
"/export/apps/splunk/i002/splunkforwarder/bin/splunk stop"
contain => execute_as_splunk;
!all_splunk.sunos_5_10.i86pc.tarball_hash_match.(!forwarder_already_installed|!is_splunk_current).!forwarder_maintenance_status::
"/usr/sfw/bin/gtar -C /export/apps/splunk/i002 -xf
/var/cfengine/media/splunkforwarder-4.2.4-110225-SunOS-x86_64.tar";
!all_splunk.sunos_5_10.sparc.tarball_hash_match.(!forwarder_already_installed|!is_splunk_current).!forwarder_maintenance_status::
"/usr/sfw/bin/gtar -C /export/apps/splunk/i002 -xf
/var/cfengine/media/splunkforwarder-4.2.4-110225-SunOS-sparc.tar";
!all_splunk.(!forwarder_already_installed|!is_splunk_current).!forwarder_maintenance_status::
"/export/apps/splunk/i002/splunkforwarder/bin/splunk start
--accept-license --answer-yes --no-prompt"
contain => execute_as_splunk;
!all_splunk.(!forwarder_already_installed|!is_splunk_current).!forwarder_maintenance_status.sunos_5_10::
"/export/apps/splunk/i002/splunkforwarder/bin/splunk stop"
contain => execute_as_splunk;
!all_splunk.(!forwarder_already_installed|!is_splunk_current).!forwarder_maintenance_status.sunos_5_10.!leave_splunk_disabled::
"/usr/sbin/svcadm enable splunkforwarder";
!all_splunk.forwarder_disabled_status.sunos_5_10.!leave_splunk_disabled::
"/usr/sbin/svcadm enable splunkforwarder";
!all_splunk.sunos_5_10.!forwarder_already_installed::
"/bin/pkill -u splunk";
!all_splunk.sunos_5_10.!forwarder_smf_service::
"/usr/sbin/svccfg import
/var/cfengine/inputs/config-solaris/check_splunk_installed/splunkforwarder.xml";
!all_splunk.linux.(splunk_restart_needed|linux_splunkd_offline)::
"/etc/init.d/splunk restart";
!all_splunk.sunos_5_10.splunk_restart_needed::
"/usr/sbin/svcadm restart
svc:/applications/splunkforwarder:default";
!all_splunk.linux.splunk_init_script_modified::
"/sbin/chkconfig --add splunk";
reports:
!all_splunk.splunk_modified::
"cf3: Splunk was modified on $(sys.host)";
!all_splunk.splunk_restart_needed::
"cf3: Splunk deploymentclient.conf was modified. Executing a
restart of Splunk.";
!all_splunk.splunk_home_directory_modified::
"cf3: The Splunk user's home directory was modified on
$(sys.host)";
!all_splunk.splunk_etc_shadow_modified::
"cf3: The password entry for the Splunk user was modified in
/etc/shadow on $(sys.host)";
!all_splunk.splunk_shell_modified::
"cf3: The shell entry for the Splunk user was changed to bash
in /etc/passwd on $(sys.host)";
!all_splunk.!forwarder_already_installed::
"cf3: Splunk was installed on $(sys.host). Executing first
start, and pointing towards the deployment server.";
!all_splunk.splunk_permissions_incorrect::
"cf3: Unix permissions under
/export/apps/splunk/i002/splunkforwarder/etc/apps were incorrect. Changing to
755 on $(sys.host)";
fat_forwarder_removed::
"cf3: The splunk fat forwarder was removed and was replaced
with the lightweight universal forwarder on $(sys.host)";
forwarder_maintenance_status::
"cf3: The Splunk universal forwarder is in maintenance status
within SMF. Issuing a clear on $(sys.host)";
fat_forwarder_smf_service::
"cf3: The Splunk fat forwarder SMF service still exists.
Removing from SMF on $(sys.host)";
splunk_fat_forwarder_data_purged::
"cf3: The Splunk fat forwarder data still exists. Removing
/export/apps/splunk/i002/splunk";
!all_splunk.!media_accessible.sunos_5_10::
"cf3: The Splunk media was missing. Executing a wget to fetch
the Splunk media from $(http_server) on $(sys.host)";
!all_splunk.linux_splunkd_offline::
"cf3: splunkd was not online on $(sys.host). Executing an
/etc/init.d/splunk restart to bring it online";
}
bundle agent manage_system_accounts
{
vars:
"groups" string => "splunk::1234:";
"users" string =>
"splunk:x:1234:1234:Splunk Server:/export/apps/splunk/i002/splunk:/bin/bash";
"shadow_entries" slist => {
"splunk:NP:::::::", };
files:
linux::
"/etc/group"
handle => "linux_app_groups_exist",
perms => mog("0644","root","root"),
edit_line =>
append_groups_starting("manage_system_accounts.groups"),
classes =>
if_repaired("etc_group_modified");
linux::
"/etc/passwd"
handle => "linux_app_users_exist",
perms => mog("0644","root","root"),
edit_line =>
append_users_starting("manage_system_accounts.users"),
classes =>
if_repaired("etc_passwd_modified");
linux::
"/etc/shadow"
handle => "linux_app_shadow_exist",
perms => mog("0400","root","root"),
edit_line =>
append_if_no_line("$(shadow_entries)"),
_______________________________________________
Help-cfengine mailing list
Help-cfengine@cfengine.org
https://cfengine.org/mailman/listinfo/help-cfengine
