# Monitor the heartbeat messages that are sent through syslog
# facilities.  This makes sure that syslog is properly configured to
# pass events of interest to the syslog server. This set of rules will
# report when a 10 minute heartbeat message is missing for a
# host/facility pair.
#
# This ruleset auto-learns a heartbeat message with a newly seen
# hostname/facility pairing. It records every new host/facility in the
# timestamp.seed file that is replayed when sec starts or
# restarts. Note that you must use the sec "-intevents" flag to get
# these events. The replay of the timestamp.seed file primes the rules
# looking for missing timestamp messages.
#
# Example event looks like:
#
#   Sep 15 12:02:26 blade1.cs.umb.edu root: \
#          [ID 234655 kern.crit] -- HEARTBEAT --
#
#  and is generated by a (root in this case) cron job that calls:
#
#   /usr/ucb/logger -p daemon.notice -- '-- HEARTBEAT --'
#

## standard preamble
type= single
continue= dontcont
ptype= nregexp
pattern= HEARTBEAT|SEC_
desc= Eliminate events that aren't processed by this ruleset.
action= none

type= single
continue= dontcont
ptype= tvalue
pattern= TRUE
desc= See if other ruleset has processed this event
action= none
context= [EVENT_PROCESSED]

type= single
continue= takenext
ptype= tvalue
pattern= TRUE
desc= Stop other rulesets from processing this event
action= create EVENT_PROCESSED

# Startup rule. Play the seed file for timestamps
# Create a 30 second context to suppress updates to the seed file
# while it is being played. This context will be deleted early
# if the timestamp events are processed within 30 seconds. If not,
# then send a warning. Make sure to delete EVENT_PROCESSED so other
# rulesets will evaluate SEC_* messages.
type= single
desc= play seed file for timestamps
ptype= regexp
pattern= SEC_STARTUP|SEC_RESTART
context= SEC_INTERNAL_EVENT
action= delete EVENT_PROCESSED ;\
         spawn (/bin/cat timestamp.seed; \
                echo "TIMESTAMP_SEED_DONE -- HEARTBEAT --"); \
         create seeding_timestamps 30 shellcmd \
           /usr/bin/mailx -s \
           "seeding timestamps incomplete after 30 seconds on `hostname`" admin

# Handle end of seeding of timestamps.
type= single
desc= Handle end of timestamp seeds
ptype= regexp
pattern= ^TIMESTAMP_SEED_DONE -- HEARTBEAT --$
context= seeding_timestamps
action= delete seeding_timestamps

# Handle unexpected arrival of end of timestamp seed message.
type= single
desc= Handle unexpected arrival of end of timestamp seed message.
ptype= regexp
pattern= ^TIMESTAMP_SEED_DONE -- HEARTBEAT --$
context= !seeding_timestamps
action= shellcmd /usr/bin/mailx -s "Received TIMESTAMP_SEED_DONE outside of seeding_timestamps window." admin

# Check to see if this timestamp host/facility has been seen.
# If so, ignore it. If not append the message to the end of the
# timestamp.seed file so its existence will be remembered.
#
# We don't do this while we are seeding with the timestamp file since
# the entry is already in the seed file since we are processing it.
type= single
desc= detect new timestamps from unseen hosts
ptype= regexp
pattern= ([A-z0-9._-]+) root: \[.* ([a-z]*)\.[a-z]*\] -- HEARTBEAT --$
context= !seen_timestamp_from_$1_$2 && ! seeding_timestamps
action= write timestamp.seed $0; create seen_timestamp_from_$1_$2
continue= takenext

# If we are replaying the seed file, set the context recording the
# existence of the heartbeat for the host/facility but don't record it
# in the seed file since we are replaying the seed file.
type= single
desc= Record seen timestamp while seeding timestamp messages.
ptype= regexp
pattern= ([A-z0-9._-]+) root: \[.* ([a-z]*)\.[a-z]*\] -- HEARTBEAT --$
context= ! seen_timestamp_from_$1_$2 && seeding_timestamps
action= create seen_timestamp_from_$1_$2
continue= takenext

# Create the context that will issue a warning on its expiration.
# timestamps arrive every 10 minutes by default, so time out in 21
# minutes.
type= single
desc= Detect missing timestamps
ptype= regexp
pattern= ([A-z0-9._-]+) root: \[.* ([a-z]*)\.[a-z]*\] -- HEARTBEAT --$
action= create timestamp_for_$1_$2_active 610 ( \
          shellcmd /usr/bin/mailx -s \
              "Missing timestamp mark for $1 facility $2" admin; \
          event 0 $0 REARM); \
        delete timestamp_double_rearm_$1_$2; \
	delete timestamp_single_rearm_$1_$2

# Rearm the timestamp_for_<host>_<facility>_active. Wait 60 minutes
# second and subsequent times through.
type= single
desc= Rearm wait for 60 minutes.
ptype= regexp
pattern= ([A-z0-9._-]+) root: \[.* ([a-z]*)\.[a-z]*\] -- HEARTBEAT -- REARM$
context= timestamp_single_rearm_$1_$2 || timestamp_double_rearm_$1_$2
action= create timestamp_for_$1_$2_active 3600 ( \
          shellcmd /usr/bin/mailx -s \
            "Missing timestamp mark for $1 facility $2" admin; \
          event 0 $0); create timestamp_double_rearm_$1_$2

# Rearm the timestamp_for_<host>_<facility>_active. Wait 30 minutes
# for first nag message.
type= single
desc= Rearm wait for 30 minutes.
ptype= regexp
pattern= ([A-z0-9._-]+) root: \[.* ([a-z]*)\.[a-z]*\] -- HEARTBEAT -- REARM$
context= ! timestamp_single_rearm_$1_$2
action= create timestamp_for_$1_$2_active 1800 ( \
          shellcmd /usr/bin/mailx -s \
            "Missing timestamp mark for $1 facility $2" admin; \
          event 0 $0); create timestamp_single_rearm_$1_$2 3600

# If we get here, we must not have handled the event.
type= single
continue= dontcont
ptype= tvalue
pattern= TRUE
desc= unset EVENT_PROCESSED
action= delete EVENT_PROCESSED

# publish