#!/usr/bin/python
# -*- encoding: utf-8; py-indent-offset: 4 -*-
# +------------------------------------------------------------------+
# |             ____ _               _        __  __ _  __           |
# |            / ___| |__   ___  ___| | __   |  \/  | |/ /           |
# |           | |   | '_ \ / _ \/ __| |/ /   | |\/| | ' /            |
# |           | |___| | | |  __/ (__|   <    | |  | | . \            |
# |            \____|_| |_|\___|\___|_|\_\___|_|  |_|_|\_\           |
# |                                                                  |
# | Copyright Mathias Kettner 2014             mk@mathias-kettner.de |
# +------------------------------------------------------------------+
#
# This file is part of Check_MK.
# The official homepage is at http://mathias-kettner.de/check_mk.
#
# check_mk is free software;  you can redistribute it and/or modify it
# under the  terms of the  GNU General Public License  as published by
# the Free Software Foundation in version 2.  check_mk is  distributed
# in the hope that it will be useful, but WITHOUT ANY WARRANTY;  with-
# out even the implied warranty of  MERCHANTABILITY  or  FITNESS FOR A
# PARTICULAR PURPOSE. See the  GNU General Public License for more de-
# tails. You should have  received  a copy of the  GNU  General Public
# License along with GNU Make; see the file  COPYING.  If  not,  write
# to the Free Software Foundation, Inc., 51 Franklin St,  Fifth Floor,
# Boston, MA 02110-1301 USA.

import cmk.paths

# Configuration variables in main.mk needed during the actual check
logwatch_dir       = cmk.paths.var_dir + '/logwatch'
logwatch_patterns = { }
logwatch_rules = []
logwatch_max_filesize = 500000 # do not save more than 500k of message (configurable)
logwatch_service_output = "default"
logwatch_groups = []

# Variables embedded in precompiled checks
check_config_variables += [ "logwatch_dir", "logwatch_max_filesize", "logwatch_service_output" ]

logwatch_spool_dir = cmk.paths.var_dir + '/logwatch_spool'


#   .--General-------------------------------------------------------------.
#   |                   ____                           _                   |
#   |                  / ___| ___ _ __   ___ _ __ __ _| |                  |
#   |                 | |  _ / _ \ '_ \ / _ \ '__/ _` | |                  |
#   |                 | |_| |  __/ | | |  __/ | | (_| | |                  |
#   |                  \____|\___|_| |_|\___|_|  \__,_|_|                  |
#   |                                                                      |
#   +----------------------------------------------------------------------+
#   |  General functions, for normal and forwarding logwatch check         |
#   '----------------------------------------------------------------------'


def logwatch_ec_forwarding_enabled(params, item):
    if 'restrict_logfiles' not in params:
        return True # matches all logs on this host
    else:
        # only logs which match the specified patterns
        for pattern in params['restrict_logfiles']:
            if re.match(pattern, item):
                return True

    return False


# Splits the number of existing logfiles into
# forwarded (to ec) and not forwarded. Returns a
# pair of forwarded and not forwarded logs.
def logwatch_select_forwarded(info):
    forwarded_logs = []
    not_forwarded_logs = []

    forward_settings = host_extra_conf(g_hostname, checkgroup_parameters.get('logwatch_ec', []))

    for l in info:
        # node info ignored (only used in regular logwatch check)
        line = " ".join(l[1:])
        if len(line) > 6 and line[0:3] == "[[[" and line[-3:] == "]]]" \
           and ':missing' not in line and ':cannotopen' not in line:
            logfile_name = line[3:-3]

            # Is forwarding enabled in general?
            if forward_settings and forward_settings[0] != None:
                if logwatch_ec_forwarding_enabled(forward_settings[0], logfile_name):
                    forwarded_logs.append(logfile_name)
                else:
                    not_forwarded_logs.append(logfile_name)

            # No forwarding rule configured
            else:
                not_forwarded_logs.append(logfile_name)

    return forwarded_logs, not_forwarded_logs


def inventory_logwatch(info):
    forwarded_logs, not_forwarded_logs = logwatch_select_forwarded(info)
    inventory = []
    for logfile in not_forwarded_logs:
        groups = logwatch_groups_of_logfile(logfile)
        if groups:
            continue
        else:
            inventory.append((logfile, None))
    return inventory


# logwatch_patterns = {
#    'System': [
#    ( 'W', 'sshd' ),
#    ( ['host1', 'host2'],        'C', 'ssh' ), # only applies to certain hosts
#    ( ['lnx', 'dmz'], ALL_HOSTS, 'C', 'ssh' ), # only applies to host having certain tags
#    ( ALL_HOSTS, (10, 20), 'x' ), # at 10 messages per interval warn, at 20 crit
#    ( 'I', '0' )
#    ],
#    'Application': [
#    ( 'W', 'crash.exe' ),
#    ( 'E', 'ssh' )
#    ]
#    }

# New rule-stule logwatch_rules in WATO friendly consistent rule notation:
#
# logwatch_rules = [
#   ( [ PATTERNS ], ALL_HOSTS, [ "Application", "System" ] ),
# ]
# All [ PATTERNS ] of matching rules will be concatenated in order of
# appearance.
#
# PATTERN is a list like:
# [ ( 'O',      ".*ssh.*" ),          # Make informational (OK) messages from these
#   ( (10, 20), "login"   ),          # Warning at 10 messages, Critical at 20
#   ( 'C',      "bad"     ),          # Always critical
#   ( 'W',      "not entirely bad" ), # Always warning
# ]
#


def logwatch_state(state):
    if state == 1:
        return "WARN"
    elif state != 0:
        return "CRIT"
    else:
        return "OK"


def logwatch_level_name(level):
    if   level == 'O': return 'OK'
    elif level == 'W': return 'WARN'
    elif level == 'u': return 'WARN' # undefined states are treated as warning
    elif level == 'C': return 'CRIT'
    else: return 'IGN'


def logwatch_level_worst(worst, level):
    if   level == 'O': return max(worst, 0)
    elif level == 'W': return max(worst, 1)
    elif level == 'u': return max(worst, 1)
    elif level == 'C': return max(worst, 2)
    else: return worst


# Extracts patterns that are relevant for the current host and item.
# Constructs simple list of pairs: [ ('W', 'crash.exe'), ('C', 'sshd.*test') ]
def logwatch_precompile(hostname, item, _unused):
    # Initialize the patterns list with the logwatch_rules
    params = {"reclassify_patterns": [], "reclassify_states": {}}
    description = check_info['logwatch']['service_description'] % item

    # This is the new (-> WATO controlled) variable
    rules = service_extra_conf(hostname, item, logwatch_rules)

    for rule in rules:
        if isinstance(rule, dict):
            params["reclassify_patterns"].extend(x[:2] for x in rule.get("reclassify_patterns"))
            if "reclassify_states" in rule:
                params["reclassify_states"] = rule["reclassify_states"]
        else:
            params["reclassify_patterns"].extend(x[:2] for x in rule)

    # Now load the old logwatch_patterns var
    patterns = logwatch_patterns.get(item)
    if patterns:
        for entry in patterns:
            hostlist = None
            tags = []

            pattern = entry[-1]
            level = entry[-2]

            if len(entry) >= 3:    # found optional host list
                hostlist = entry[-3]
            if len(entry) >= 4:    # found optional host tags
                tags = entry[-4]

            if hostlist and not \
                   (hosttags_match_taglist(tags_of_host(hostname), tags) and \
                    in_extraconf_hostlist(hostlist, hostname)):
                continue

            params["reclassify_patterns"].append((level, pattern))

    return params


def logwatch_reclassify(counts, patterns, text, old_level):
    new_level = None

    # Reclassify state to another state
    change_state_paramkey = ("%s_to" % old_level).lower()
    if change_state_paramkey in patterns.get("reclassify_states", {}) and\
        patterns["reclassify_states"][change_state_paramkey] != old_level:
            new_level = patterns["reclassify_states"][change_state_paramkey]

    # Reclassify state if a given regex pattern matches
    # A match overrules the previous state->state reclassification
    for level, pattern in patterns.get("reclassify_patterns", []):
        reg = regex(pattern)
        if reg.search(text):
            # If the level is not fixed like 'C' or 'W' but a pair like (10, 20),
            # then we count how many times this pattern has already matched and
            # assign the levels according to the number of matches of this pattern.
            if type(level) == tuple:
                warn, crit = level
                newcount = counts.setdefault(id(pattern), 0) + 1
                counts[id(pattern)] = newcount
                if newcount >= crit:
                    return 'C'
                elif newcount >= warn:
                    return 'W'
                else:
                    return 'I'
            else:
                return level

    return new_level


def logwatch_parse_line(line):
    parts = line.split(None, 1)
    level = parts[0]
    if len(parts) > 1:
        text = parts[1]
    else:
        text = ""
    return level, text


#.
#   .--Logwatch------------------------------------------------------------.
#   |              _                           _       _                   |
#   |             | | ___   __ ___      ____ _| |_ ___| |__                |
#   |             | |/ _ \ / _` \ \ /\ / / _` | __/ __| '_ \               |
#   |             | | (_) | (_| |\ V  V / (_| | || (__| | | |              |
#   |             |_|\___/ \__, | \_/\_/ \__,_|\__\___|_| |_|              |
#   |                      |___/                                           |
#   +----------------------------------------------------------------------+
#   |  Normal logwatch check                                               |
#   '----------------------------------------------------------------------'


# In case of a precompiled check, params contains the precompiled
# logwatch_patterns for the logfile we deal with. If using check_mk
# without precompiled checks, the params must be None an will be
# ignored.
def check_logwatch(item, params, info):
    def cachefile_path(node):
        return cmk.paths.tcp_cache_dir + "/" + node

    now = time.time()
    last_run = get_item_state("logwatch_last_run_%s" % item, 0)
    set_item_state("logwatch_last_run_%s" % item, now)
    cache_new = {}

    def is_cache_new(node):
        if node is None:
            return True
        if node not in cache_new:
            path = cachefile_path(node)
            if not os.path.exists(path):
                raise MKGeneralException("cache not found: %s" % path)
            else:
                cache_new[node] = os.stat(path).st_mtime > last_run
        return cache_new[node]

    if len(info) == 1:
        line = " ".join(info[0][1:])
        if line.startswith("CANNOT READ CONFIG FILE"):
            return 3, "Error in agent configuration: %s" % " ".join(info[0][4:])

    found = False
    right_block = False
    loglines = []
    for l in info:
        node = l[0]
        line = " ".join(l[1:])
        if line == "[[[%s]]]" % item:
            found = right_block = True
        elif len(line) > 6 and line[0:3] == "[[[" and line[-3:] == "]]]":
            right_block = False
        elif right_block and is_cache_new(node):
            loglines.append(line)

    return check_logwatch_generic(item, params, loglines, found)


check_info['logwatch'] = {
    'check_function'      : check_logwatch,
    'inventory_function'  : inventory_logwatch,
    'service_description' : "Log %s",
    'node_info'           : True,
    'group'               : 'logwatch',
}


precompile_params['logwatch'] = logwatch_precompile


#   .--lw.groups-----------------------------------------------------------.
#   |              _                                                       |
#   |             | |_      ____ _ _ __ ___  _   _ _ __  ___               |
#   |             | \ \ /\ / / _` | '__/ _ \| | | | '_ \/ __|              |
#   |             | |\ V  V / (_| | | | (_) | |_| | |_) \__ \              |
#   |             |_| \_/\_(_)__, |_|  \___/ \__,_| .__/|___/              |
#   |                        |___/                |_|                      |
#   +----------------------------------------------------------------------+
#   |                                                                      |
#   '----------------------------------------------------------------------'


def logwatch_group_precompile(hostname, item, _unused):
    return logwatch_precompile(hostname, item, None), host_extra_conf(hostname, logwatch_groups)


def logwatch_groups_of_logfile(filename, params=False):
    groups = []
    if not params:
        params = host_extra_conf(g_hostname, logwatch_groups)
    else:
        params = params[1]

    for line in params:
        for group_name, pattern in line:
            inclusion, exclusion = pattern

            inclusion_is_regex = False
            exclusion_is_regex = False
            if inclusion.startswith("~"):
                inclusion_is_regex = True
                inclusion = inclusion[1:]
            if exclusion.startswith("~"):
                exclusion_is_regex = True
                exclusion = exclusion[1:]

            if inclusion_is_regex:
                reg = regex(inclusion)
                incl_match = reg.match( filename )
            else:
                incl_match = fnmatch.fnmatch( filename, inclusion )

            if exclusion_is_regex:
                reg = regex(exclusion)
                excl_match = reg.match( filename )
            else:
                excl_match = fnmatch.fnmatch( filename, exclusion )

            if incl_match and not excl_match:
                groups.append(group_name)

    return groups


def inventory_logwatch_groups(info):
    forwarded_logs, not_forwarded_logs = logwatch_select_forwarded(info)
    added_groups = []
    inventory = []
    for logfile in not_forwarded_logs:
        groups = logwatch_groups_of_logfile(logfile)
        for group in groups:
            if group not in added_groups:
                added_groups.append(group)
                inventory.append((group, None))
    return inventory


def check_logwatch_groups(item, params, info):
    if len(info) == 1:
        line = " ".join(info[0][1:])
        if line.startswith("CANNOT READ CONFIG FILE"):
            return 3, "Error in agent configuration: %s" % " ".join(info[0][4:])

    found = False
    logfile_found = False
    loglines = []
    for l in info:
        # node info ignored (only used in regular logwatch check)
        line = " ".join(l[1:])
        if logfile_found == True and not line.startswith('[[['):
            loglines.append(line)
        if line.startswith('[[['):
            logfile = line[3:-3]
            if item in logwatch_groups_of_logfile(logfile, params):
                found = True
                logfile_found = True
            else:
                logfile_found = False
            continue
    return check_logwatch_generic(item, params, loglines, found, True)


check_info['logwatch.groups'] = {
    'check_function'      : check_logwatch_groups,
    'inventory_function'  : inventory_logwatch_groups,
    'service_description' : "Log %s",
    'node_info'           : True,
    'group'               : 'logwatch',
}


precompile_params['logwatch.groups'] = logwatch_group_precompile


#.


# truncate a file near the specified offset while keeping lines intact
def truncate_by_line(filename, offset):
    f = open(filename, 'r+')
    f.seek(offset)
    f.readline()  # ensures we don't cut inside a line
    f.truncate()
    f.close()


def logwatch_username():
    import getpass
    return getpass.getuser()


def check_logwatch_generic(item, params, loglines, found, groups=False):
    logdir = logwatch_dir + "/" + g_hostname

    # Create directories, if neccessary
    try:
        os.makedirs(logdir)
    except OSError, e:
        if e.errno == 17:
            pass # Exists
        else:
            raise

    logfile = logdir + "/" + item.replace("/", "\\")

    # Logfile (=item) section not found and no local file found. This usually
    # means, that the corresponding logfile also vanished on the target host.
    if type(logfile) == unicode:
        logfile = logfile.encode("utf-8")

    if not found and not os.path.exists(logfile):
        return (3, "log not present anymore")

    # Get the patterns (either compile or reuse the precompiled ones)
    # Check_MK creates an empty string if the precompile function has
    # not been executed yet. The precompile function creates an empty
    # list when no ruless/patterns are defined. In case of the logwatch.groups
    # checks, params are a tuple with the normal logwatch parameters on the first
    # and the grouping patterns on the second position
    if params not in ('', None):
        if groups:
            patterns = params[0]
        else:
            patterns = params # patterns already precompiled
    else:
        patterns = logwatch_precompile(g_hostname, item, None)


    state_counts = {} # for counting number of blocks with a certain state

    class LogwatchBlock(object):
        def __init__(self, header):
            self.worst  = -1
            self.header = header
            self.lines  = []
            self.last_worst_line = ''
            self.counts = {} # for counting number of matches of a certain pattern

        def __reclassify_line(self, patterns, text, level):
            if patterns:
                newlevel = logwatch_reclassify(self.counts, patterns, text, level)
                if newlevel != None:
                    level = newlevel
            return level

        def finalize(self):
            start = " ".join(self.header.split(' ')[0:2])
            return [ start + " %s>>>\n" % logwatch_state(self.worst) ] + self.lines

        def add_line(self, line, skip_reclassification):
            old_level, text = logwatch_parse_line(line)

            if skip_reclassification:
                level = old_level
            else:
                level = self.__reclassify_line(patterns, text, old_level)

            self.worst = logwatch_level_worst(self.worst, level)

            # Save the last worst line of this block
            if logwatch_level_worst(0, level) == self.worst:
                self.last_worst_line = text

            # Count the number of lines by state
            if level != '.':
                state_counts[level] = state_counts.get(level, 0) + 1

            if not skip_reclassification and level != "I":
                self.lines.append(level + " " + text + "\n")

    def collect_block(block):
        if block and block.worst > -1:
            collect_block.output_lines += block.finalize()

            if block.worst >= collect_block.worst:
                collect_block.worst = block.worst
                collect_block.last_worst_line = block.last_worst_line

    collect_block.worst = 0
    collect_block.last_worst_line = ''
    collect_block.output_lines = []

    current_block = None

    log_exists = os.path.exists(logfile)
    try:
        if log_exists:
            logwatch_file = open(logfile, 'r+')
        else:
            logwatch_file = open(logfile, 'w')
    except Exception, e:
        raise MKGeneralException("User %s cannot open file for writing: %s" % (logwatch_username(), e))

    pattern_hash = hash(tuple(patterns))

    net_lines = 0

    # parse cached log lines
    if log_exists:
        # new format contains hash of patterns on the first line so we only reclassify if they
        # changed
        initline = logwatch_file.readline().rstrip('\n')
        if initline.startswith('[[[') and initline.endswith(']]]'):
            old_pattern_hash = int(initline[3:-3])
            skip_reclassification = old_pattern_hash == pattern_hash
        else:
            logwatch_file.seek(0)
            skip_reclassification = False

        logfile_size = os.path.getsize(logfile)
        if skip_reclassification and logfile_size > logwatch_max_filesize:
            # early out: without reclassification the file wont shrink and if it is already at
            # the maximum size, all input is dropped anyway
            if logfile_size > logwatch_max_filesize * 2:
                # if the file is far too large, truncate it
                truncate_by_line(logfile, logwatch_max_filesize)
            return (2, "unacknowledged messages have exceeded max size, "
                    "new messages are dropped (limit %d Bytes)" % logwatch_max_filesize)

        for line in logwatch_file:
            line = line.rstrip('\n')
            # Skip empty lines
            if not line:
                continue
            elif line.startswith('<<<') and line.endswith('>>>'):
                # The section is finished here. Add it to the list of reclassified lines if the
                # state of the block is not "I" -> "ignore"
                collect_block(current_block)
                current_block = LogwatchBlock(line)
            elif current_block is not None:
                current_block.add_line(line, skip_reclassification)
                net_lines += 1

        # The last section is finished here. Add it to the list of reclassified lines if the
        # state of the block is not "I" -> "ignore"
        collect_block(current_block)

        if skip_reclassification:
            output_size = logwatch_file.tell()
            # when skipping reclassification, output lines contains only headers anyway
            collect_block.output_lines = []
        else:
            output_size = sum([len(line) for line in collect_block.output_lines])
    else:
        output_size = 0
        skip_reclassification = False

    header = time.strftime("<<<%Y-%m-%d %H:%M:%S UNKNOWN>>>\n")
    output_size += len(header)

    # process new input lines - but only when there is some room left in the file
    if output_size < logwatch_max_filesize:
        current_block = LogwatchBlock(header)
        for line in loglines:
            current_block.add_line(line.encode("utf-8"), False)
            net_lines += 1
            output_size += len(line)
            if output_size >= logwatch_max_filesize:
                break
        collect_block(current_block)

    # when reclassifying, rewrite the whole file, outherwise append
    if not skip_reclassification and collect_block.output_lines:
        logwatch_file.seek(0)
        logwatch_file.truncate()
        logwatch_file.write("[[[%d]]]\n" % pattern_hash)

    if collect_block.output_lines:
        logwatch_file.writelines(collect_block.output_lines)
    # correct output size
    logwatch_file.close()
    if net_lines == 0 and log_exists:
        os.unlink(logfile)

    # if logfile has reached maximum size, abort with critical state
    if os.path.exists(logfile) and os.path.getsize(logfile) > logwatch_max_filesize:
        return (2, "unacknowledged messages have exceeded max size, "
                   "new messages are lost (limit %d Bytes)" % logwatch_max_filesize)

    #
    # Render output
    #

    if collect_block.worst <= 0:
        return (0, "no error messages")
    else:
        count_txt = []
        for level, num in state_counts.iteritems():
            count_txt.append('%d %s' % (num, logwatch_level_name(level)))
        if logwatch_service_output == 'default':
            return (collect_block.worst, "%s messages (Last worst: \"%s\")" %
                                           (', '.join(count_txt), collect_block.last_worst_line))
        else:
            return (collect_block.worst, "%s messages" % ', '.join(count_txt))

#.
#   .--Event Console Forwarding--------------------------------------------.
#   |        _                           _       _                         |
#   |       | | ___   __ ___      ____ _| |_ ___| |__    ___  ___          |
#   |       | |/ _ \ / _` \ \ /\ / / _` | __/ __| '_ \  / _ \/ __|         |
#   |       | | (_) | (_| |\ V  V / (_| | || (__| | | ||  __/ (__          |
#   |       |_|\___/ \__, | \_/\_/ \__,_|\__\___|_| |_(_)___|\___|         |
#   |                |___/                                                 |
#   +----------------------------------------------------------------------+
#   | Forwarding logwatch messages to event console                        |
#   '----------------------------------------------------------------------'


# OK      -> priority 5 (notice)
# WARN    -> priority 4 (warning)
# CRIT    -> priority 2 (crit)
# context -> priority 6 (info)
# u = Uknown
def logwatch_to_prio(level):
    if level == 'W':
        return 4
    elif level == 'C':
        return 2
    elif level == 'O':
        return 5
    elif level == '.':
        return 6
    else:
        return 4


def inventory_logwatch_ec(info, one_svc_per_log):
    forwarded_logs, not_forwarded_logs = logwatch_select_forwarded(info)

    if forwarded_logs:
        forward_settings = host_extra_conf(g_hostname, checkgroup_parameters.get('logwatch_ec', []))

        merged_rules = {}
        for rule in forward_settings[-1::-1]:
            if type(rule) == dict:
                for key, value in rule.items():
                    merged_rules[key] = value
            elif type(rule) == str:
                return # Configured "no forwarding"

        separate_checks = merged_rules.get("separate_checks", False)
        if separate_checks != one_svc_per_log:
            return

        if separate_checks:
            single_log_params = {}
            for key in [ "method", "facility", "monitor_logfilelist", "logwatch_reclassify" ]:
                if key in merged_rules:
                    single_log_params[key] = merged_rules[key]
            for log in forwarded_logs:
                single_log_params["expected_logfiles"] = [log]
                yield log, single_log_params.copy()
        else:
            yield None, { "expected_logfiles": forwarded_logs }


def check_logwatch_ec(item, params, info):
    if len(info) == 1:
        line = " ".join(info[0][1:])
        if line.startswith("CANNOT READ CONFIG FILE"):
            yield 3, "Error in agent configuration: %s" % " ".join(info[0][5:])
            return

    # 1. Parse lines in info and separate by logfile
    logs = {}
    logfile = None
    for l in info:
        # node info ignored (only used in regular logwatch check)
        line = " ".join(l[1:])
        if len(line) > 6 and line[0:3] == "[[[" and line[-3:] == "]]]":
            # new logfile, extract name
            logfile = line[3:-3]
            logs.setdefault(logfile, [])

        elif logfile and line:
            # new regular line, skip context lines and ignore lines
            if line[0] not in ['.', 'I']:
                logs[logfile].append(line)

    # 2. Maybe filter logfiles if some should be excluded
    if 'restrict_logfiles' in params:
        for logfile in logs.keys():
            if not logwatch_ec_forwarding_enabled(params, logfile):
                del logs[logfile]

    # 3. If this check has an item (logwatch.ec_single), only forward the information from this log
    if item:
        if item not in logs.keys():
            return
        else:
            single_logs = {}
            single_logs[item] = logs[item]
            logs = single_logs

    # Check if the number of expected files matches the actual one
    if params.get('monitor_logfilelist'):
        if 'expected_logfiles' not in params:
            yield 1, "You enabled monitoring the list of forwarded logfiles. " \
                     "You need to redo service discovery."
        else:
            expected = params['expected_logfiles']
            missing = []
            for f in expected:
                if f not in logs:
                    missing.append(f)
            if missing:
                yield 1, "Missing logfiles: %s" % (", ".join(missing))

            exceeding = []
            for f in logs:
                if f not in expected:
                    exceeding.append(f)
            if exceeding:
                yield 1, "Newly appeared logfiles: %s" % (", ".join(exceeding))

    # 3. create syslog message of each line
    # <128> Oct 24 10:44:27 Klappspaten /var/log/syslog: Oct 24 10:44:27 Klappspaten logger: asdasdad as
    # <facility+priority> timestamp hostname logfile: message
    facility = params.get('facility', 17) << 3 # default to "local1"
    messages = []
    cur_time = int(time.time())

    forwarded_logfiles = set([])

    # Get the logwatch patterns if they are not already precompiled
    # The precompile feature is not required when using the cmc as core
    # HACK: opt_keepalive is used to detect cmc
    if not opt_keepalive and "logwatch_settings" not in params:
        params = logwatch_ec_precompile(g_hostname, item, params)

    # Keep track of reclassifed lines
    rclfd_total     = 0
    rclfd_to_ignore = 0

    logfile_reclassify_settings = {}
    def add_reclassify_settings(settings):
        if isinstance(settings, dict):
            logfile_reclassify_settings["reclassify_patterns"].extend(x[:2] for x in settings.get("reclassify_patterns"))
            if "reclassify_states" in settings:
                logfile_reclassify_settings["reclassify_states"] = settings["reclassify_states"]
        else: # legacy configuration
            logfile_reclassify_settings["reclassify_patterns"].extend(x[:2] for x in settings)

    for logfile, lines in logs.items():
        logfile_reclassify_settings["reclassify_patterns"] = []
        logfile_reclassify_settings["reclassify_states"]   = {}
        # Determine logwatch patterns specifically for this logfile
        # HACK: opt_keepalive is used to detect cmc
        if opt_keepalive:
            logfile_settings = service_extra_conf(g_hostname, logfile, logwatch_rules)
            for settings in logfile_settings:
                add_reclassify_settings(settings)
        else:
            for entry in params["logwatch_settings"]:
                settings, log_items = entry
                for log_item in log_items:
                    reg = regex(log_item)
                    if reg.search(logfile):
                        add_reclassify_settings(settings)

        for line in lines:
            rclfd_level = None
            if logfile_reclassify_settings:
                counts = {} # unused...
                old_level, text = line.split(" ", 1)
                level = logwatch_reclassify(counts, logfile_reclassify_settings, line[2:], old_level) or old_level
                if level != old_level:
                    rclfd_total += 1
                    rclfd_level = level
                    if level == "I": # Ignored lines are not forwarded
                        rclfd_to_ignore += 1
                        continue

            msg = '<%d>' % (facility + logwatch_to_prio(rclfd_level or line[0]),)
            msg += '@%s;%d;; %s %s: %s' % (cur_time, params.get("service_level", 0), g_hostname, logfile, line[2:])

            messages.append(msg)
            forwarded_logfiles.add(logfile)

    try:
        if forwarded_logfiles:
            logfile_info = " from " + ",".join(list(forwarded_logfiles))
        else:
            logfile_info = ""

        result = logwatch_forward_messages(params.get("method"), item, messages)

        yield 0, "Forwarded %d messages%s" % (result.num_forwarded, logfile_info), \
                            [('messages', result.num_forwarded)]

        exc_txt = " (%s)" % result.exception if result.exception else ""

        if result.num_spooled:
            yield 1, "Spooled %d messages%s" % (result.num_spooled, exc_txt)

        if result.num_dropped:
            yield 2, "Dropped %d messages%s" % (result.num_dropped, exc_txt)

    except Exception, e:
        if cmk.debug.enabled():
            raise
        yield (2, 'Failed to forward messages (%s). Lost %d messages.' %
                (e, len(messages)))


    if rclfd_total:
        yield 0, 'Reclassified %d messages through logwatch patterns (%d to IGNORE)' % \
                             (rclfd_total, rclfd_to_ignore)


class LogwatchFordwardResult(object):
    def __init__(self, num_forwarded=0, num_spooled=0, num_dropped=0, exception=None):
        self.num_forwarded = num_forwarded
        self.num_spooled   = num_spooled
        self.num_dropped   = num_dropped
        self.exception     = exception


# send messages to event console
# a) local in same omd site
# b) local pipe
# c) remote via udp
# d) remote via tcp
def logwatch_forward_messages(method, item, messages):
    if not method:
        method = os.getenv('OMD_ROOT') + "/tmp/run/mkeventd/eventsocket"
    elif method == 'spool:':
        method += os.getenv('OMD_ROOT') + "/var/mkeventd/spool"

    if isinstance(method, tuple):
        return logwatch_forward_tcp(method, messages)

    elif not method.startswith('spool:'):
        return logwatch_forward_pipe(method, messages)

    else:
        return logwatch_forward_spool_directory(method, item, messages)


# write into local event pipe
# Important: When the event daemon is stopped, then the pipe
# is *not* existing! This prevents us from hanging in such
# situations. So we must make sure that we do not create a file
# instead of the pipe!
def logwatch_forward_pipe(method, messages):
    if not messages:
        return LogwatchFordwardResult()

    sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
    sock.connect(method)
    sock.send(('\n'.join(messages)).encode("utf-8") + '\n')
    sock.close()

    return LogwatchFordwardResult(num_forwarded=len(messages))


# Spool the log messages to given spool directory.
# First write a file which is not read into ec, then
# perform the move to make the file visible for ec
def logwatch_forward_spool_directory(method, item, messages):
    if not messages:
        return LogwatchFordwardResult()

    spool_path = method[6:]
    file_name  = '.%s_%s%d' % (g_hostname, item and item.replace('/', '\\') + '_' or '', time.time())
    if not os.path.exists(spool_path):
        os.makedirs(spool_path)
    file('%s/%s' % (spool_path, file_name), 'w').write(('\n'.join(messages)).encode("utf-8") + '\n')
    os.rename('%s/%s' % (spool_path, file_name), '%s/%s' % (spool_path, file_name[1:]))

    return LogwatchFordwardResult(num_forwarded=len(messages))


def logwatch_forward_tcp(method, new_messages):
    # Transform old format: (proto, address, port)
    if type(method[1]) != dict:
        method = (method[0], {"address": method[1], "port": method[2]})

    result = LogwatchFordwardResult()

    message_chunks = []

    if logwatch_shall_spool_messages(method):
        message_chunks += logwatch_load_spooled_messages(method, result)

    # Add chunk of new messages (when there are new ones)
    if new_messages:
        message_chunks.append((time.time(), 0, new_messages))

    if not message_chunks:
        return result # Nothing to process

    try:
        logwatch_forward_send_tcp(method, message_chunks, result)
    except Exception, e:
        result.exception = e

    if logwatch_shall_spool_messages(method):
        logwatch_spool_messages(message_chunks, result)
    else:
        result.num_dropped = sum([ len(c[2]) for c in message_chunks ])

    return result


def logwatch_shall_spool_messages(method):
    return type(method) == tuple and method[0] == "tcp" \
            and type(method[1]) == dict and "spool" in method[1]


def logwatch_forward_send_tcp(method, message_chunks, result):
    protocol, method_params = method

    if protocol == 'udp':
        sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    elif protocol == 'tcp':
        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    else:
        raise NotImplementedError()

    sock.connect((method_params["address"], method_params["port"]))

    try:
        for _time_spooled, _num_spooled, message_chunk in message_chunks:
            while message_chunk:
                try:
                    message = message_chunk[0]
                except IndexError:
                    break # chunk complete

                sock.send(message.encode("utf-8") + "\n")
                message_chunk.pop(0) # remove sent message
                result.num_forwarded += 1
    except Exception, e:
        result.exception = e
    finally:
        sock.close()


# a) Rewrite chunks that have been processed partially
# b) Write files for new chunk
def logwatch_spool_messages(message_chunks, result):
    path = logwatch_spool_path()

    try:
        os.makedirs(path)
    except OSError, e:
        if e.errno == 17:
            pass # Exists
        else:
            raise

    # Now write updated/new and delete emtpy spool files
    for time_spooled, num_already_spooled, message_chunk in message_chunks:
        spool_file_path = "%s/spool.%0.2f" % (path, time_spooled)

        if not message_chunk:
            # Cleanup empty spool files
            try:
                os.unlink(spool_file_path)
                continue
            except OSError, e:
                if e.errno == 2:
                    continue
                else:
                    raise

        try:
            # Partially processed chunks or the new one
            with open(spool_file_path, "w") as f:
                f.write(repr(message_chunk))

            result.num_spooled += len(message_chunk)
        except:
            if cmk.debug.enabled():
                raise

            if num_already_spooled == 0:
                result.num_dropped += len(message_chunk)


def logwatch_load_spooled_messages(method, result):
    import ast

    spool_params = method[1]["spool"]

    try:
        spool_files = sorted(os.listdir(logwatch_spool_path()))
    except OSError, e:
        if e.errno == 2: # No such file or directory
            return []
        else:
            raise

    message_chunks = []

    total_size = 0
    for filename in spool_files:
        path = logwatch_spool_path() + "/" + filename

        # Delete unknown files
        if not filename.startswith("spool."):
            os.unlink(path)
            continue

        time_spooled = float(filename[6:])
        file_size = os.stat(path).st_size
        total_size += file_size

        # Delete fully processed files
        if file_size in [ 0, 2 ]:
            os.unlink(path)
            continue

        # Delete too old files by age
        if time_spooled < time.time() - spool_params["max_age"]:
            logwatch_spool_drop_messages(path, result)
            continue

    # Delete by size till half of target size has been deleted (oldest spool files first)
    if total_size > spool_params["max_size"]:
        target_size = spool_params["max_size"]/2

        for filename in spool_files:
            path = logwatch_spool_path() + "/" + filename

            total_size -= logwatch_spool_drop_messages(path, result)
            if target_size >= total_size:
                break # cleaned up enough

    # Now process the remaining files
    for filename in spool_files:
        path = logwatch_spool_path() + "/" + filename
        time_spooled = float(filename[6:])

        try:
            messages = ast.literal_eval(open(path).read())
        except IOError, e:
            if e.errno == 2: # No such file or directory
                continue
            else:
                raise

        message_chunks.append((time_spooled, len(messages), messages))

    return message_chunks


def logwatch_spool_drop_messages(path, result):
    import ast
    messages = ast.literal_eval(open(path).read())
    result.num_dropped += len(messages)

    file_size = os.stat(path).st_size
    os.unlink(path)
    return file_size


def logwatch_spool_path():
    return logwatch_spool_dir + "/" + g_hostname


def logwatch_ec_precompile(hostname, item, params):
    params = params.copy()

    params["service_level"] = get_effective_service_level()

    # HACK: opt_keepalive is used to detect cmc
    if (opt_keepalive or not params.get("logwatch_reclassify")):
        params.update({"logwatch_settings": []})
        return params

    # The following code is used for creating the params of precompiled checks
    # and when Check_MK is called on the command line.
    # Check_MK in keepalive mode (in a running cmc) should never reach this code...
    ruleset = []
    for rule in logwatch_rules:
        rule, rule_options = get_rule_options(rule)
        if rule_options.get("disabled"):
            continue

        if len(rule) == 3:
            settings, hostlist, servlist = rule
            tags = []
        elif len(rule) == 4:
            settings, tags, hostlist, servlist = rule
        else:
            continue

        # Directly compute set of all matching hosts here, this
        # will avoid recomputation later
        hosts = all_matching_hosts(tags, hostlist, with_foreign_hosts=False)
        ruleset.append((settings, hosts, servlist))

    # Filter out any logwatch_rules which do apply to to this host
    # 1st filter: Do not use rules where the hostname does not match
    # 2nd filter: Do not use rules with configured items where no item matches the
    #             "restrict_logfiles" condition (if applicable)
    logwatch_settings = []
    for rule in ruleset:
        settings, hosts, rule_items = rule
        if hostname in hosts:
            if params.get("restrict_logfiles"):
                for rule_item in rule_items:
                    if rule_item == "" or logwatch_ec_forwarding_enabled(params, rule_item):
                        logwatch_settings.append((settings, rule_items))
                        break
            else:
                logwatch_settings.append((settings, rule_items))

    params.update({"logwatch_settings": logwatch_settings})

    return params


precompile_params['logwatch.ec'] = logwatch_ec_precompile
precompile_params['logwatch.ec_single'] = logwatch_ec_precompile


check_info['logwatch.ec'] = {
    'check_function'      : check_logwatch_ec,
    'inventory_function'  : lambda info: inventory_logwatch_ec(info, False),
    'service_description' : "Log Forwarding",
    'group'               : 'logwatch_ec',
    'node_info'           : True,
    'has_perfdata'        : True,
}


check_info['logwatch.ec_single'] = {
    'check_function'      : check_logwatch_ec,
    'inventory_function'  : lambda info: inventory_logwatch_ec(info, True),
    'service_description' : "Log %s",
    'node_info'           : True,
    'has_perfdata'        : True,
}
