#!/usr/bin/python
# -*- encoding: utf-8; py-indent-offset: 4 -*-
# +------------------------------------------------------------------+
# |             ____ _               _        __  __ _  __           |
# |            / ___| |__   ___  ___| | __   |  \/  | |/ /           |
# |           | |   | '_ \ / _ \/ __| |/ /   | |\/| | ' /            |
# |           | |___| | | |  __/ (__|   <    | |  | | . \            |
# |            \____|_| |_|\___|\___|_|\_\___|_|  |_|_|\_\           |
# |                                                                  |
# | Copyright Mathias Kettner 2014             mk@mathias-kettner.de |
# +------------------------------------------------------------------+
#
# This file is part of Check_MK.
# The official homepage is at http://mathias-kettner.de/check_mk.
#
# check_mk is free software;  you can redistribute it and/or modify it
# under the  terms of the  GNU General Public License  as published by
# the Free Software Foundation in version 2.  check_mk is  distributed
# in the hope that it will be useful, but WITHOUT ANY WARRANTY;  with-
# out even the implied warranty of  MERCHANTABILITY  or  FITNESS FOR A
# PARTICULAR PURPOSE. See the  GNU General Public License for more de-
# tails. You should have  received  a copy of the  GNU  General Public
# License along with GNU Make; see the file  COPYING.  If  not,  write
# to the Free Software Foundation, Inc., 51 Franklin St,  Fifth Floor,
# Boston, MA 02110-1301 USA.


# Example output from agent:
# <<<jolokia_metrics>>>
# 8080 NonHeapMemoryUsage 101078952
# 8080 NonHeapMemoryMax 184549376
# 8080 HeapMemoryUsage 2362781664
# 8080 HeapMemoryMax 9544663040
# 8080 ThreadCount 78
# 8080 DeamonThreadCount 72
# 8080 PeakThreadCount 191
# 8080 TotalStartedThreadCount 941
# 8080 Uptime 572011375
# 8080,java.lang:name=PS_MarkSweep,type=GarbageCollector CollectionCount 0

# MB warn, crit
# jolokia_metrics_mem_default_levels = (2000, 3000)

# Number of threads warn, crit
jolokia_metrics_threads_default_levels  = (80, 100)

# Number of sessions low crit, low warn, high warn, high crit
jolokia_metrics_app_sess_default_levels = (-1, -1, 800, 1000)

# Number of requests low crit, low warn, high warn, high crit
jolokia_metrics_serv_req_default_levels = (-1, -1, 5000, 6000)

jolokia_metrics_queue_default_levels = ( 20, 50 )

# Tomcat ThreadPools Count/Busy in relation to max value
factory_settings["jolokia_metrics_tp_default_levels"] = {
            'currentThreadsBusy': (80, 90),
}


#   .--Parse function------------------------------------------------------.
#   |  ____                        __                  _   _               |
#   | |  _ \ __ _ _ __ ___  ___   / _|_   _ _ __   ___| |_(_) ___  _ __    |
#   | | |_) / _` | '__/ __|/ _ \ | |_| | | | '_ \ / __| __| |/ _ \| '_ \   |
#   | |  __/ (_| | |  \__ \  __/ |  _| |_| | | | | (__| |_| | (_) | | | |  |
#   | |_|   \__,_|_|  |___/\___| |_|  \__,_|_| |_|\___|\__|_|\___/|_| |_|  |
#   |                                                                      |
#   '----------------------------------------------------------------------'


def jolokia_metrics_parse(info):
    parsed = {}
    for line in info:
        if line[1] == "ERROR":
            parsed[line[0]] = None # error
            continue

        inst, var, value = line
        module  = None
        attributes = {}
        positional = []

        # app, servlet, gc, tp = None, None, None, None
        if ',' in inst:
            parts = inst.split(',')
            inst = parts[0]

            for part in parts[1:]:
                if ":" in part:
                    module, part = part.split(":")
                if "=" in part:
                    key, val = part.split("=")
                    attributes[key] = val
                else:
                    positional.append(part)

        parsed.setdefault(inst, {})

        if 'type' in attributes:
            bean_name = attributes['name']
            bean_type = attributes['type']
            # backwards compatibility
            bean_type = {
                "GarbageCollector": "gc",
                "ThreadPool": "tp"
            }.get(bean_type, bean_type)
            # maybe do this for all types?
            if bean_type == "tp":
                bean_name = bean_name.replace('"', '')

            parsed[inst].setdefault(bean_type, {}).setdefault(bean_name, {})
            parsed[inst][bean_type][bean_name][var] = value
            for attr_key, attr_value in attributes.iteritems():
                if attr_key not in set(['name', 'type']):
                    parsed[inst][bean_type][bean_name][attr_key] = attr_value
        else:
            if len(positional) > 0:
                app = positional[0]
                parsed[inst].setdefault('apps', {}).setdefault(app, {})
                if len(positional) > 1:
                    servlet = positional[1]
                    parsed[inst]['apps'][app].setdefault('servlets', {}).setdefault(servlet, {})
                    parsed[inst]['apps'][app]['servlets'][servlet][var] = value
                else:
                    parsed[inst]['apps'][app][var] = value
            else:
                parsed[inst][var] = value
    return parsed


#.
#   .--Generic inventory functions-----------------------------------------.
#   |                   ____                      _                        |
#   |                  / ___| ___ _ __   ___ _ __(_) ___                   |
#   |                 | |  _ / _ \ '_ \ / _ \ '__| |/ __|                  |
#   |                 | |_| |  __/ | | |  __/ |  | | (__                   |
#   |                  \____|\___|_| |_|\___|_|  |_|\___|                  |
#   |                                                                      |
#   |             _                      _                                 |
#   |            (_)_ ____   _____ _ __ | |_ ___  _ __ _   _               |
#   |            | | '_ \ \ / / _ \ '_ \| __/ _ \| '__| | | |              |
#   |            | | | | \ V /  __/ | | | || (_) | |  | |_| |              |
#   |            |_|_| |_|\_/ \___|_| |_|\__\___/|_|   \__, |              |
#   |                                                  |___/               |
#   |              __                  _   _                               |
#   |             / _|_   _ _ __   ___| |_(_) ___  _ __  ___               |
#   |            | |_| | | | '_ \ / __| __| |/ _ \| '_ \/ __|              |
#   |            |  _| |_| | | | | (__| |_| | (_) | | | \__ \              |
#   |            |_|  \__,_|_| |_|\___|\__|_|\___/|_| |_|___/              |
#   |                                                                      |
#   '----------------------------------------------------------------------'

def inventory_jolokia_metrics(info, what):
    parsed = jolokia_metrics_parse(info)

    levels = None

    if what == 'mem':
        levels = {}
    elif what == 'threads':
        levels = 'jolokia_metrics_threads_default_levels'

    for instance, data in parsed.items():
        if data == None:
            continue # No connection to agent currently

        if what == 'uptime' and "Uptime" not in data:
            continue
        if what == 'mem' and ("HeapMemoryUsage" not in data or "NonHeapMemoryUsage" not in data
                              or "HeapMemoryMax" not in data or "NonHeapMemoryMax" not in data):
            # don't add memory check if we don't have the necessary data
            continue
        yield instance, levels


def inventory_jolokia_metrics_apps(info, what):
    inv = []
    parsed = jolokia_metrics_parse(info)

    if what == 'app_sess':
        levels = 'jolokia_metrics_app_sess_default_levels'
        needed_key = [ "Sessions", "activeSessions" ]
    elif what  == 'bea_app_sess':
        levels = 'jolokia_metrics_app_sess_default_levels'
        needed_key = [ "OpenSessionsCurrentCount" ]
    elif what == 'queue':
        needed_key = [ "QueueLength" ]
        levels = "jolokia_metrics_queue_default_levels"
    # Only works on BEA
    elif what == 'bea_requests':
        needed_key = [ "CompletedRequestCount" ]
        levels = None
    elif what == 'requests':
        needed_key = [ "requestCount" ]
        levels = None
    elif what == 'threads':
        needed_key = [ "StandbyThreadCount" ]
        levels = None
    else:
        needed_key = [ "Running", "stateName" ]
        levels = None

    # this handles information from BEA, they stack one level
    # higher than the rest.
    if what == 'bea_app_sess':
        for inst, vals in parsed.iteritems():
            if vals == None:
                continue # no data from agent

            for app, appstate in vals.get('apps', {}).items():
                if 'servlets' in appstate:
                    for nk in needed_key:
                        for servlet in appstate['servlets']:
                            if nk in appstate['servlets'][servlet]:
                                inv.append(('%s %s %s' % (inst, app, servlet), levels))
                                continue
    # This does the same for tomcat
    for inst, vals in parsed.iteritems():
        if vals == None:
            continue # no data from agent

        for app, appstate in vals.get('apps', {}).items():
            for nk in needed_key:
                if nk in appstate:
                    inv.append(('%s %s' % (inst, app), levels))
                    continue
    return inv


#.
#   .--Arcane helpers------------------------------------------------------.
#   |                     _                                                |
#   |                    / \   _ __ ___ __ _ _ __   ___                    |
#   |                   / _ \ | '__/ __/ _` | '_ \ / _ \                   |
#   |                  / ___ \| | | (_| (_| | | | |  __/                   |
#   |                 /_/   \_\_|  \___\__,_|_| |_|\___|                   |
#   |                                                                      |
#   |                  _          _                                        |
#   |                 | |__   ___| |_ __   ___ _ __ ___                    |
#   |                 | '_ \ / _ \ | '_ \ / _ \ '__/ __|                   |
#   |                 | | | |  __/ | |_) |  __/ |  \__ \                   |
#   |                 |_| |_|\___|_| .__/ \___|_|  |___/                   |
#   |                              |_|                                     |
#   +----------------------------------------------------------------------+
#   | TODO: See if these can be removed altogether                         |
#   '----------------------------------------------------------------------'

# This bisects the app server and its values
def jolokia_metrics_app(info, (inst, app)):
    parsed = jolokia_metrics_parse(info)
    if parsed.get(inst, "") == None:
        raise MKCounterWrapped("No information from Jolokia agent")

    if not inst in parsed \
       or not app in parsed[inst].get('apps', {}):
        return None
    return parsed[inst]['apps'][app]

# This bisects info from BEA and passes on to jolokia_metrics_app
def jolokia_metrics_serv(info, (inst, app, serv)):
    app = jolokia_metrics_app(info, (inst, app))
    if not app or not serv in app.get('servlets', {}):
        return None
    return app['servlets'][serv]

def jolokia_metrics_gc(info, (inst, typ, gc)):
    parsed = jolokia_metrics_parse(info)
    if parsed.get(inst, "") == None:
        raise MKCounterWrapped("No information from Jolokia agent")

    if not inst in parsed \
       or not gc in parsed[inst].get('gc', {}):
        return None
    return parsed[inst]['gc'][gc]

def jolokia_metrics_tp(info, (inst, typ, tp)):
    parsed = jolokia_metrics_parse(info)
    if parsed.get(inst, "") == None:
        raise MKCounterWrapped("No information from Jolokia agent")

    if not inst in parsed \
       or not tp in parsed[inst].get('tp', {}):
        return None
    return parsed[inst]['tp'][tp]


#.
#   .--Number of Requests--------------------------------------------------.
#   |               ____                            _                      |
#   |              |  _ \ ___  __ _ _   _  ___  ___| |_ ___                |
#   |              | |_) / _ \/ _` | | | |/ _ \/ __| __/ __|               |
#   |              |  _ <  __/ (_| | |_| |  __/\__ \ |_\__ \               |
#   |              |_| \_\___|\__, |\__,_|\___||___/\__|___/               |
#   |                            |_|                                       |
#   '----------------------------------------------------------------------'

def inventory_jolokia_metrics_serv(info):
    inv = []
    parsed = jolokia_metrics_parse(info)
    levels = None
    levels = 'jolokia_metrics_serv_req_default_levels'
    needed_key = "Requests"
    for inst, vals in parsed.iteritems():
        if vals == None:
            continue # no data from agent
        for app, val in vals.get('apps', {}).iteritems():
            for serv, servinfo in val.get('servlets', {}).items():
                if needed_key in servinfo:
                    inv.append(('%s %s %s' % (inst, app, serv), levels))
    return inv


def check_jolokia_metrics_serv_req(item, params, info):
    lo_crit, lo_warn, hi_warn, hi_crit = params
    serv = jolokia_metrics_serv(info, item.split())
    if not serv or not 'Requests' in serv:
        return (3, "data not found in agent output")
    req = saveint(serv['Requests'])

    status    = 0
    status_txt = ''
    if lo_crit is not None and req <= lo_crit:
        status = 2
        status_txt = ' (Below or equal %d)' % lo_crit
    elif lo_warn is not None and req <= lo_warn:
        status = 1
        status_txt = ' (Below or equal %d)' % lo_warn
    elif hi_crit is not None and req >= hi_crit:
        status = 2
        status_txt = ' (Above or equal %d)' % hi_crit
    elif hi_warn is not None and req >= hi_warn:
        status = 1
        status_txt = ' (Above or equal %d)' % hi_warn

    output    = ['Requests: %d%s' % (req, status_txt)]
    perfdata  = [('Requests', req, hi_warn, hi_crit)]
    wrapped   = False
    this_time = time.time()
    try:
        rate = get_rate("jolokia_metrics.serv_req.%s" % item, this_time, req)
        output.append('RequestRate: %0.2f' % rate)
        perfdata.append(('RequestRate', rate))
    except MKCounterWrapped:
        wrapped = True

    if wrapped:
        return (status, ', '.join(output))
    else:
        return (status, ', '.join(output), perfdata)


check_info["jolokia_metrics.serv_req"] = {
    "service_description" : "JVM %s Requests",
    "check_function"      : check_jolokia_metrics_serv_req,
    "inventory_function"  : inventory_jolokia_metrics_serv,
    "group"               : "jvm_requests",
    "has_perfdata"        : True,
}

#.
#   .--Garbage collection--------------------------------------------------.
#   |                ____            _                                     |
#   |               / ___| __ _ _ __| |__   __ _  __ _  ___                |
#   |              | |  _ / _` | '__| '_ \ / _` |/ _` |/ _ \               |
#   |              | |_| | (_| | |  | |_) | (_| | (_| |  __/               |
#   |               \____|\__,_|_|  |_.__/ \__,_|\__, |\___|               |
#   |                                            |___/                     |
#   |                        _ _           _   _                           |
#   |               ___ ___ | | | ___  ___| |_(_) ___  _ __                |
#   |              / __/ _ \| | |/ _ \/ __| __| |/ _ \| '_ \               |
#   |             | (_| (_) | | |  __/ (__| |_| | (_) | | | |              |
#   |              \___\___/|_|_|\___|\___|\__|_|\___/|_| |_|              |
#   |                                                                      |
#   '----------------------------------------------------------------------'

def inventory_jolokia_metrics_gc(info):
    inv = []
    parsed = jolokia_metrics_parse(info)
    for inst, vals in parsed.iteritems():
        if vals == None:
            continue # no data from agent
        for gc, val in vals.get('gc', {}).iteritems():
            inv.append(("%s GC %s" % (inst ,gc) , {}))
    return inv


def check_jolokia_metrics_gc(item, params, info):
    gc = jolokia_metrics_gc(info, item.split())
    if gc == None:
        return

    if params == None:
        params = {}


    crate = get_rate("jvm.gc.count.%s" % (item), \
                     time.time(), int(gc['CollectionCount']))
    crate = crate * 60.0

    ctext = ''
    status = 0
    cwarn, ccrit = params.get('CollectionCount', (None, None))
    if cwarn != None and ccrit != None:
        if crate >= int(ccrit):
            status = 2
            ctext = " (Level %s) " % ccrit
        elif crate >= int(cwarn):
            status = 1
            ctext = " (Level %s) " % cwarn

    yield status, "%.2f GC Count/minute%s" % (crate, ctext), \
                    [('CollectionCount', crate, cwarn, ccrit)]

    if 'CollectionTime' in gc:
        twarn, tcrit = params.get('CollectionTime', (None, None))
        trate = get_rate("jvm.gc.time.%s" % (item), \
                         time.time(), int(gc['CollectionTime']))
        trate = trate * 60.0

        ttext = ''
        status = 0
        if twarn != None and tcrit != None:
            if trate >= int(tcrit):
                status = 2
                ttext = "(Level %s) " % tcrit
            elif trate >= int(twarn):
                status = 1
                ttext = "(Level %s) " % twarn

        yield status, "%.2f GC ms/minute%s" % (trate, ttext), \
                            [('CollectionTime', trate, twarn, tcrit)]


check_info["jolokia_metrics.gc"] = {
    "service_description"       : "JVM %s",
    "check_function"            : check_jolokia_metrics_gc,
    "inventory_function"        : inventory_jolokia_metrics_gc,
    "group"                     : "jvm_gc",
    "has_perfdata"              : True,
}

#.
#   .--Tomcat thread pool--------------------------------------------------.
#   | _____                         _     _   _                        _   |
#   ||_   _|__  _ __ ___   ___ __ _| |_  | |_| |__  _ __ ___  __ _  __| |  |
#   |  | |/ _ \| '_ ` _ \ / __/ _` | __| | __| '_ \| '__/ _ \/ _` |/ _` |  |
#   |  | | (_) | | | | | | (_| (_| | |_  | |_| | | | | |  __/ (_| | (_| |  |
#   |  |_|\___/|_| |_| |_|\___\__,_|\__|  \__|_| |_|_|  \___|\__,_|\__,_|  |
#   |                                                                      |
#   |                                           _                          |
#   |                         _ __   ___   ___ | |                         |
#   |                        | '_ \ / _ \ / _ \| |                         |
#   |                        | |_) | (_) | (_) | |                         |
#   |                        | .__/ \___/ \___/|_|                         |
#   |                        |_|                                           |
#   '----------------------------------------------------------------------'

def inventory_jolokia_metrics_tp(info):
    inv = []
    parsed = jolokia_metrics_parse(info)
    for inst, vals in parsed.iteritems():
        if vals == None:
            continue # no data from agent
        for tp, val in vals.get('tp', {}).iteritems():
            inv.append(("%s ThreadPool %s" % (inst ,tp) , {}))
    return inv


def check_jolokia_metrics_tp(item, params, info):

    if params == None:
        params = {}

    tp = jolokia_metrics_tp(info, item.split())

    if tp == None or "maxThreads" not in tp:
        return

    max_threads = float(tp["maxThreads"])

    def check_thread_levels(what):
        threads_abs = int(tp[what])
        threads_rel = threads_abs / max_threads
        infotext = "%s: %d (%.1f%%)" % (what, threads_abs, threads_rel)
        status = 0

        _params = params.get(what)
        if _params and _params[0] != None and _params[1] != None:
            warn, crit = params[what]
            levelstext = " (warn/crit at %d/%d%%)" % (warn, crit)
            if threads_rel >= crit:
                status = 2
                infotext += levelstext
            elif threads_rel >= warn:
                status = 1
                infotext += levelstext
        else:
            warn, crit = ("", "")
        perfdata = [ (what, threads_abs, warn, crit, 0, max_threads) ]
        return status, infotext, perfdata

    if 'currentThreadsBusy' in tp:
        yield check_thread_levels('currentThreadsBusy')
    if 'currentThreadCount' in tp:
        yield check_thread_levels('currentThreadCount')
    yield 0, "Maximum threads: %d" % max_threads


check_info["jolokia_metrics.tp"] = {
    "default_levels_variable"   : "jolokia_metrics_tp_default_levels",
    "service_description"       : "JVM %s",
    "check_function"            : check_jolokia_metrics_tp,
    "inventory_function"        : inventory_jolokia_metrics_tp,
    "group"                     : "jvm_tp",
    "has_perfdata"              : True,
}

#.
#   .--Memory--------------------------------------------------------------.
#   |               __  __                                                 |
#   |              |  \/  | ___ _ __ ___   ___  _ __ _   _                 |
#   |              | |\/| |/ _ \ '_ ` _ \ / _ \| '__| | | |                |
#   |              | |  | |  __/ | | | | | (_) | |  | |_| |                |
#   |              |_|  |_|\___|_| |_| |_|\___/|_|   \__, |                |
#   |                                                |___/                 |
#   '----------------------------------------------------------------------'

def check_jolokia_metrics_mem(item, params, info):
    parsed = jolokia_metrics_parse(info)

    if parsed.get(item, "") == None:
        raise MKCounterWrapped("No information from Jolokia agent")

    if item not in parsed:
        return

    # convert old parameter version ( warn, crit )
    # represented levels of total heap
    if type(params) == tuple:
        params = {"total": params}

    # rename totalheap to total
    # this block can be removed in the future (today 22.02.13)
    if "totalheap" in params:
        params = params.copy()
        params["total"] = params["totalheap"]
        del params["totalheap"]

    d = parsed[item]
    mb = 1024 * 1024.0

    if "HeapMemoryUsage" not in d or "NonHeapMemoryUsage" not in d\
        or "HeapMemoryMax" not in d or "NonHeapMemoryMax" not in d:
            return 3, "data in agent output incomplete"

    heap = saveint(d["HeapMemoryUsage"]) / mb
    heapmax = saveint(d.get("HeapMemoryMax",-1)) / mb
    nonheap = saveint(d["NonHeapMemoryUsage"]) / mb
    nonheapmax = saveint(d.get("NonHeapMemoryMax",-1)) / mb
    total = heap + nonheap
    if heapmax > 0 and nonheapmax > 0:
        totalmax = heapmax + nonheapmax
    else:
        totalmax    = 0
        heapmax     = max(0, heapmax)
        nonheapmax  = max(0, nonheapmax)

    worst_state = 0
    perfdata    = []
    info_list   = []

    for what, value, value_max in [
            ("heap",     heap,    heapmax),
            ("nonheap",  nonheap, nonheapmax),
            ("total",    total,   totalmax),
        ]:
        param_state = 0
        level_info  = ""
        used_info   = ""
        if params.get(what):
            warn_level = 0
            crit_level = 0
            if type(params[what][0]) == int:
                if value_max:
                    warn_level = value_max - params[what][0]
                    crit_level = value_max - params[what][1]

                if what != "total":
                    perfdata.append((what, value, warn_level, crit_level, "", value_max))

                if not value_max:
                    param_state = 0
                elif value >= crit_level:
                    param_state = 2
                    level_info = "%s(crit at %sMB free)" % (state_markers[2], params[what][1])
                elif value >= warn_level:
                    param_state = 1
                    level_info = "%s(warn at %sMB free)" % (state_markers[1], params[what][0])
            else:
                if value_max:
                    warn_level = value_max * params[what][0] / 100.0
                    crit_level = value_max * params[what][1] / 100.0

                if what != "total":
                    perfdata.append((what, value, warn_level, crit_level, "", value_max))

                if not value_max:
                    param_state = 0
                elif value >= crit_level:
                    param_state = 2
                    level_info = "%s(crit at %s%%)" % (state_markers[2], params[what][1])
                elif value >= warn_level:
                    param_state = 1
                    level_info = "%s(warn at %s%%)" % (state_markers[1], params[what][0])
        else:
            if what != "total":
                perfdata.append((what, value, "", "", "", value_max))

        if value_max:
            used_info = "/%.1f%% used" % (value / value_max * 100)
        info_list.append("%s: %0.fMB%s%s" % (what.title(), value, used_info, level_info))
        worst_state = max(param_state, worst_state)

    return (worst_state, ', '.join(info_list), perfdata)


check_info["jolokia_metrics.mem"] = {
    "service_description"     : "JVM %s Memory",
    "check_function"          : check_jolokia_metrics_mem,
    "inventory_function"      : lambda i: inventory_jolokia_metrics(i, "mem"),
    "has_perfdata"            : True,
    "group"                   : "jvm_memory",
    "default_levels_variable" : "jolokia_metrics_mem_default_levels"
}

#.
#   .--Threads-------------------------------------------------------------.
#   |                _____ _                        _                      |
#   |               |_   _| |__  _ __ ___  __ _  __| |___                  |
#   |                 | | | '_ \| '__/ _ \/ _` |/ _` / __|                 |
#   |                 | | | | | | | |  __/ (_| | (_| \__ \                 |
#   |                 |_| |_| |_|_|  \___|\__,_|\__,_|___/                 |
#   |                                                                      |
#   '----------------------------------------------------------------------'

def check_jolokia_metrics_threads(item, params, info):
    warn, crit = params
    parsed = jolokia_metrics_parse(info)
    if parsed.get(item, "") == None:
        raise MKCounterWrapped("No information from Jolokia agent")
    if item not in parsed:
        return (3, "data not found in agent output")
    d = parsed[item]

    this_time = time.time()
    perfdata = []
    output   = []
    status   = 0
    for key in [ 'ThreadCount', 'DeamonThreadCount', 'PeakThreadCount', 'TotalStartedThreadCount' ]:
        if key not in d:
            continue # The keys might be optional (saw jboss only sending ThreadCount)

        val = int(d[key])
        status_info = ""
        if key == 'ThreadCount':
            # Thread count might lead to a warn/crit state
            if val >= crit:
                status = 2
                status_info = "(!!) (Levels at %d/%d)" % (warn, crit)
            elif val >= warn:
                status = 1
                status_info = "(!) (Levels at %d/%d)" % (warn, crit)

            # Calculate the thread increase rate
            rate = get_rate("jolokia_metrics.threads.%s" % item, this_time, val, allow_negative=True)
            output.append('ThreadRate: %0.2f' % rate)
            perfdata.append(('ThreadRate', rate))

        perfdata.append((key, val))
        output.append('%s: %d%s' % (key, val, status_info))
    return (status, ', '.join(output), perfdata)


check_info["jolokia_metrics.threads"] = {
    "service_description" : "JVM %s Threads",
    "check_function"      : check_jolokia_metrics_threads,
    "inventory_function"  : lambda i: inventory_jolokia_metrics(i, "threads"),
    "group"               : "jvm_threads",
    "has_perfdata"        : True,
}

#.
#   .--Uptime--------------------------------------------------------------.
#   |                  _   _       _   _                                   |
#   |                 | | | |_ __ | |_(_)_ __ ___   ___                    |
#   |                 | | | | '_ \| __| | '_ ` _ \ / _ \                   |
#   |                 | |_| | |_) | |_| | | | | | |  __/                   |
#   |                  \___/| .__/ \__|_|_| |_| |_|\___|                   |
#   |                       |_|                                            |
#   '----------------------------------------------------------------------'

def check_jolokia_metrics_uptime(item, params, info):
    parsed = jolokia_metrics_parse(info)
    if parsed.get(item, "") == None:
        raise MKCounterWrapped("No information from Jolokia agent")
    if item in parsed:
        if "Uptime" in parsed[item]:
            uptime = int(parsed[item]['Uptime']) / 1000
            return check_uptime_seconds(params, uptime)


check_info["jolokia_metrics.uptime"] = {
    "service_description" : "JVM %s Uptime",
    "check_function"      : check_jolokia_metrics_uptime,
    "inventory_function"  : lambda i: inventory_jolokia_metrics(i, "uptime"),
    "group"               : "jvm_uptime",
    'includes'            : [ 'uptime.include' ],
    "has_perfdata"        : True,
}

#.
#   .--App state-----------------------------------------------------------.
#   |                _                      _        _                     |
#   |               / \   _ __  _ __    ___| |_ __ _| |_ ___               |
#   |              / _ \ | '_ \| '_ \  / __| __/ _` | __/ _ \              |
#   |             / ___ \| |_) | |_) | \__ \ || (_| | ||  __/              |
#   |            /_/   \_\ .__/| .__/  |___/\__\__,_|\__\___|              |
#   |                    |_|   |_|                                         |
#   '----------------------------------------------------------------------'

def check_jolokia_metrics_app_state(item, _unused, info):
    app_state=3
    app = jolokia_metrics_app(info, item.split())

    # FIXME: this could be nicer.
    if   app and "Running" in app:
        if app['Running']   == '1':
            app_state = 0
        else:
            app_state = 2
    # wenn in app statename steht
    elif app and "stateName" in app:
        if app['stateName'] == 'STARTED':
            app_state = 0
        else:
            app_state = 2
    if   app_state == 3:
        return (3, "data not found in agent output")
    elif app_state == 0:
        return (0, 'application is running')
    elif app_state == 2:
        return (2, 'application is not running (Running: %s)')


    return (3, 'error in agent output')


check_info["jolokia_metrics.app_state"] = {
    "service_description" : "JVM %s State",
    "check_function"      : check_jolokia_metrics_app_state,
    "inventory_function"  : lambda i: inventory_jolokia_metrics_apps(i, "app_state"),
    "has_perfdata"        : False,
}


#.
#   .--Unsorted------------------------------------------------------------.
#   |              _   _                      _           _                |
#   |             | | | |_ __  ___  ___  _ __| |_ ___  __| |               |
#   |             | | | | '_ \/ __|/ _ \| '__| __/ _ \/ _` |               |
#   |             | |_| | | | \__ \ (_) | |  | ||  __/ (_| |               |
#   |              \___/|_| |_|___/\___/|_|   \__\___|\__,_|               |
#   |                                                                      |
#   '----------------------------------------------------------------------'

def check_jolokia_metrics_app_sess(item, params, info):
    lo_crit, lo_warn, hi_warn, hi_crit = params
    if   len(item.split()) == 3:
        app = jolokia_metrics_serv(info, item.split())
    elif len (item.split()) == 2:
        app = jolokia_metrics_app(info, item.split())
    if not app:
        return (3, "application not found")
    sessions = app.get('Sessions', app.get('activeSessions', app.get('OpenSessionsCurrentCount')))
    if sessions == None:
        return (3, "data not found in agent output")
    sess = saveint(sessions)
    maxActive = saveint(app.get('Sessions', app.get('maxActiveSessions', app.get('OpenSessionsCurrentCount'))))

    status = 0
    status_txt = ''
    if lo_crit is not None and sess <= lo_crit:
        status = 2
        status_txt = ' (Below or equal %d)' % lo_crit
    elif lo_warn is not None and sess <= lo_warn:
        status = 1
        status_txt = ' (Below or equal %d)' % lo_warn
    elif hi_crit is not None and sess >= hi_crit:
        status = 2
        status_txt = ' (Above or equal %d)' % hi_crit
    elif hi_warn is not None and sess >= hi_warn:
        status = 1
        status_txt = ' (Above or equal %d)' % hi_warn

    if maxActive and maxActive > 0:
        status_txt += " (max active sessions: %d)" % maxActive

    return (status, '%d Sessions%s' % (sess, status_txt),
            [('sessions', sess, hi_warn, hi_crit)])


def check_jolokia_metrics_bea_queue(item, params, info):
    app = jolokia_metrics_app(info, item.split())
    if not app:
        return (3, "application not found")
    if "QueueLength" not in app:
        return (3, "data not found in agent output")
    queuelength = int(app['QueueLength'])

    status = 0
    warn, crit = params
    if queuelength >= crit:
        status = 2
    elif queuelength >= warn:
        status = 1
    return (status, 'queue length is %d' % queuelength,
             [("length", queuelength, warn, crit)])


# FIXME: This check could work with any JVM
# It has no levels
# A candidate for 1.2.1 overhaul
def check_jolokia_metrics_bea_requests(item, _no_params, info):
    app = jolokia_metrics_app(info, item.split())
    if not app:
        return (3, "application not found")

    for nk in [ "CompletedRequestCount", "requestCount" ]:
       if nk in app:
           requests = int(app[nk])
           rate = get_rate("j4p.bea.requests.%s" % item, time.time(), requests)
           return (0, "%.2f requests/sec" % rate, [("rate", rate)])

    return (3, "data not found in agent output")


def check_jolokia_metrics_bea_threads(item, _no_params, info):
    app = jolokia_metrics_app(info, item.split())
    if not app:
        return (3, "data not found in agent output")

    perfdata = []
    infos = []
    for varname, title in [
        ( "ExecuteThreadTotalCount", "total" ),
        ( "ExecuteThreadIdleCount", "idle" ),
        ( "StandbyThreadCount", "standby" ),
        ( "HoggingThreadCount", "hogging" ) ]:
        value = int(app[varname])
        perfdata.append((varname, value))
        infos.append("%s: %d" % (title, value))

    return (0, ", ".join(infos), perfdata)


check_info["jolokia_metrics.app_sess"] = {
    "service_description" : "JVM %s Sessions",
    "check_function"      : check_jolokia_metrics_app_sess,
    "inventory_function"  : lambda i: inventory_jolokia_metrics_apps(i, "app_sess"),
    "group"               : "jvm_sessions",
    "has_perfdata"        : True,
}

check_info["jolokia_metrics.requests"] = {
    "service_description" : "JVM %s Requests",
    "check_function"      : check_jolokia_metrics_bea_requests,
    "inventory_function"  : lambda i: inventory_jolokia_metrics_apps(i, "requests"),
    "group"               : "jvm_requests",
    "has_perfdata"        : True,
}

# Stuff found on BEA Weblogic
check_info["jolokia_metrics.bea_queue"] = {
    "service_description" : "JVM %s Queue",
    "check_function"      : check_jolokia_metrics_bea_queue,
    "inventory_function"  : lambda i: inventory_jolokia_metrics_apps(i, "queue"),
    "group"               : "jvm_queue",
    "has_perfdata"        : True,
}

check_info["jolokia_metrics.bea_requests"] = {
    "service_description" : "JVM %s Requests",
    "check_function"      : check_jolokia_metrics_bea_requests,
    "inventory_function"  : lambda i: inventory_jolokia_metrics_apps(i, "bea_requests"),
    "group"               : "jvm_requests",
    "has_perfdata"        : True,
}

check_info["jolokia_metrics.bea_threads"] = {
    "service_description" : "JVM %s Threads",
    "check_function"      : check_jolokia_metrics_bea_threads,
    "inventory_function"  : lambda i: inventory_jolokia_metrics_apps(i, "threads"),
    "group"               : "jvm_threads",
    "has_perfdata"        : True,
}

check_info["jolokia_metrics.bea_sess"] = {
    "service_description" : "JVM %s Sessions",
    "check_function"      : check_jolokia_metrics_app_sess,
    "inventory_function"  : lambda i: inventory_jolokia_metrics_apps(i, "bea_app_sess"),
    "group"               : "jvm_sessions",
    "has_perfdata"        : True,
}

jolokia_metrics_perm_gen_default_levels = { "perm" : ( 80.0, 100.0 ) }

def inventory_jolokia_metrics_perm_gen(info):
    parsed = jolokia_metrics_parse(info)
    for instance, values in parsed.items():
        if values and "PermGenUsage" in values:
            yield instance, "jolokia_metrics_perm_gen_default_levels"

def check_jolokia_metrics_perm_gen(item, params, info):
    parsed = jolokia_metrics_parse(info)
    if parsed.get(item, "") == None:
        raise MKCounterWrapped("No information from Jolokia agent")
    values = parsed.get(item)
    if values:
        warn, crit = params['perm']
        usage_bytes = int(values['PermGenUsage'])
        size_bytes = int(values['PermGenMax'])
        usage_perc = 100.0 / size_bytes * usage_bytes
        warn_bytes = size_bytes / 100.0 * warn
        crit_bytes = size_bytes / 100.0 * crit
        perfdata = [("mem_perm_used", usage_bytes, warn_bytes, crit_bytes, 0, size_bytes)]
        state = 0
        if usage_perc >= crit:
            state = 2
        elif usage_perc >= warn:
            state = 1
        if state != 0:
            levels = "(warn/crit at %2.f/%.2f %%)" % ( warn, crit )
        else:
            levels = ""
        return state, "Usage: %.2f %% (%s of %s used) %s" % \
          ( usage_perc, get_bytes_human_readable(usage_bytes), get_bytes_human_readable(size_bytes), levels ), \
          perfdata

check_info["jolokia_metrics.perm_gen"] = {
    "service_description" : "JVM %s PermGen Usage",
    "check_function"      : check_jolokia_metrics_perm_gen,
    "inventory_function"  : inventory_jolokia_metrics_perm_gen,
    "group"               : "jvm_memory",
    "has_perfdata"        : True,
}


jolokia_metrics_cache_hits_default_levels = {}

def inventory_jolokia_metrics_cache(key, metrics, info):
    parsed = jolokia_metrics_parse(info)
    metrics_set = set(metrics)
    for inst, vals in [ x for x in parsed.iteritems() if x[1] != None ]:
        for cache, cache_vars in vals.get("CacheStatistics", {}).iteritems():
            if metrics_set.intersection(cache_vars) == metrics_set:
                if key is None:
                    yield "%s %s" % (inst, cache), None
                else:
                    yield "%s %s" % (inst, cache), "jolokia_metrics_%s_default_levels" % key


def check_jolokia_metrics_cache(metrics, totals, item, params, info):
    type_map = {
        "CacheHitPercentage"    : (float, 100.0, "%.1f%%"),
        "InMemoryHitPercentage" : (float, 100.0, "%.1f%%"),
        "OnDiskHitPercentage"   : (float, 100.0, "%.1f%%"),
        "OffHeapHitPercentage"  : (float, 100.0, "%.1f%%"),
    }

    parsed = jolokia_metrics_parse(info)
    try:
        inst, cache = item.split(" ")

        # we display the "metrics" first, totals after, but to "fix" metrics based on zero-totals
        # we need to go over the totals once
        totals_zero = len(totals) > 0
        for total in totals:
            val = int(parsed[inst]["CacheStatistics"][cache][total])
            if val != 0:
                totals_zero = False
                break

        for metric in metrics:
            type_, scale, format = type_map.get(metric, (int, 1,"%d"))

            val = type_(parsed[inst]["CacheStatistics"][cache][metric]) * scale
            if isinstance(val, float) and val == 0.0:
                # what a hack! we assume the float is based on the totals (all of them) and if they
                # were all 0, so this float is 0/0, we want to display it as 1 as to not cause
                # an alert
                val = 1.0 * scale
            yield 0, ("%s: " + format) % (metric, val), [(metric, val)]

        for total in totals:
            type_, scale, format = type_map.get(total, (int, 1,"%d"))
            val = type_(parsed[inst]["CacheStatistics"][cache][total]) * scale
            yield 0, ("%s: " + format) % (total, val), []
    except KeyError, e:
        # some element of the item was missing
        pass


check_info["jolokia_metrics.cache_hits"] = {
    "service_description" : "JVM %s Cache Usage",
    "check_function"      : lambda item, params, parsed: check_jolokia_metrics_cache(["CacheHitPercentage", "ObjectCount"], ["CacheHits", "CacheMisses"], item, params, parsed),
    "inventory_function"  : lambda info: inventory_jolokia_metrics_cache("cache_hits", ["CacheHitPercentage", "ObjectCount", "CacheHits", "CacheMisses"], info),
    "has_perfdata"        : True,
}

check_info["jolokia_metrics.in_memory"] = {
    "service_description" : "JVM %s In Memory",
    "check_function"      : lambda item, params, parsed: check_jolokia_metrics_cache(["InMemoryHitPercentage", "MemoryStoreObjectCount"], ["InMemoryHits", "InMemoryMisses"], item, params, parsed),
    "inventory_function"  : lambda info: inventory_jolokia_metrics_cache("cache_hits", ["InMemoryHitPercentage", "MemoryStoreObjectCount", "InMemoryHits", "InMemoryMisses"], info),
    "has_perfdata"        : True,
}

check_info["jolokia_metrics.on_disk"] = {
    "service_description" : "JVM %s On Disk",
    "check_function"      : lambda item, params, parsed: check_jolokia_metrics_cache(["OnDiskHitPercentage", "DiskStoreObjectCount"], ["OnDiskHits", "OnDiskMisses"], item, params, parsed),
    "inventory_function"  : lambda info: inventory_jolokia_metrics_cache("cache_hits", ["OnDiskHitPercentage", "DiskStoreObjectCount", "OnDiskHits", "OnDiskMisses"], info),
    "has_perfdata"        : True,
}

check_info["jolokia_metrics.off_heap"] = {
    "service_description" : "JVM %s Off Heap",
    "check_function"      : lambda item, params, parsed: check_jolokia_metrics_cache(["OffHeapHitPercentage", "OffHeapStoreObjectCount"], ["OffHeapHits", "OffHeapMisses"], item, params, parsed),
    "inventory_function"  : lambda info: inventory_jolokia_metrics_cache("cache_hits", ["OffHeapHitPercentage", "OffHeapStoreObjectCount", "OffHeapHits", "OffHeapMisses"], info),
    "has_perfdata"        : True,
}

check_info["jolokia_metrics.writer"] = {
    "service_description" : "JVM %s Cache Writer",
    "check_function"      : lambda item, params, parsed: check_jolokia_metrics_cache(["WriterQueueLength", "WriterMaxQueueSize"], [], item, params, parsed),
    "inventory_function"  : lambda info: inventory_jolokia_metrics_cache("cache_hits", ["WriterQueueLength", "WriterMaxQueueSize"], info),
    "has_perfdata"        : True,
}

