#!/usr/bin/env python
# -*- encoding: utf-8; py-indent-offset: 4 -*-
# +------------------------------------------------------------------+
# |             ____ _               _        __  __ _  __           |
# |            / ___| |__   ___  ___| | __   |  \/  | |/ /           |
# |           | |   | '_ \ / _ \/ __| |/ /   | |\/| | ' /            |
# |           | |___| | | |  __/ (__|   <    | |  | | . \            |
# |            \____|_| |_|\___|\___|_|\_\___|_|  |_|_|\_\           |
# |                                                                  |
# | Copyright Mathias Kettner 2014             mk@mathias-kettner.de |
# +------------------------------------------------------------------+
#
# This file is part of Check_MK.
# The official homepage is at http://mathias-kettner.de/check_mk.
#
# check_mk is free software;  you can redistribute it and/or modify it
# under the  terms of the  GNU General Public License  as published by
# the Free Software Foundation in version 2.  check_mk is  distributed
# in the hope that it will be useful, but WITHOUT ANY WARRANTY;  with-
# out even the implied warranty of  MERCHANTABILITY  or  FITNESS FOR A
# PARTICULAR PURPOSE. See the  GNU General Public License for more de-
# tails. You should have  received  a copy of the  GNU  General Public
# License along with GNU Make; see the file  COPYING.  If  not,  write
# to the Free Software Foundation, Inc., 51 Franklin St,  Fifth Floor,
# Boston, MA 02110-1301 USA.

__version__ = "1.4.0p9"

# TODO:
# - Create a more detailed report out of the test results. For example
#   graph the different values together with the timeline of the different
#   test cases.

# Problems with large installations:
# - Helper startup phases
# - Memory usage of helpers
# - Huge history files during restarts / rotations
# - Config compilation is serialized
# - Service check spread with many new services
# Config tweaks needed:
# - Number of helpers
# - Immediate history log file rotation (too large files) -> cmc_log_limit

#.
#   .--Imports-------------------------------------------------------------.
#   |               ___                            _                       |
#   |              |_ _|_ __ ___  _ __   ___  _ __| |_ ___                 |
#   |               | || '_ ` _ \| '_ \ / _ \| '__| __/ __|                |
#   |               | || | | | | | |_) | (_) | |  | |_\__ \                |
#   |              |___|_| |_| |_| .__/ \___/|_|   \__|___/                |
#   |                            |_|                                       |
#   '----------------------------------------------------------------------'

import ast
import collections
import getopt
import glob
import os
import pprint
import re
import select
import shutil
import socket
import sys
import signal
import subprocess
import tempfile
import time
import threading
import traceback
import logging
import multiprocessing

import psutil
import npyscreen

from collections import namedtuple

import livestatus


#.
#   .--Helpers-------------------------------------------------------------.
#   |                  _   _      _                                        |
#   |                 | | | | ___| |_ __   ___ _ __ ___                    |
#   |                 | |_| |/ _ \ | '_ \ / _ \ '__/ __|                   |
#   |                 |  _  |  __/ | |_) |  __/ |  \__ \                   |
#   |                 |_| |_|\___|_| .__/ \___|_|  |___/                   |
#   |                              |_|                                     |
#   +----------------------------------------------------------------------+
#   | Different helper methods and classes                                 |
#   '----------------------------------------------------------------------'

# Changes the process name and process command line on of the running process
# This works at least with Python 2.x on Linux
def set_cmdline(cmdline):
    import ctypes, ctypes.util
    libc = ctypes.cdll.LoadLibrary(ctypes.util.find_library('c'))

    argv = ctypes.POINTER(ctypes.c_char_p)()
    argc = ctypes.c_int()
    ctypes.pythonapi.Py_GetArgcArgv(ctypes.byref(argc), ctypes.byref(argv))
    cmdlen = sum([len(argv[i]) for i in range(argc.value)]) + argc.value
    new_cmdline = ctypes.c_char_p(cmdline.ljust(cmdlen, '\0'))

    # replace the command line, which is available via /proc/<pid>/cmdline.
    # This is .e.g used by ps
    libc.memcpy(argv.contents, new_cmdline, cmdlen)

    # replace the prctl name, which is available via /proc/<pid>/status.
    # This is for example used by top and killall
    libc.prctl(15, new_cmdline, 0, 0, 0)


def omd_root():
    return os.environ["OMD_ROOT"]


def site_id():
    return os.environ.get("OMD_SITE")


def fmt_timespan(seconds):
    hours, secs = divmod(seconds, 3600)
    mins, secs = divmod(secs, 60)
    return "%02d:%02d:%02d" % (hours, mins, secs)


def fmt_datetime(seconds):
    return time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(seconds))


def fmt_bytes(b, base=1024.0, bytefrac=True, unit="B"):
    base = float(base)

    # Handle negative bytes correctly
    prefix = ''
    if b < 0:
        prefix = '-'
        b *= -1

    if b >= base * base * base * base:
        return '%s%.2f T%s' % (prefix, b / base / base / base / base, unit)
    elif b >= base * base * base:
        return '%s%.2f G%s' % (prefix, b / base / base / base, unit)
    elif b >= base * base:
        return '%s%.2f M%s' % (prefix, b / base / base, unit)
    elif b >= base:
        return '%s%.2f k%s' % (prefix, b / base, unit)
    elif bytefrac:
        return '%s%.2f %s' % (prefix, b, unit)
    else: # Omit byte fractions
        return '%s%.0f %s' % (prefix, b, unit)


class MKGeneralException(Exception):
    pass


class MKTerminate(MKGeneralException):
    pass


class MKSkipTestCase(MKGeneralException):
    pass


class MKFailTestCase(MKGeneralException):
    pass


class MKFailTestCaseHighLatency(MKFailTestCase):
    pass



class LocalConnection(livestatus.SingleSiteConnection):
    def __init__(self, *args, **kwargs):
        livestatus.SingleSiteConnection.__init__(self,
            "unix:%s/tmp/run/live" % omd_root(), *args, **kwargs)
        self.connect()



def interrupt_handler(signum, frame):
    raise MKTerminate("Caught signal: %d" % signum)


def register_signal_handlers():
    signal.signal(signal.SIGTERM, interrupt_handler)


def usage(error=None):
    if error:
        sys.stderr.write("ERROR: %s\n" % error)
    sys.stdout.write("Usage: mkbench [OPTIONS] [MODE] [MODE_OPTIONS...]\n")
    sys.stdout.write("\n")
    sys.stdout.write("The tool is meant to execute different tasks in a Check_MK site\n")
    sys.stdout.write("on a system and measure the performance of the task on that\n")
    sys.stdout.write("system. The test is saving different static information of\n")
    sys.stdout.write("the system (Installed CPUs, memory, disks, ...) and captures\n")
    sys.stdout.write("the most important peformance relevant values during testing\n")
    sys.stdout.write("for later analysis and comparision.\n")
    sys.stdout.write("\n")
    sys.stdout.write("The results of mkbench are stored in 'var/mkbench'.\n")
    sys.stdout.write("\n")
    sys.stdout.write("The tool also monitors the performance of different values of\n")
    sys.stdout.write("the system and aborts tests in case thresholds are reached.\n")
    sys.stdout.write("\n")
    sys.stdout.write("BE AWARE: mkbench modifies the configuration of your site and will\n")
    sys.stdout.write("          delete existing files. Only execute this in test sites.\n")
    sys.stdout.write("\n")
    sys.stdout.write("MODES:\n")
    sys.stdout.write("\n")
    sys.stdout.write("    NONE             Execute a test in the current site\n")
    sys.stdout.write("    --report         List test results available for reporting\n")
    sys.stdout.write("    --report ID1     Create reports out of previous results\n")
    sys.stdout.write("\n")
    sys.stdout.write("TEST OPTIONS:\n")
    sys.stdout.write("    --num-hosts H    Set the initial number of hosts (Defaults to 100)\n")
    sys.stdout.write("    --num-helpers N  Set the initial number of Check_MK helpers\n")
    sys.stdout.write("\n")
    sys.stdout.write("OPTIONS:\n")
    sys.stdout.write("\n")
    sys.stdout.write("    --debug          Let Python exceptions come through\n")
    sys.stdout.write("    --version        Print the version of the program\n")
    sys.stdout.write("\n")

    sys.exit(3)


def get_host_ip(host_id):
    start_ip_int = ip_int_from_string("127.0.0.1")
    return string_from_ip_int(start_ip_int + host_id)


def ip_int_from_string(ip_str):
    packed_ip = 0
    octets = ip_str.split('.')
    for oc in octets:
        packed_ip = (packed_ip << 8) | int(oc)
    return packed_ip


def string_from_ip_int(ip_int):
    octets = []
    for _ in range(4):
        octets.insert(0, str(ip_int & 0xFF))
        ip_int >>= 8
    return '.'.join(octets)


#.
#   .--Testing-------------------------------------------------------------.
#   |                   _____         _   _                                |
#   |                  |_   _|__  ___| |_(_)_ __   __ _                    |
#   |                    | |/ _ \/ __| __| | '_ \ / _` |                   |
#   |                    | |  __/\__ \ |_| | | | | (_| |                   |
#   |                    |_|\___||___/\__|_|_| |_|\__, |                   |
#   |                                             |___/                    |
#   +----------------------------------------------------------------------+
#   | The major part of this tool. Execution of tests and measurement      |
#   '----------------------------------------------------------------------'

def run_performance_test(num_hosts, num_helpers):
    verify_site_is_running()
    verify_site_is_empty()

    # Create a new test, running in a dedicated thread
    runner = TestRunner(num_hosts, num_helpers)

    # Start measuring in a dedicated thread
    analyzer = Analyzer(runner)

    # Show state on console
    renderer = ConsoleRenderer(runner)

    try:
        # Save current state before starting
        runner.save()

        # And now start the work
        analyzer.start()
        runner.start()

        try:
            renderer.run(fork=False)
        except npyscreen.wgwidget.NotEnoughSpaceForWidget, e:
            raise MKGeneralException("Too small terminal (%s)" % e)

        finally:
            sys.stdout.write("Terminating...\n")
            runner.test.log_to_stdout()
            runner.stop()
            runner.join()
    finally:
        analyzer.stop()


class Test(object):
    STATE_INITIALIZING = "initializing"
    STATE_RUNNING      = "running"
    STATE_STOPPED      = "stopped"

    RESULT_COMPLETED   = "completed"
    RESULT_ABORTED     = "aborted" # TODO: Use this when aborted
    RESULT_FAILED      = "failed"

    @classmethod
    def data_dir(cls):
        return os.path.join(omd_root(), "var/mkbench")


    @classmethod
    def test_dir(cls, id):
        return os.path.join(cls.data_dir(), id)


    @classmethod
    def exists(cls, id):
        return os.path.exists(cls.test_dir(id))


    def __init__(self, id=None):
        self._log           = []
        self._log_to_stdout = False
        self.version        = __version__
        self.state          = Test.STATE_INITIALIZING
        self._measurements  = []
        self.result         = None
        self.result_msg     = None
        self.system_info    = SystemInfo()

        if id is None:
            self.system_info.collect()
            self.id         = self._get_test_id()
            self.test_cases = []
        else:
            self.id = id
            self.load()


    def load(self):
        raw_state = ast.literal_eval(open(os.path.join(self.dir(), "state")).read())
        self.deserialize(raw_state)
        self.load_measurements()


    # Persists the current state of the test. The measurements are persisting
    # their results on their own.
    def save(self):
        test_dir = self.dir()
        try:
            os.makedirs(test_dir)
        except OSError, e:
            if e.errno == 17: # file exists
                pass
            else:
                raise

        path = os.path.join(test_dir, "state")
        with tempfile.NamedTemporaryFile("w", dir=os.path.dirname(path),
            prefix=".%s.new" % os.path.basename(path), delete=False) as tmp:
            tmp_path = tmp.name
            tmp.write(pprint.pformat(self.serialize()) + "\n")
        os.rename(tmp_path, path)


    def deserialize(self, raw_state):
        self._log       = raw_state["log"]
        self.version    = raw_state["version"]
        self.state      = raw_state["state"]
        self.result     = raw_state["result"]
        self.result_msg = raw_state["result_msg"]
        self.system_info.deserialize(raw_state["system_info"])
        self.deserialize_test_cases(raw_state["test_cases"])


    def serialize(self):
        return {
            "log"         : self._log,
            "version"     : self.version,
            "state"       : self.state,
            "result"      : self.result,
            "result_msg"  : self.result_msg,
            "system_info" : self.system_info.serialize(),
            "test_cases"  : self.serialize_test_cases(),
        }


    def serialize_test_cases(self):
        return [ test_case.serialize() for test_case in self.test_cases ]


    def deserialize_test_cases(self, raw_cases):
        self.test_cases = []
        for raw_case in raw_cases:
            cls = globals()[raw_case.pop("__class_name__")]
            test_case = cls()
            test_case.deserialize(raw_case)
            self.test_cases.append(test_case)


    def load_measurements(self):
        self._measurements = []
        for entry in sorted(os.listdir(self.dir())):
            if entry[0] == "." or not entry.endswith(".data"):
                continue

            m = Measurement()
            m.load(os.path.join(self.dir(), entry))
            self.add_measurement(m)


    def _get_test_id(self):
        parts = [
            self.system_info.omd_version,
            self.system_info.core,
            "%d" % time.time(),
        ]
        return "_".join(parts)


    def set_state(self, state, result=None, result_msg=None):
        self.state      = state
        self.result     = result
        self.result_msg = result_msg


    def state_text(self):
        return self.state


    def first_measurement(self):
        return self._measurements[0]


    def current_measurement(self):
        return self._measurements[-1]


    def add_measurement(self, m):
        self._measurements.append(m)


    def has_measurements(self):
        return bool(self._measurements)


    def get_measurements_by_test_case(self):
        by_test_case = collections.OrderedDict()
        for m in self._measurements:
            if m.test_case:
                l = by_test_case.setdefault(m.test_case, [])
                l.append(m)
        return by_test_case


    def is_stopped(self):
        return self.state == Test.STATE_STOPPED


    def get_log(self):
        return self._log


    def log(self, level, msg):
        if self._log_to_stdout:
            sys.stdout.write("%d %s\n" % (time.time(), msg))
        self._log.append((time.time(), level, msg))


    def info(self, msg):
        self.log(logging.INFO, msg)


    def error(self, msg):
        self.log(logging.ERROR, msg)


    def log_to_stdout(self):
        self._log_to_stdout = True


    def dir(self):
        return Test.test_dir(self.id)


    def duration(self):
        return self.current_measurement().time - self.first_measurement().time




class TestRunner(threading.Thread):
    host_steps = [
        100,
        200,
        500,
        1000,
        2000,
        5000,
        8000,
        10000,
        20000,
    ]

    def __init__(self, num_hosts=100, num_helpers=20):
        super(TestRunner, self).__init__()
        self.name      = "test-runner"
        self._stop     = threading.Event()
        self.test      = Test()
        self.test_case = None

        # Helper vars needed for test case generator
        self._num_hosts   = num_hosts
        self._num_helpers = num_helpers


    def run(self):
        try:
            result, result_msg = None, None

            self.wait_for_load_cooldown(max_wait=900, expected_load=5)
            self._prepare_site()
            self._set_test_state(Test.STATE_RUNNING)
            self._run_test()

            result = Test.RESULT_COMPLETED

        except Exception, e:
            if opt_debug:
                raise
            self.test.error("Test failed: %s" % traceback.format_exc())
            result = Test.RESULT_FAILED
            result_msg = "Test failed: %s" % e

        finally:
            self._set_test_state(Test.STATE_STOPPED, result, result_msg)
            self._cleanup_host_files()


    def wait_for_load_cooldown(self, max_wait, expected_load):
        # After all helpers are ready, check for reaching previous load level
        reached = False
        while max_wait > 0:
            if self.shall_stop():
                break

            loadavg_5 = os.getloadavg()[1]
            if loadavg_5 > expected_load:
                self.test.info("Waiting %d sec for cooldown (5min load: %0.2f > %0.2f)" %
                                                            (max_wait, loadavg_5, expected_load))
            else:
                reached = True
                break
            time.sleep(1)
            max_wait -= 1

        if not reached:
            raise MKGeneralException("System has too high load")


    def _set_test_state(self, state, result=None, result_msg=None):
        self.test.set_state(state, result, result_msg)
        self.save()


    def _set_test_case_state(self, state, result=None, result_msg=None):
        self.test_case.set_state(state, result, result_msg)
        self.save()


    def _prepare_site(self):
        self.test.info("Preparing site for testing")
        self._prepare_rrdcached()


    def _prepare_rrdcached(self):
        with open("%s/etc/rrdcached.conf" % omd_root(), "w") as f:
            f.write(
                "TIMEOUT=300\n"
                "RANDOM_DELAY=10\n"
                "FLUSH_TIMEOUT=7200\n"
            )

        p = subprocess.Popen(["omd", "restart", "rrdcached"],
                close_fds=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
        stdout = p.communicate()[0]
        if p.returncode != 0:
            raise MKGeneralException("Failed to restart rrdcached: %s" % stdout)


    def _run_test(self):
        for test_case in self._generate_test_cases():
            if self._stop.isSet():
                break

            self.test.test_cases.append(test_case)

            result, result_msg = None, None
            try:
                self.test.info("Initializing %s" % test_case.name())

                for step in test_case.run():
                    if isinstance(step, Start):
                        # The measurements are only assigned to the test case when
                        # it tells us to do so by providing a Start() object
                        self.test.info("Start %s" % test_case.name())
                        self.test_case = test_case
                        self._set_test_case_state(TestCase.STATE_RUNNING)

                    elif isinstance(step, Wait):
                        # Do many small sleeps while the test is running so that
                        # the test thread can persist the current state of the test
                        result = TestCase.RESULT_COMPLETED
                        while time.time() <= step.until:
                            try:
                                self._while_waiting()
                            except MKSkipTestCase, e:
                                self.test.info("Skipping %s (%s): %s" % (test_case.name(),
                                          fmt_timespan(test_case.duration()), e))
                                result = TestCase.RESULT_SKIPPED
                                result_msg = "Skipped: %s" % e
                                break # Will iterate to next step (till test case end)

                            if self._stop.isSet():
                                result = TestCase.RESULT_ABORTED
                                result_msg = "Test runner was stopped"
                                return

                            time.sleep(1)

                self.test.info("Finished %s (%s)" % (test_case.name(),
                                        fmt_timespan(test_case.duration())))
                self._increase_hosts()

            except MKFailTestCase, e:
                result = TestCase.RESULT_FAILED
                result_msg = "Test case aborted: %s" % e
                self.test.error(result_msg)

                if type(e) == MKFailTestCaseHighLatency:
                    self._increase_helpers()
                else:
                    break

            except Exception, e:
                result = TestCase.RESULT_FAILED
                result_msg = "Test case failed: %s" % e
                self.test.error(result_msg)
                raise

            finally:
                if self.test_case:
                    self._set_test_case_state(TestCase.STATE_STOPPED, result, result_msg)
                    self.test_case = None

        self.test.info("Finished testing")


    def _increase_hosts(self):
        try:
            self._num_hosts = TestRunner.host_steps[TestRunner.host_steps.index(self._num_hosts)+1]
        except (ValueError, IndexError), e:
            # Reached end!
            self.stop()


    def _increase_helpers(self):
        self._num_helpers += 40


    def _generate_test_cases(self):
        while not self._stop.isSet():
            yield TestAgentBasedStandardHosts(self,
                num_hosts=self._num_hosts,
                planned_duration=15*60,
                num_cmk_helpers=self._num_helpers
            )
        #return [
        #    TestAgentBasedStandardHosts(self, num_hosts=100,   planned_duration=15*60, num_cmk_helpers=20),
        #    TestAgentBasedStandardHosts(self, num_hosts=200,   planned_duration=15*60, num_cmk_helpers=20),
        #    TestAgentBasedStandardHosts(self, num_hosts=500,   planned_duration=15*60, num_cmk_helpers=20),
        #    TestAgentBasedStandardHosts(self, num_hosts=1000,  planned_duration=15*60, num_cmk_helpers=20),
        #    TestAgentBasedStandardHosts(self, num_hosts=2000,  planned_duration=15*60, num_cmk_helpers=20),
        #    TestAgentBasedStandardHosts(self, num_hosts=5000,  planned_duration=15*60, num_cmk_helpers=20),
        #    TestAgentBasedStandardHosts(self, num_hosts=8000,  planned_duration=15*60, num_cmk_helpers=20),
        #    TestAgentBasedStandardHosts(self, num_hosts=10000, planned_duration=15*60, num_cmk_helpers=20),

        #    TestAgentBasedStandardHosts(self, num_hosts=2000,  planned_duration=15*60, num_cmk_helpers=40),
        #    TestAgentBasedStandardHosts(self, num_hosts=5000,  planned_duration=15*60, num_cmk_helpers=40),
        #    TestAgentBasedStandardHosts(self, num_hosts=8000,  planned_duration=15*60, num_cmk_helpers=40),
        #    TestAgentBasedStandardHosts(self, num_hosts=10000, planned_duration=15*60, num_cmk_helpers=40),

        #    TestAgentBasedStandardHosts(self, num_hosts=5000,  planned_duration=15*60, num_cmk_helpers=60),
        #    TestAgentBasedStandardHosts(self, num_hosts=8000,  planned_duration=15*60, num_cmk_helpers=60),
        #    TestAgentBasedStandardHosts(self, num_hosts=10000, planned_duration=15*60, num_cmk_helpers=60),

        #    TestAgentBasedStandardHosts(self, num_hosts=8000,  planned_duration=15*60, num_cmk_helpers=100),
        #    TestAgentBasedStandardHosts(self, num_hosts=10000, planned_duration=15*60, num_cmk_helpers=100),
        #    TestAgentBasedStandardHosts(self, num_hosts=20000, planned_duration=15*60, num_cmk_helpers=100),
        #]


    def _while_waiting(self):
        self._check_thresholds()
        self.save()


    def _check_thresholds(self):
        try:
            measurement = self.test.current_measurement()
        except IndexError:
            return

        self._check_upper_thresholds(measurement)
        self._check_lower_thresholds(measurement)


    def _check_upper_thresholds(self, m):
        # Abort when less than 100 MB are free
        if m.disk_usage.free < 100*1024*1024:
            raise MKFailTestCase("The disk %s has only %s left." %
                (self.test.system_info.block_device, fmt_bytes(m.disk_usage.free)))

        if m.cpu_load[1] > self.test.system_info.cpu_count * 1.2:
            raise MKFailTestCase("The 5 min average CPU load %0.2f is higher than "
                "number of CPUs (%d)." % (m.cpu_load[1], self.test.system_info.cpu_count))

        # Check memory threshold. There are different values provided by psutil.
	# this value sounds good to me for testing.
	if m.memory.available < 100*1024*1024:
            raise MKFailTestCase("There is only %s memory available." %
                (fmt_bytes(m.memory.available)))


        # Initial scheduling of the CMC spreads the service checks for a maximum of
        # 5 * check_interval. So we should terminate the test only after some initialization
        # time.
        if self.test_case.duration() < 450:
            return

        if m.site_stats and m.site_stats["average_latency_cmk"] > 30:
            raise MKFailTestCaseHighLatency("The average check latency is too high (%0.2f sec)" %
                                                  m.site_stats["average_latency_cmk"])


    def _check_lower_thresholds(self, m):
        if self.test_case.duration() < 30:
            return # Check for lower thresholds after X seconds

        if m.disk_usage.percent > 70:
            return

        if m.cpu_load[0] > self.test.system_info.cpu_count * 0.2:
            return

	if m.memory.percent > 10:
            return

        if m.site_stats and m.site_stats["helper_usage_cmk"]*100 > 10:
            return

        if m.site_stats and m.site_stats["average_latency_cmk"] > 2:
            return

        raise MKSkipTestCase("System is bored.")


    def save(self):
        set_symlink = not os.path.exists(self.test.dir())

        self.test.save()

        if set_symlink:
            self._update_last_symlink()


    def _update_last_symlink(self):
        path = os.path.join(Test.data_dir(), "last")
        try:
            os.unlink(path)
        except OSError:
            pass
        os.symlink(self.test.id, path)


    def stop(self):
        self._stop.set()


    def shall_stop(self):
        return self._stop.isSet()


    def _cleanup_host_files(self, cleanup_rrds=False):
        for path in glob.glob("%s/var/check_mk/autochecks/*" % omd_root()):
            os.unlink(path)

        for path in glob.glob("%s/tmp/check_mk/counters/*" % omd_root()):
            os.unlink(path)

        if cleanup_rrds:
            for path in glob.glob("%s/var/check_mk/rrd/*" % omd_root()):
                shutil.rmtree(path)

            for path in glob.glob("%s/var/pnp4nagios/perfdata/*" % omd_root()):
                shutil.rmtree(path)

        for path in glob.glob("%s/var/check_mk/core/archive/history-*" % omd_root()):
            os.unlink(path)

        for path in glob.glob("%s/var/check_mk/core/state*" % omd_root()):
            os.unlink(path)



class Analyzer(threading.Thread):
    def __init__(self, runner):
        self._runner = runner
        super(Analyzer, self).__init__()

        self.name   = "analyzer"
        self.daemon = True
        self._stop = threading.Event()


    def run(self):
        #self._runner.test.info("Analyzer started")
        try:
            while not self._stop.isSet() and not self._runner.test.is_stopped():
                self._measure()
                time.sleep(1)
        except Exception, e:
            if opt_debug:
                raise
            self._runner.test.error("Analyzer failed: %s" % traceback.format_exc())
        self._runner.test.info("Analyzer stopped")


    def _measure(self):
        m = Measurement(self._runner)
        if self._runner.test.has_measurements():
            last = self._runner.test.current_measurement()
        else:
            last = None

        m.measure(last=last)
        self._runner.test.add_measurement(m)

        m.save()


    def is_stopped(self):
        return not self.is_alive()


    def stop(self):
        self._stop.set()



class Start(object):
    pass



class Wait(object):
    def __init__(self, duration):
        self.until = time.time() + duration




class SystemInfo(object):
    def __init__(self):
        super(SystemInfo, self).__init__()
        self.cpu_count    = None
        self.cpu_info     = None
        self.memory       = None
        self.block_device = None
        self.linux_distro = None
        self.omd_version  = None
        self.omd_site     = None
        self.core         = None


    def collect(self):
        # Number of logical CPUs
        self.cpu_count    = psutil.cpu_count()
        self.cpu_info     = self._get_cpu_info()
        self.memory       = psutil.virtual_memory().total
        self.block_device = self._find_block_device("/omd/sites")
        self.linux_distro = self._get_linux_distro()
        self.omd_version  = self._get_omd_version()
        self.omd_site     = site_id()
        self.core         = self._get_monitoring_core()


    def serialize(self):
        return dict([ (k, v) for k, v in self.__dict__.items() if k[0] != "_" ])


    def deserialize(self, raw):
        self.__dict__.update(raw)


    def _find_block_device(self, mount_path):
        mount_path = os.path.realpath(mount_path)
        while not os.path.ismount(mount_path):
            mount_path= os.path.dirname(mount_path)

        dev = os.stat(mount_path).st_dev
        major = os.major(dev)
        minor = os.minor(dev)

        return os.path.realpath("/sys/dev/block/%d:%d" % (major, minor))


    def _get_linux_distro(self):
        return open("%s/share/omd/distro.info" % omd_root()).readline().split("=", 1)[1].strip()


    def _get_omd_version(self):
        return os.path.basename(os.path.realpath("%s/version" % omd_root()))


    def _get_monitoring_core(self):
        for l in open("%s/etc/omd/site.conf" % omd_root()):
            if l.startswith("CONFIG_CORE="):
                return l.strip().split("=", 1)[1].strip("'")
        return None


    def _get_cpu_info(self):
        node = {}
        num_threads_total = 0
        sockets = set([])
        for l in open("/proc/cpuinfo"):
            parts = l.split(": ", 1)
            if len(parts) != 2:
                continue
            varname, value = parts[0].strip(), parts[1].strip()

            if varname == "cpu cores":
                node["cores_per_cpu"] = int(value)
            elif varname == "siblings":
                node["threads_per_cpu"] = int(value)
            elif varname == "vendor_id":
                node["vendor"] = {
                    "GenuineIntel" : "intel",
                    "AuthenticAMD" : "amd",
                }.get(value, value)
            elif varname == "cache size":
                node["cache_size"] = int(value.split()[0]) * 1024 # everything is normalized to bytes!
            elif varname == "model name":
                node["model"] = value
            # For the following two entries we assume that all
            # entries are numbered in increasing order in /proc/cpuinfo.
            elif varname == "processor":
                num_threads_total = int(value) + 1
            elif varname == "physical id":
                sockets.add(int(value))
            elif varname == "flags":
                if re.search(" lm ", value):
                    node["arch"] = "x86_64"
                else:
                    node["arch"] = "i386"

        num_sockets = len(sockets)

        if num_threads_total:
            node.setdefault("cores_per_cpu", 1)
            node.setdefault("threads_per_cpu", 1)
            node["cores"]   = num_sockets * node["cores_per_cpu"]
            node["threads"] = num_sockets * node["threads_per_cpu"]
            node["cpus"]    = num_sockets

        return node





DiskIO = namedtuple("DiskIO", ["read_bytes", "write_bytes"])



class Measurement(object):
    @classmethod
    def summary(self, measurements):
        average_keys = [
            "cpu_load",
            "disk_io",
            "disk_usage",
            "memory",
            "site_stats",
        ]

        average = Measurement()

        for k in average_keys:
            values = []
            for m in measurements:
                values.append(getattr(m, k))

            if values:
                if isinstance(values[0], tuple):
                    # averages of equal length tuples
                    averaged = tuple([ sum(y) / len(y) for y in zip(*values) ])
                else:
                    # averages of dict items
                    averaged = {}

                    real_values = [v for v in values if v is not None ]
                    num_values  = len(real_values)

                    if real_values:
                        for key, value in real_values[0].items():
                            if isinstance(value, unicode):
                                averaged[key] = value
                            else:
                                averaged[key] = float(sum([ v[key] for v in real_values ])) / num_values
            else:
                averaged = None

            average.set_value(k, averaged)

        return average


    def __init__(self, runner=None):
        super(Measurement, self).__init__()
        self._runner = runner

        self.test_case     = None
        self.disk_counters = None
        self.site_state    = None

        self.time          = None
        self.cpu_load      = None
        self.memory        = None
        self.disk_usage    = None
        self.disk_io       = None
        self.site_stats    = None


    def measure(self, last=None):
        self.test_case  = self._runner.test_case.name() \
                            if self._runner.test_case else None
        self.time       = time.time()
        self.cpu_load   = os.getloadavg()
        self.memory     = psutil.virtual_memory()
        self.disk_usage = psutil.disk_usage(omd_root())

        self._get_disk_io(last)
        self._get_site_state()
        self._get_site_stats()


    def load(self, path):
        try:
            raw_data = ast.literal_eval(open(path).read())
            self.deserialize(raw_data)
        except Exception, e:
            if opt_debug:
                raise
            raise MKGeneralException("Failed to load '%s': %s" % (path, e))


    def save(self):
        test_dir = self._runner.test.dir()

        path = os.path.join(test_dir, "%d.data" % self.time)
        with tempfile.NamedTemporaryFile("w", dir=os.path.dirname(path),
            prefix=".%s.new" % os.path.basename(path), delete=False) as tmp:
            tmp_path = tmp.name
            tmp.write(pprint.pformat(self.serialize()) + "\n")
        os.rename(tmp_path, path)


    def serialize(self):
        raw = {}

        for k, v in self.__dict__.items():
            if k[0] == "_":
                continue

            # change named tuples to real tuples (for ast.literal_eval)
            if isinstance(v, tuple) and hasattr(v, "_fields"):
                raw[k] = tuple(v)
            else:
                raw[k] = v

        return raw


    def deserialize(self, raw):
        for k in self.__dict__.keys():
            if k[0] != "_":
                self.set_value(k, raw[k])


    def set_value(self, k, val):
        """Set an attribute of the measurement. Cares about converting tuples
        to the correct namedtuples."""
        from psutil._common import sdiskusage
        from psutil._pslinux import svmem, sdiskio

        if k == "memory":
            val = svmem(*val)
        elif k == "disk_usage":
            val = sdiskusage(*val)
        elif k == "disk_counters":
            val = sdiskio(*val)
        elif k == "disk_io":
            val = DiskIO(*val)

        setattr(self, k, val)


    def _get_disk_io(self, last):
        self.disk_counters = self._get_disk_counters()

        if not last:
            self.disk_io = DiskIO(0.0, 0.0)
            return

        read_bytes_per_sec = (self.disk_counters.read_bytes - last.disk_counters.read_bytes) \
                                / (self.time - last.time)

        write_bytes_per_sec = (self.disk_counters.write_bytes - last.disk_counters.write_bytes) \
                                / (self.time - last.time)

        self.disk_io = DiskIO(read_bytes_per_sec, write_bytes_per_sec)


    def _get_disk_counters(self):
        # Use total disk IO of the system because e.g. software raid IO is not
        # accounted to the md0p2 partition in case of CMK is on that volume.
        #device_name = os.path.basename(self._runner.test.system_info.block_device)
        #disk_io     = psutil.disk_io_counters(perdisk=True)
        #return disk_io[device_name]

        #part_name   = os.path.basename(self._runner.test.system_info.block_device)
        #device_name = part_name.rsplit("p", 1)[0]
        #disk_io     = psutil.disk_io_counters(perdisk=True)
        #return disk_io[device_name]
        return psutil.disk_io_counters()


    def _get_site_state(self):
        self.site_state = subprocess.call(["omd", "status", "--bare"],
                            stdout=file(os.devnull, "w"), close_fds=True)


    def _get_site_stats(self):
        if self.site_state not in [ 0, 2 ] \
           or not os.path.exists("%s/tmp/run/live" % omd_root()):
            self.site_stats = None
            return

        try:
            live = LocalConnection()
            live.set_timeout(5)
            self.site_stats = live.query_row_assoc("GET status")
            live.disconnect()
        except livestatus.MKLivestatusException, e:
            self._runner.test.error("Livestatus: %s" % e)
            self.site_stats = None


#.
#   .--AgentListener-------------------------------------------------------.
#   |      _                    _   _     _     _                          |
#   |     / \   __ _  ___ _ __ | |_| |   (_)___| |_ ___ _ __   ___ _ __    |
#   |    / _ \ / _` |/ _ \ '_ \| __| |   | / __| __/ _ \ '_ \ / _ \ '__|   |
#   |   / ___ \ (_| |  __/ | | | |_| |___| \__ \ ||  __/ | | |  __/ |      |
#   |  /_/   \_\__, |\___|_| |_|\__|_____|_|___/\__\___|_| |_|\___|_|      |
#   |          |___/                                                       |
#   +----------------------------------------------------------------------+
#   | Is executed in a dedicated process and serves the dynamic agent data |
#   '----------------------------------------------------------------------'

class AgentListener(multiprocessing.Process):
    def __init__(self, host_ids, *args, **kwargs):
        self._hosts      = host_ids
        self._sockets    = []
        self._socket_map = {}
        super(AgentListener, self).__init__(*args, **kwargs)
        self.name        = "mkbench_agent-listener"

        # Holds the counters per host that are used to track the dynamic agent data
        self._counters   = {}


    def run(self):
        try:
            self._set_process_name()
            self._open_sockets()
            self._serve()
        except (KeyboardInterrupt, MKTerminate):
            pass
        finally:
            self._close_sockets()


    def _set_process_name(self):
        set_cmdline(self.name)


    def _open_sockets(self):
        for host_id in self._hosts:
            address = (get_host_ip(host_id), 6559)
            try:
                s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
                s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
                s.bind(address)
                s.listen(1)
                self._sockets.append(s)
                self._socket_map[s.getsockname()] = host_id
            except Exception, e:
                if opt_debug:
                    raise
                sys.stderr.write("Failed to open %s:%d: %s\n" %
                    (address[0], address[1], e))
                raise


    def _serve(self):
        while True:
            try:
                try:
                    readable = select.select(self._sockets, [], [], 0.2)[0]
                except select.error, e:
                    if e[0] == 4:
                        continue
                    else:
                        raise

                for s in readable:
                    time.sleep(0.5)
                    client_socket, addr_info = s.accept()
                    host_id = self._socket_map[s.getsockname()]
                    client_socket.sendall(self._get_answer(host_id))
                    client_socket.close()
            except (KeyboardInterrupt, MKTerminate):
                break
            except Exception, e:
                sys.stderr.write("Error in listener: %s - %s\n" % (type(e), e))


    def _close_sockets(self):
        for s in self._sockets:
            s.close()
        del self._sockets[:]


    def _get_answer(self, host_id):
        agent_data = static_agent_data
        agent_data += self._section_kernel(host_id)
        return agent_data


    def _update_counter(self, host_id, ident, per_sec):
        counters = self._counters.setdefault(host_id, {})

        now = time.time()
        if ident not in counters:
            counters[ident] = now, 0
            return # just initialied. Don't increase

        else:
            last_update, last_value = counters[ident]
            duration = now - last_update

            counters[ident] = now, last_value + (duration * per_sec)


    def _section_kernel(self, host_id):
        self._update_counter(host_id, "kernel_ctxt",         per_sec=5000)
        self._update_counter(host_id, "kernel_processes",    per_sec=1)
        self._update_counter(host_id, "kernel_pgmajfault",   per_sec=50)

        # hundrets of a second
        num_cpus = 4
        for i in [ "" ] + range(num_cpus):
            self._update_counter(host_id, "kernel_cpu%s_user" % i,    per_sec=0.10)
            self._update_counter(host_id, "kernel_cpu%s_nice" % i,    per_sec=0.05)
            self._update_counter(host_id, "kernel_cpu%s_system" % i,  per_sec=0.10)
            self._update_counter(host_id, "kernel_cpu%s_idle" % i,    per_sec=0.30)
            self._update_counter(host_id, "kernel_cpu%s_iowait" % i,  per_sec=0.05)


        counters = dict([ (k, v) for k, (t, v) in self._counters[host_id].items() ])
        counters["now"] = time.time()

        cpu_output = ""
        for i in [ "" ] + range(num_cpus):
            cpu_output += "cpu%s %d %d %d %d %d 0 0 0 0 0\n" % \
                (i,
                 counters["kernel_cpu%s_user" % i],
                 counters["kernel_cpu%s_nice" % i],
                 counters["kernel_cpu%s_system" % i],
                 counters["kernel_cpu%s_idle" % i],
                 counters["kernel_cpu%s_iowait" % i])

        section = """<<<kernel>>>
%(now)d
nr_free_pages 2544893
nr_alloc_batch 3169
nr_inactive_anon 52641
nr_active_anon 980946
nr_inactive_file 87489
nr_active_file 314428
nr_unevictable 0
nr_mlock 0
nr_anon_pages 979995
nr_mapped 82615
nr_file_pages 455554
nr_dirty 10237
nr_writeback 0
nr_slab_reclaimable 34034
nr_slab_unreclaimable 15947
nr_page_table_pages 17569
nr_kernel_stack 927
nr_unstable 0
nr_bounce 0
nr_vmscan_write 0
nr_vmscan_immediate_reclaim 0
nr_writeback_temp 0
nr_isolated_anon 0
nr_isolated_file 0
nr_shmem 53638
nr_dirtied 3163803
nr_written 1642042
nr_pages_scanned 0
numa_hit 43145578
numa_miss 0
numa_foreign 0
numa_interleave 32848
numa_local 43145578
numa_other 0
workingset_refault 0
workingset_activate 0
workingset_nodereclaim 0
nr_anon_transparent_hugepages 495
nr_free_cma 0
nr_dirty_threshold 580548
nr_dirty_background_threshold 289919
pgpgin 886510
pgpgout 6647136
pswpin 0
pswpout 0
pgalloc_dma 0
pgalloc_dma32 5351681
pgalloc_normal 40330396
pgalloc_movable 0
pgfree 48229229
pgactivate 194701
pgdeactivate 0
pgfault 45121049
pgmajfault %(kernel_pgmajfault)d
pgrefill_dma 0
pgrefill_dma32 0
pgrefill_normal 0
pgrefill_movable 0
pgsteal_kswapd_dma 0
pgsteal_kswapd_dma32 0
pgsteal_kswapd_normal 0
pgsteal_kswapd_movable 0
pgsteal_direct_dma 0
pgsteal_direct_dma32 0
pgsteal_direct_normal 0
pgsteal_direct_movable 0
pgscan_kswapd_dma 0
pgscan_kswapd_dma32 0
pgscan_kswapd_normal 0
pgscan_kswapd_movable 0
pgscan_direct_dma 0
pgscan_direct_dma32 0
pgscan_direct_normal 0
pgscan_direct_movable 0
pgscan_direct_throttle 0
zone_reclaim_failed 0
pginodesteal 0
slabs_scanned 0
kswapd_inodesteal 0
kswapd_low_wmark_hit_quickly 0
kswapd_high_wmark_hit_quickly 0
pageoutrun 1
allocstall 0
pgrotated 0
drop_pagecache 0
drop_slab 0
numa_pte_updates 0
numa_huge_pte_updates 0
numa_hint_faults 0
numa_hint_faults_local 0
numa_pages_migrated 0
pgmigrate_success 0
pgmigrate_fail 0
compact_migrate_scanned 0
compact_free_scanned 0
compact_isolated 0
compact_stall 0
compact_fail 0
compact_success 0
htlb_buddy_alloc_success 0
htlb_buddy_alloc_fail 0
unevictable_pgs_culled 2964
unevictable_pgs_scanned 0
unevictable_pgs_rescued 1580
unevictable_pgs_mlocked 4145
unevictable_pgs_munlocked 4145
unevictable_pgs_cleared 0
unevictable_pgs_stranded 0
thp_fault_alloc 2907
thp_fault_fallback 0
thp_collapse_alloc 500
thp_collapse_alloc_failed 0
thp_split 428
thp_zero_page_alloc 1
thp_zero_page_alloc_failed 0
balloon_inflate 0
balloon_deflate 0
balloon_migrate 0
intr 1664579 22 0 0 0 0 0 0 0 1 33196 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 20295 70340 0 0 0 0 0 0 0 2419 352 0 11 571 752 1616 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
ctxt %(kernel_ctxt)d
btime 1484812069
processes %(kernel_processes)d
procs_running 1
procs_blocked 0
softirq 3526225 0 845354 2 177933 55525 0 6372 577174 0 1863865
""" % counters

        return section + cpu_output


static_agent_data = """<<<check_mk>>>
Version: 2016.12.23
AgentOS: linux
Hostname: Klappspaten
AgentDirectory: /etc/check_mk
DataDirectory: /var/lib/check_mk_agent
SpoolDirectory: /var/lib/check_mk_agent/spool
PluginsDirectory: /usr/lib/check_mk_agent/plugins
LocalDirectory: /usr/lib/check_mk_agent/local
<<<df>>>
/dev/mapper/rootvg-lv_root         ext4     1032088     509248    470412      52% /
/dev/sda1                          ext4      516040      84856    404972      18% /boot
/dev/mapper/rootvg-lv_home         ext4     3096336    1603828   1335248      55% /home
/dev/mapper/rootvg-lv_opt          ext4     1548144     409064   1060452      28% /opt
/dev/mapper/rootvg-lv_tmp          ext4     2064208      64980   1894392       4% /tmp
/dev/mapper/rootvg-lv_usr          ext4     4128448    1738152   2180584      45% /usr
/dev/mapper/rootvg-lv_local        ext4      516040      44136    445692      10% /usr/local
/dev/mapper/rootvg-lv_var          ext4     2064208    1524036    435328      78% /var
/dev/mapper/rootvg-lv_install      ext4    34060032   10740480  21589632      34% /install
/dev/mapper/rootvg-lv_agent        ext4     4128448    1975432   1943316      51% /agent
/dev/mapper/dwhvg-lv_dingdata   ext4   258030980  103616116 151793424      41% /dingdata
/dev/mapper/dwhvg-lv_conndir       ext4     1032088      61064    918596       7% /conndir
/dev/mapper/dwhvg-lv_backup        ext4    61927420   11445540  47336152      20% /backup
<<<df>>>
[df_inodes_start]
/dev/mapper/rootvg-lv_root         ext4     65536  13287     52249   21% /
/dev/sda1                          ext4     32768     50     32718    1% /boot
/dev/mapper/rootvg-lv_home         ext4    196608   3193    193415    2% /home
/dev/mapper/rootvg-lv_opt          ext4     98304    637     97667    1% /opt
/dev/mapper/rootvg-lv_tmp          ext4    131072    161    130911    1% /tmp
/dev/mapper/rootvg-lv_usr          ext4    262144  62008    200136   24% /usr
/dev/mapper/rootvg-lv_local        ext4     32768    124     32644    1% /usr/local
/dev/mapper/rootvg-lv_var          ext4    131072  61541     69531   47% /var
/dev/mapper/rootvg-lv_install      ext4   2162688   8503   2154185    1% /install
/dev/mapper/rootvg-lv_agent        ext4    262144  12469    249675    5% /agent
/dev/mapper/dwhvg-lv_dingdata   ext4  16384000   2429  16381571    1% /dingdata
/dev/mapper/dwhvg-lv_conndir       ext4     65536    205     65331    1% /conndir
/dev/mapper/dwhvg-lv_backup        ext4   3932160     82   3932078    1% /backup
[df_inodes_end]
<<<nfsmounts>>>
<<<mounts>>>
/dev/mapper/rootvg-lv_root / ext4 rw,relatime,barrier=1,data=ordered 0 0
/dev/sda1 /boot ext4 rw,relatime,barrier=1,data=ordered 0 0
/dev/mapper/rootvg-lv_home /home ext4 rw,relatime,barrier=0,data=ordered 0 0
/dev/mapper/rootvg-lv_opt /opt ext4 rw,relatime,barrier=0,data=ordered 0 0
/dev/mapper/rootvg-lv_tmp /tmp ext4 rw,relatime,barrier=0,data=ordered 0 0
/dev/mapper/rootvg-lv_usr /usr ext4 rw,relatime,barrier=0,data=ordered 0 0
/dev/mapper/rootvg-lv_local /usr/local ext4 rw,relatime,barrier=0,data=ordered 0 0
/dev/mapper/rootvg-lv_var /var ext4 rw,relatime,barrier=0,data=ordered 0 0
/dev/mapper/rootvg-lv_install /install ext4 rw,relatime,barrier=0,data=ordered 0 0
/dev/mapper/rootvg-lv_agent /agent ext4 rw,relatime,barrier=0,data=ordered 0 0
/dev/mapper/dwhvg-lv_dingdata /dingdata ext4 rw,relatime,barrier=0,data=ordered 0 0
/dev/mapper/dwhvg-lv_conndir /conndir ext4 rw,relatime,barrier=0,data=ordered 0 0
/dev/mapper/dwhvg-lv_backup /backup ext4 rw,relatime,barrier=0,data=ordered 0 0
<<<ps>>>
(root,120480,6764,00:00:02/24:28,1) /sbin/init splash
(root,0,0,00:00:00/24:28,2) [kthreadd]
(root,0,0,00:00:00/24:28,3) [ksoftirqd/0]
(root,0,0,00:00:00/24:28,5) [kworker/0:0H]
(root,0,0,00:00:02/24:28,7) [rcu_sched]
(root,0,0,00:00:00/24:28,8) [rcu_bh]
(root,0,0,00:00:00/24:28,9) [migration/0]
(root,0,0,00:00:00/24:28,10) [watchdog/0]
(root,0,0,00:00:00/24:28,11) [watchdog/1]
(root,0,0,00:00:00/24:28,12) [migration/1]
(root,0,0,00:00:00/24:28,13) [ksoftirqd/1]
(root,0,0,00:00:00/24:28,15) [kworker/1:0H]
(root,0,0,00:00:00/24:28,16) [watchdog/2]
(root,0,0,00:00:00/24:28,17) [migration/2]
(root,0,0,00:00:00/24:28,18) [ksoftirqd/2]
(root,0,0,00:00:00/24:28,20) [kworker/2:0H]
(root,0,0,00:00:00/24:28,21) [watchdog/3]
(root,0,0,00:00:00/24:28,22) [migration/3]
(root,0,0,00:00:00/24:28,23) [ksoftirqd/3]
(root,0,0,00:00:00/24:28,25) [kworker/3:0H]
(root,0,0,00:00:00/24:28,26) [watchdog/4]
(root,0,0,00:00:00/24:28,27) [migration/4]
(root,0,0,00:00:00/24:28,28) [ksoftirqd/4]
(root,0,0,00:00:00/24:28,29) [kworker/4:0]
(root,0,0,00:00:00/24:28,30) [kworker/4:0H]
(root,0,0,00:00:00/24:28,31) [watchdog/5]
(root,0,0,00:00:00/24:28,32) [migration/5]
(root,0,0,00:00:00/24:28,33) [ksoftirqd/5]
(root,0,0,00:00:00/24:28,34) [kworker/5:0]
(root,0,0,00:00:00/24:28,35) [kworker/5:0H]
(root,0,0,00:00:00/24:28,36) [watchdog/6]
(root,0,0,00:00:00/24:28,37) [migration/6]
(root,0,0,00:00:00/24:28,38) [ksoftirqd/6]
(root,0,0,00:00:00/24:28,40) [kworker/6:0H]
(root,0,0,00:00:00/24:28,41) [watchdog/7]
(root,0,0,00:00:00/24:28,42) [migration/7]
(root,0,0,00:00:00/24:28,43) [ksoftirqd/7]
(root,0,0,00:00:00/24:28,45) [kworker/7:0H]
(root,0,0,00:00:00/24:28,46) [kdevtmpfs]
(root,0,0,00:00:00/24:28,47) [netns]
(root,0,0,00:00:00/24:28,48) [perf]
(root,0,0,00:00:00/24:28,49) [khungtaskd]
(root,0,0,00:00:00/24:28,50) [writeback]
(root,0,0,00:00:00/24:28,51) [ksmd]
(root,0,0,00:00:00/24:28,52) [khugepaged]
(root,0,0,00:00:00/24:28,53) [crypto]
(root,0,0,00:00:00/24:28,54) [kintegrityd]
(root,0,0,00:00:00/24:28,55) [bioset]
(root,0,0,00:00:00/24:28,56) [kblockd]
(root,0,0,00:00:01/24:28,58) [kworker/u16:1]
(root,0,0,00:00:00/24:28,59) [ata_sff]
(root,0,0,00:00:00/24:28,60) [md]
(root,0,0,00:00:00/24:28,61) [devfreq_wq]
(root,0,0,00:00:00/24:28,62) [kworker/2:1]
(root,0,0,00:00:00/24:28,65) [kswapd0]
(root,0,0,00:00:00/24:28,66) [vmstat]
(root,0,0,00:00:00/24:28,67) [fsnotify_mark]
(root,0,0,00:00:00/24:28,68) [ecryptfs-kthrea]
(root,0,0,00:00:00/24:28,84) [kthrotld]
(root,0,0,00:00:00/24:28,85) [kworker/1:1]
(root,0,0,00:00:00/24:28,88) [kworker/6:1]
(root,0,0,00:00:00/24:28,89) [kworker/7:1]
(root,0,0,00:00:00/24:28,90) [acpi_thermal_pm]
(root,0,0,00:00:00/24:28,91) [bioset]
(root,0,0,00:00:00/24:28,92) [bioset]
(root,0,0,00:00:00/24:28,93) [bioset]
(root,0,0,00:00:00/24:28,94) [bioset]
(root,0,0,00:00:00/24:28,95) [bioset]
(root,0,0,00:00:00/24:28,96) [bioset]
(root,0,0,00:00:00/24:28,97) [bioset]
(root,0,0,00:00:00/24:28,98) [bioset]
(root,0,0,00:00:00/24:28,99) [bioset]
(root,0,0,00:00:00/24:28,100) [bioset]
(root,0,0,00:00:00/24:28,101) [bioset]
(root,0,0,00:00:00/24:28,102) [bioset]
(root,0,0,00:00:00/24:28,103) [bioset]
(root,0,0,00:00:00/24:28,104) [bioset]
(root,0,0,00:00:00/24:28,105) [bioset]
(root,0,0,00:00:00/24:28,106) [bioset]
(root,0,0,00:00:00/24:28,107) [bioset]
(root,0,0,00:00:00/24:28,108) [bioset]
(root,0,0,00:00:00/24:28,109) [bioset]
(root,0,0,00:00:00/24:28,110) [bioset]
(root,0,0,00:00:00/24:28,111) [bioset]
(root,0,0,00:00:00/24:28,112) [bioset]
(root,0,0,00:00:00/24:28,113) [bioset]
(root,0,0,00:00:00/24:28,114) [bioset]
(root,0,0,00:00:01/24:27,115) [kworker/u16:2]
(root,0,0,00:00:00/24:27,118) [kworker/1:2]
(root,0,0,00:00:00/24:27,122) [ipv6_addrconf]
(root,0,0,00:00:00/24:27,135) [deferwq]
(root,0,0,00:00:00/24:27,136) [charger_manager]
(root,0,0,00:00:00/24:27,177) [scsi_eh_0]
(root,0,0,00:00:00/24:27,178) [scsi_tmf_0]
(root,0,0,00:00:00/24:27,179) [usb-storage]
(root,0,0,00:00:01/24:26,211) [irq/36-(null)]
(root,0,0,00:00:00/24:26,212) [scsi_eh_1]
(root,0,0,00:00:00/24:26,213) [scsi_tmf_1]
(root,0,0,00:00:00/24:26,218) [bioset]
(root,0,0,00:00:00/24:26,289) [bioset]
(root,0,0,00:00:01/24:26,291) [kworker/u16:4]
(root,0,0,00:00:00/24:21,300) [kworker/7:1H]
(root,0,0,00:00:00/24:20,301) [kworker/3:1H]
(root,0,0,00:00:00/24:19,311) [kdmflush]
(root,0,0,00:00:00/24:19,348) [bioset]
(root,0,0,00:00:00/24:19,349) [kcryptd_io]
(root,0,0,00:00:00/24:19,350) [kcryptd]
(root,0,0,00:00:00/24:19,351) [dmcrypt_write]
(root,0,0,00:00:00/24:19,352) [bioset]
(root,0,0,00:00:00/24:19,362) [kdmflush]
(root,0,0,00:00:00/24:19,364) [bioset]
(root,0,0,00:00:00/24:19,367) [kdmflush]
(root,0,0,00:00:00/24:19,368) [bioset]
(root,0,0,00:00:00/02:55,393) [kworker/u16:8]
(root,0,0,00:00:00/24:19,401) [jbd2/dm-1-8]
(root,0,0,00:00:00/24:19,402) [ext4-rsv-conver]
(root,0,0,00:00:01/24:19,404) [kworker/u16:5]
(root,0,0,00:00:01/24:19,418) [kworker/u16:6]
(root,0,0,00:00:01/24:19,419) [kworker/u16:7]
(root,0,0,00:00:00/24:18,440) [kauditd]
(root,35496,7576,00:00:00/24:18,461) /lib/systemd/systemd-journald
(root,0,0,00:00:00/24:18,477) [rpciod]
(root,168504,3752,00:00:00/24:18,490) /sbin/lvmetad -f
(root,45468,4628,00:00:00/24:18,499) /lib/systemd/systemd-udevd
(root,0,0,00:00:00/24:18,584) [kworker/4:2]
(root,0,0,00:00:00/24:18,612) [kworker/6:2]
(root,0,0,00:00:00/24:18,616) [irq/47-mei_me]
(root,0,0,00:00:00/24:18,632) [cfg80211]
(root,0,0,00:00:00/24:18,657) [kvm-irqfd-clean]
(root,0,0,00:00:00/24:18,658) [kworker/4:1H]
(root,0,0,00:00:00/24:18,662) [kworker/0:1H]
(root,0,0,00:00:00/24:18,691) [kworker/2:3]
(root,0,0,00:00:00/24:18,723) [kworker/6:1H]
(root,0,0,00:00:00/24:18,724) [kworker/2:1H]
(root,0,0,00:00:00/24:18,832) [kworker/1:1H]
(root,0,0,00:00:00/24:18,874) [applesmc-led]
(root,0,0,00:00:00/24:18,917) [ext4-rsv-conver]
(root,23536,216,00:00:00/24:18,1015) /usr/sbin/rpc.idmapd
(root,47628,3556,00:00:00/24:18,1095) /sbin/rpcbind -f -w
(root,25216,4108,00:00:00/24:18,1112) /usr/sbin/smartd -n
(mongodb,4484316,68116,00:00:03/24:18,1114) /usr/bin/mongod --config /etc/mongodb.conf
(root,30536,3288,00:00:00/24:18,1115) /usr/sbin/cron -f
(daemon,26044,2144,00:00:00/24:18,1119) /usr/sbin/atd -f
(root,4400,1320,00:00:00/24:18,1121) /usr/sbin/acpid
(root,0,0,00:00:00/24:18,1131) [kworker/5:3]
(root,7492,948,00:00:00/24:18,1135) /usr/sbin/mbpfan -f
(avahi,44932,3092,00:00:00/24:18,1137) avahi-daemon: running [Klappspaten.local]
(root,278044,7180,00:00:00/24:18,1145) /usr/lib/accountsservice/accounts-daemon
(root,10708,856,00:00:00/24:18,1147) /usr/sbin/avahi-dnsconfd -s
(syslog,256396,3140,00:00:00/24:18,1169) /usr/sbin/rsyslogd -n
(root,337316,8356,00:00:00/24:18,1172) /usr/sbin/ModemManager
(messagebus,44276,5052,00:00:01/24:18,1178) /usr/bin/dbus-daemon --system --address=systemd: --nofork --nopidfile --systemd-activation
(avahi,44788,344,00:00:00/24:18,1195) avahi-daemon: chroot helper
(root,0,0,00:00:00/24:18,1238) [kworker/5:1H]
(root,215828,18372,00:00:00/24:18,1241) /usr/lib/snapd/snapd
(whoopsie,269224,9168,00:00:00/24:18,1243) /usr/bin/whoopsie -f
(root,28680,3124,00:00:00/24:18,1245) /lib/systemd/systemd-logind
(root,274960,9424,00:00:00/24:18,1247) /usr/sbin/cups-browsed
(root,524564,16932,00:00:00/24:18,1257) /usr/sbin/NetworkManager --no-daemon
(root,0,0,00:00:00/24:18,1292) [iprt]
(root,0,0,00:00:00/24:17,1315) [irq/50-brcmf_pc]
(root,0,0,00:00:00/24:17,1316) [msgbuf_txflow]
(colord,302680,11180,00:00:00/24:17,1333) /usr/lib/colord/colord
(root,283576,10984,00:00:00/24:17,1334) /usr/lib/policykit-1/polkitd --no-debug
(root,0,0,00:00:00/24:17,1401) [kworker/3:2]
(root,44128,6368,00:00:00/24:17,1402) /sbin/wpa_supplicant -u -s -O /run/wpa_supplicant
(root,0,0,00:00:00/24:17,1421) [kworker/u17:0]
(root,0,0,00:00:00/24:17,1422) [hci0]
(root,0,0,00:00:00/24:17,1423) [hci0]
(root,0,0,00:00:00/24:17,1424) [kworker/u17:1]
(root,32084,4600,00:00:00/24:17,1433) /usr/lib/bluetooth/bluetoothd
(root,65520,5688,00:00:00/24:17,1629) /usr/sbin/sshd -D
(root,37976,900,00:00:00/24:17,1636) /usr/sbin/rpc.mountd --manage-gids
(root,0,0,00:00:00/24:17,1642) [nfsd4_callbacks]
(root,0,0,00:00:00/24:17,1643) [lockd]
(root,0,0,00:00:00/24:17,1645) [nfsd]
(root,0,0,00:00:00/24:17,1646) [nfsd]
(root,0,0,00:00:00/24:17,1647) [nfsd]
(root,0,0,00:00:00/24:17,1648) [nfsd]
(root,0,0,00:00:00/24:17,1649) [nfsd]
(root,0,0,00:00:00/24:17,1650) [nfsd]
(root,0,0,00:00:00/24:17,1651) [nfsd]
(root,0,0,00:00:00/24:17,1652) [nfsd]
(root,209964,13996,00:00:04/24:17,1718) /opt/teamviewer/tv_bin/teamviewerd -d
(root,19568,2216,00:00:00/24:17,1770) /usr/sbin/irqbalance --pid=/var/run/irqbalance.pid
(ntp,110036,5144,00:00:00/24:17,1772) /usr/sbin/ntpd -p /var/run/ntpd.pid -g -u 108:115
(root,15056,2096,00:00:00/24:17,1818) /usr/sbin/xinetd -pidfile /run/xinetd.pid -stayalive -inetd_compat -inetd_ipv6
(root,276648,6344,00:00:00/24:17,1840) /usr/sbin/lightdm
(snmp,61284,8000,00:00:00/24:17,1842) /usr/sbin/snmpd -Lsd -Lf /dev/null -u snmp -g snmp -I -smux mteTrigger mteTriggerConf -p /run/snmpd.pid
(uml-net,4376,792,00:00:00/24:17,1868) /usr/bin/uml_switch -tap tap0 -unix /var/run/uml-utilities/uml_switch.ctl
(root,83636,6164,00:00:00/24:17,1906) /usr/sbin/apache2 -k start
(www-data,83552,3692,00:00:00/24:17,1911) /usr/sbin/apache2 -k start
(www-data,83948,4356,00:00:00/24:17,1912) /usr/sbin/apache2 -k start
(www-data,83948,4356,00:00:00/24:17,1913) /usr/sbin/apache2 -k start
(www-data,83980,4356,00:00:00/24:17,1915) /usr/sbin/apache2 -k start
(www-data,83980,4356,00:00:00/24:17,1916) /usr/sbin/apache2 -k start
(www-data,83948,4356,00:00:00/24:17,1917) /usr/sbin/apache2 -k start
(root,537576,93684,00:00:10/24:17,1991) /usr/lib/xorg/Xorg -core :0 -seat seat0 -auth /var/run/lightdm/root/:0 -nolisten tcp vt7 -novtswitch
(root,17468,1704,00:00:00/24:17,1992) /sbin/agetty --noclear tty1 linux
(root,65408,4548,00:00:00/24:16,2027) /usr/lib/postfix/sbin/master
(postfix,67476,4416,00:00:00/24:16,2029) pickup -l -t unix -u -c
(postfix,67648,4536,00:00:00/24:16,2030) qmgr -l -t unix -u
(heute,53952,7716,00:00:00/24:16,2046) python /omd/sites/heute/bin/liveproxyd
(root,241532,5356,00:00:00/24:16,2052) /usr/sbin/nmbd -D
(heute,78568,9460,00:00:00/24:16,2073) python /omd/sites/heute/bin/mknotifyd
(root,339396,15748,00:00:00/24:16,2168) /usr/sbin/smbd -D
(root,331424,5816,00:00:00/24:16,2169) /usr/sbin/smbd -D
(root,339404,8928,00:00:00/24:16,2171) /usr/sbin/smbd -D
(heute,101268,10584,00:00:00/24:16,2173) /usr/sbin/apache2 -f /omd/sites/heute/etc/apache/apache.conf
(heute,101228,4196,00:00:00/24:16,2176) /usr/sbin/apache2 -f /omd/sites/heute/etc/apache/apache.conf
(heute,311868,68852,00:00:01/24:16,2177) /usr/sbin/apache2 -f /omd/sites/heute/etc/apache/apache.conf
(root,0,0,00:00:00/10:32,2189) [kworker/u16:10]
(root,228240,6648,00:00:00/24:16,2190) lightdm --session-child 12 15
(heute2,221652,12080,00:00:00/24:16,2248) python /omd/sites/heute2/bin/mkeventd
(heute2,53952,7448,00:00:00/24:16,2279) python /omd/sites/heute2/bin/liveproxyd
(heute2,78852,9724,00:00:00/24:16,2289) python /omd/sites/heute2/bin/mknotifyd
(heute2,251656,3172,00:00:00/24:15,2305) /omd/sites/heute2/bin/rrdcached -w 3600 -z 1800 -f 7200 -s heute2 -m 660 -l unix:/omd/sites/heute2/tmp/run/rrdcached.sock -p /omd/sites/heute2/tmp/rrdcached.pid -j /omd/sites/heute2/var/rrdcached
(heute2,1530808,8520,00:00:00/24:15,2341) /omd/sites/heute2/bin/cmc /omd/sites/heute2/var/check_mk/core/config
(heute2,218676,38616,00:00:01/24:15,2379) python /omd/sites/heute2/bin/cmk --create-rrd --keepalive
(heute2,100348,27400,00:00:00/24:15,2383) python /omd/sites/heute2/bin/cmk --handle-alerts --keepalive
(heute2,100644,28100,00:00:00/24:15,2387) python /omd/sites/heute2/bin/cmk --notify --keepalive
(heute2,10492,1068,00:00:00/24:15,2391) checkhelper
(heute2,10492,1004,00:00:00/24:15,2395) checkhelper
(heute2,10492,968,00:00:00/24:15,2397) checkhelper
(heute2,10492,1004,00:00:00/24:15,2398) checkhelper
(heute2,10492,1016,00:00:00/24:15,2399) checkhelper
(heute2,104244,31840,00:00:01/24:15,2401) python /omd/sites/heute2/bin/cmk --keepalive
(heute2,104760,32396,00:00:00/24:15,2404) python /omd/sites/heute2/bin/cmk --keepalive
(heute2,100344,27364,00:00:00/24:15,2405) python /omd/sites/heute2/bin/cmk --keepalive
(heute2,100348,27540,00:00:00/24:15,2406) python /omd/sites/heute2/bin/cmk --keepalive
(heute2,100348,27516,00:00:00/24:15,2407) python /omd/sites/heute2/bin/cmk --keepalive
(heute2,100360,27488,00:00:00/24:15,2408) python /omd/sites/heute2/bin/cmk --keepalive
(heute2,100348,27480,00:00:00/24:15,2409) python /omd/sites/heute2/bin/cmk --keepalive
(heute2,100348,27564,00:00:00/24:15,2411) python /omd/sites/heute2/bin/cmk --keepalive
(heute2,100348,27456,00:00:00/24:15,2413) python /omd/sites/heute2/bin/cmk --keepalive
(heute2,100352,27316,00:00:00/24:15,2414) python /omd/sites/heute2/bin/cmk --keepalive
(heute2,100352,27388,00:00:00/24:15,2415) python /omd/sites/heute2/bin/cmk --keepalive
(heute2,100352,27468,00:00:00/24:15,2416) python /omd/sites/heute2/bin/cmk --keepalive
(heute2,100356,27472,00:00:00/24:15,2417) python /omd/sites/heute2/bin/cmk --keepalive
(heute2,100360,27376,00:00:00/24:15,2418) python /omd/sites/heute2/bin/cmk --keepalive
(heute2,100356,27540,00:00:00/24:15,2419) python /omd/sites/heute2/bin/cmk --keepalive
(heute2,100356,27512,00:00:00/24:15,2420) python /omd/sites/heute2/bin/cmk --keepalive
(heute2,100348,27400,00:00:00/24:15,2421) python /omd/sites/heute2/bin/cmk --keepalive
(heute2,100348,27444,00:00:00/24:15,2422) python /omd/sites/heute2/bin/cmk --keepalive
(heute2,100348,27600,00:00:00/24:15,2441) python /omd/sites/heute2/bin/cmk --keepalive
(heute2,100352,27300,00:00:00/24:15,2442) python /omd/sites/heute2/bin/cmk --keepalive
(heute2,102436,27812,00:00:00/24:15,2443) python /omd/sites/heute2/bin/cmk --keepalive --real-time-checks
(root,12596,1060,00:00:00/24:15,2448) icmpsender 8 0 1000
(root,14848,1024,00:00:00/24:15,2449) icmpreceiver
(heute2,101268,10624,00:00:00/24:15,2450) /usr/sbin/apache2 -f /omd/sites/heute2/etc/apache/apache.conf
(heute2,101228,4216,00:00:00/24:15,2457) /usr/sbin/apache2 -f /omd/sites/heute2/etc/apache/apache.conf
(heute2,313420,70412,00:00:01/24:15,2458) /usr/sbin/apache2 -f /omd/sites/heute2/etc/apache/apache.conf
(test1,221484,11476,00:00:00/24:15,2511) python /omd/sites/test1/bin/mkeventd
(rtkit,183544,3124,00:00:00/24:15,2524) /usr/lib/rtkit/rtkit-daemon
(test1,54204,7964,00:00:00/24:15,2526) python /omd/sites/test1/bin/liveproxyd
(test1,78848,9756,00:00:00/24:15,2568) python /omd/sites/test1/bin/mknotifyd
(root,367748,8604,00:00:00/24:14,2580) /usr/lib/udisks2/udisksd --no-debug
(test1,177660,3136,00:00:00/24:14,2582) /omd/sites/test1/bin/rrdcached -w 3600 -z 1800 -f 7200 -s test1 -m 660 -l unix:/omd/sites/test1/tmp/run/rrdcached.sock -p /omd/sites/test1/tmp/rrdcached.pid -j /omd/sites/test1/var/rrdcached
(test1,1529284,6224,00:00:00/24:14,2642) /omd/sites/test1/bin/cmc /omd/sites/test1/var/check_mk/core/config
(test1,218580,38660,00:00:01/24:14,2698) python /omd/sites/test1/share/check_mk/modules/check_mk.py --create-rrd --keepalive
(test1,100332,27696,00:00:00/24:14,2700) python /omd/sites/test1/share/check_mk/modules/check_mk.py --handle-alerts --keepalive
(test1,100336,27624,00:00:00/24:14,2702) python /omd/sites/test1/share/check_mk/modules/check_mk.py --notify --keepalive
(root,0,0,00:00:00/24:14,2704) [krfcommd]
(test1,4364,632,00:00:00/24:14,2711) checkhelper
(test1,4364,636,00:00:00/24:14,2712) checkhelper
(test1,4364,784,00:00:00/24:14,2713) checkhelper
(test1,4364,652,00:00:00/24:14,2717) checkhelper
(test1,4364,740,00:00:00/24:14,2737) checkhelper
(test1,100332,27452,00:00:00/24:14,2738) python /omd/sites/test1/share/check_mk/modules/check_mk.py --keepalive
(test1,100332,27748,00:00:00/24:14,2739) python /omd/sites/test1/share/check_mk/modules/check_mk.py --keepalive
(test1,100332,27612,00:00:00/24:14,2741) python /omd/sites/test1/share/check_mk/modules/check_mk.py --keepalive
(test1,100336,27660,00:00:00/24:14,2742) python /omd/sites/test1/share/check_mk/modules/check_mk.py --keepalive
(root,349032,10440,00:00:00/24:14,2744) /usr/lib/upower/upowerd
(test1,100324,27752,00:00:00/24:14,2747) python /omd/sites/test1/share/check_mk/modules/check_mk.py --keepalive
(test1,100340,27676,00:00:00/24:14,2748) python /omd/sites/test1/share/check_mk/modules/check_mk.py --keepalive
(test1,100332,27608,00:00:00/24:14,2751) python /omd/sites/test1/share/check_mk/modules/check_mk.py --keepalive
(test1,100332,27612,00:00:00/24:14,2752) python /omd/sites/test1/share/check_mk/modules/check_mk.py --keepalive
(test1,100332,27712,00:00:00/24:14,2753) python /omd/sites/test1/share/check_mk/modules/check_mk.py --keepalive
(test1,100328,27660,00:00:00/24:14,2754) python /omd/sites/test1/share/check_mk/modules/check_mk.py --keepalive
(test1,100328,27692,00:00:00/24:14,2755) python /omd/sites/test1/share/check_mk/modules/check_mk.py --keepalive
(test1,100332,27572,00:00:00/24:14,2756) python /omd/sites/test1/share/check_mk/modules/check_mk.py --keepalive
(test1,100328,27416,00:00:00/24:14,2757) python /omd/sites/test1/share/check_mk/modules/check_mk.py --keepalive
(test1,100332,27668,00:00:00/24:14,2758) python /omd/sites/test1/share/check_mk/modules/check_mk.py --keepalive
(test1,100388,27732,00:00:00/24:14,2759) python /omd/sites/test1/share/check_mk/modules/check_mk.py --keepalive
(test1,100328,27916,00:00:00/24:14,2760) python /omd/sites/test1/share/check_mk/modules/check_mk.py --keepalive
(test1,100332,27444,00:00:00/24:14,2761) python /omd/sites/test1/share/check_mk/modules/check_mk.py --keepalive
(test1,100328,27524,00:00:00/24:14,2765) python /omd/sites/test1/share/check_mk/modules/check_mk.py --keepalive
(test1,100328,27800,00:00:00/24:14,2768) python /omd/sites/test1/share/check_mk/modules/check_mk.py --keepalive
(test1,100328,27412,00:00:00/24:14,2774) python /omd/sites/test1/share/check_mk/modules/check_mk.py --keepalive
(test1,102668,27744,00:00:00/24:14,2775) python /omd/sites/test1/share/check_mk/modules/check_mk.py --keepalive --real-time-checks
(root,12520,1160,00:00:00/24:14,2776) icmpsender 8 0 1000
(root,8720,752,00:00:00/24:14,2777) icmpreceiver
(heute,311868,68908,00:00:01/24:06,3812) /usr/sbin/apache2 -f /omd/sites/heute/etc/apache/apache.conf
(www-data,83948,4364,00:00:00/24:06,4132) /usr/sbin/apache2 -k start
(test1,101264,10604,00:00:00/24:05,4165) /usr/sbin/apache2 -f /omd/sites/test1/etc/apache/apache.conf
(test1,15056,2052,00:00:00/24:05,4178) /omd/sites/test1/var/tmp/xinetd -pidfile /omd/sites/test1/tmp/run/xinetd.pid -filelog /omd/sites/test1/var/log/xinetd.log -f /omd/sites/test1/etc/xinetd.conf
(test1,101224,4192,00:00:00/24:05,4186) /usr/sbin/apache2 -f /omd/sites/test1/etc/apache/apache.conf
(test1,316088,70952,00:00:01/24:05,4187) /usr/sbin/apache2 -f /omd/sites/test1/etc/apache/apache.conf
(test2,221480,11520,00:00:00/24:05,4200) python /omd/sites/test2/bin/mkeventd
(test2,54204,7768,00:00:00/24:05,4209) python /omd/sites/test2/bin/liveproxyd
(test2,78848,9656,00:00:00/24:05,4219) python /omd/sites/test2/bin/mknotifyd
(test2,177660,3080,00:00:00/24:05,4224) /omd/sites/test2/bin/rrdcached -w 3600 -z 1800 -f 7200 -s test2 -m 660 -l unix:/omd/sites/test2/tmp/run/rrdcached.sock -p /omd/sites/test2/tmp/rrdcached.pid -j /omd/sites/test2/var/rrdcached
(test2,1529284,6060,00:00:00/24:05,4244) /omd/sites/test2/bin/cmc /omd/sites/test2/var/check_mk/core/config
(test2,218592,38636,00:00:01/24:05,4276) python /omd/sites/test2/share/check_mk/modules/check_mk.py --create-rrd --keepalive
(test2,100332,27728,00:00:00/24:05,4277) python /omd/sites/test2/share/check_mk/modules/check_mk.py --handle-alerts --keepalive
(test2,100332,27588,00:00:00/24:05,4279) python /omd/sites/test2/share/check_mk/modules/check_mk.py --notify --keepalive
(test2,4364,784,00:00:00/24:05,4280) checkhelper
(test2,4364,640,00:00:00/24:05,4282) checkhelper
(test2,4364,652,00:00:00/24:05,4283) checkhelper
(test2,4364,788,00:00:00/24:05,4284) checkhelper
(test2,4364,732,00:00:00/24:05,4285) checkhelper
(test2,100336,27776,00:00:00/24:05,4286) python /omd/sites/test2/share/check_mk/modules/check_mk.py --keepalive
(test2,100332,27708,00:00:00/24:05,4287) python /omd/sites/test2/share/check_mk/modules/check_mk.py --keepalive
(test2,100328,27540,00:00:00/24:05,4288) python /omd/sites/test2/share/check_mk/modules/check_mk.py --keepalive
(test2,100356,27716,00:00:00/24:05,4289) python /omd/sites/test2/share/check_mk/modules/check_mk.py --keepalive
(test2,100360,27400,00:00:00/24:05,4290) python /omd/sites/test2/share/check_mk/modules/check_mk.py --keepalive
(test2,100336,27848,00:00:00/24:05,4291) python /omd/sites/test2/share/check_mk/modules/check_mk.py --keepalive
(test2,100332,27704,00:00:00/24:05,4292) python /omd/sites/test2/share/check_mk/modules/check_mk.py --keepalive
(test2,100328,27660,00:00:00/24:05,4293) python /omd/sites/test2/share/check_mk/modules/check_mk.py --keepalive
(test2,100336,27696,00:00:00/24:05,4297) python /omd/sites/test2/share/check_mk/modules/check_mk.py --keepalive
(test2,100332,27612,00:00:00/24:05,4298) python /omd/sites/test2/share/check_mk/modules/check_mk.py --keepalive
(test2,100320,27428,00:00:00/24:05,4299) python /omd/sites/test2/share/check_mk/modules/check_mk.py --keepalive
(test2,100328,27712,00:00:00/24:05,4300) python /omd/sites/test2/share/check_mk/modules/check_mk.py --keepalive
(test2,100336,27620,00:00:00/24:05,4302) python /omd/sites/test2/share/check_mk/modules/check_mk.py --keepalive
(test2,100328,27612,00:00:00/24:05,4303) python /omd/sites/test2/share/check_mk/modules/check_mk.py --keepalive
(test2,100332,27584,00:00:00/24:05,4304) python /omd/sites/test2/share/check_mk/modules/check_mk.py --keepalive
(test2,100336,27572,00:00:00/24:05,4305) python /omd/sites/test2/share/check_mk/modules/check_mk.py --keepalive
(test2,100332,27844,00:00:00/24:05,4306) python /omd/sites/test2/share/check_mk/modules/check_mk.py --keepalive
(test2,100332,27764,00:00:00/24:05,4307) python /omd/sites/test2/share/check_mk/modules/check_mk.py --keepalive
(test2,100332,27680,00:00:00/24:05,4308) python /omd/sites/test2/share/check_mk/modules/check_mk.py --keepalive
(test2,100328,27468,00:00:00/24:05,4317) python /omd/sites/test2/share/check_mk/modules/check_mk.py --keepalive
(test2,102792,27924,00:00:00/24:05,4318) python /omd/sites/test2/share/check_mk/modules/check_mk.py --keepalive --real-time-checks
(root,12520,1080,00:00:00/24:05,4319) icmpsender 8 0 1000
(root,8720,732,00:00:00/24:05,4320) icmpreceiver
(test2,101264,10592,00:00:00/23:56,4874) /usr/sbin/apache2 -f /omd/sites/test2/etc/apache/apache.conf
(test2,15056,2056,00:00:00/23:56,4887) /omd/sites/test2/var/tmp/xinetd -pidfile /omd/sites/test2/tmp/run/xinetd.pid -filelog /omd/sites/test2/var/log/xinetd.log -f /omd/sites/test2/etc/xinetd.conf
(test2,101224,4188,00:00:00/23:56,4888) /usr/sbin/apache2 -f /omd/sites/test2/etc/apache/apache.conf
(test2,316032,70940,00:00:01/23:56,4889) /usr/sbin/apache2 -f /omd/sites/test2/etc/apache/apache.conf
(test_modes,221652,11624,00:00:00/23:56,4907) python /omd/sites/test_modes/bin/mkeventd
(test_modes,53952,7652,00:00:00/23:56,4915) python /omd/sites/test_modes/bin/liveproxyd
(test_modes,78856,9744,00:00:00/23:56,4923) python /omd/sites/test_modes/bin/mknotifyd
(test_modes,251392,3184,00:00:00/23:56,4927) /omd/sites/test_modes/bin/rrdcached -w 3600 -z 1800 -f 7200 -s test_modes -m 660 -l unix:/omd/sites/test_modes/tmp/run/rrdcached.sock -p /omd/sites/test_modes/tmp/rrdcached.pid -j /omd/sites/test_modes/var/rrdcached
(test_modes,1529456,6236,00:00:00/23:56,4944) /omd/sites/test_modes/bin/cmc /omd/sites/test_modes/var/check_mk/core/config
(test_modes,219156,43348,00:00:01/23:56,4974) python /omd/sites/test_modes/bin/cmk --create-rrd --keepalive
(test_modes,97708,31144,00:00:01/23:56,4976) python /omd/sites/test_modes/bin/cmk --handle-alerts --keepalive
(test_modes,97684,31224,00:00:01/23:56,4977) python /omd/sites/test_modes/bin/cmk --notify --keepalive
(test_modes,7396,2260,00:00:00/23:56,4978) checkhelper
(test_modes,7396,948,00:00:00/23:56,4979) checkhelper
(test_modes,7396,792,00:00:00/23:56,4980) checkhelper
(test_modes,7396,840,00:00:00/23:56,4981) checkhelper
(test_modes,7396,808,00:00:00/23:56,4984) checkhelper
(test_modes,108680,34108,00:00:01/23:56,4986) python /omd/sites/test_modes/bin/cmk --keepalive
(test_modes,97440,30660,00:00:01/23:56,4987) python /omd/sites/test_modes/bin/cmk --keepalive
(test_modes,97400,30628,00:00:01/23:56,4988) python /omd/sites/test_modes/bin/cmk --keepalive
(test_modes,97444,30652,00:00:01/23:56,4989) python /omd/sites/test_modes/bin/cmk --keepalive
(test_modes,97408,30580,00:00:01/23:56,4990) python /omd/sites/test_modes/bin/cmk --keepalive
(test_modes,97420,30624,00:00:01/23:56,4991) python /omd/sites/test_modes/bin/cmk --keepalive
(test_modes,97384,30736,00:00:01/23:56,4992) python /omd/sites/test_modes/bin/cmk --keepalive
(test_modes,97416,30640,00:00:01/23:56,4993) python /omd/sites/test_modes/bin/cmk --keepalive
(test_modes,97440,30620,00:00:01/23:56,4994) python /omd/sites/test_modes/bin/cmk --keepalive
(test_modes,97440,30576,00:00:01/23:56,4996) python /omd/sites/test_modes/bin/cmk --keepalive
(test_modes,97404,30560,00:00:01/23:56,4997) python /omd/sites/test_modes/bin/cmk --keepalive
(test_modes,97408,30744,00:00:01/23:56,4998) python /omd/sites/test_modes/bin/cmk --keepalive
(test_modes,97392,30572,00:00:01/23:56,4999) python /omd/sites/test_modes/bin/cmk --keepalive
(test_modes,97420,30568,00:00:01/23:56,5000) python /omd/sites/test_modes/bin/cmk --keepalive
(test_modes,97448,30556,00:00:01/23:56,5004) python /omd/sites/test_modes/bin/cmk --keepalive
(test_modes,97436,30704,00:00:01/23:56,5005) python /omd/sites/test_modes/bin/cmk --keepalive
(test_modes,97448,30660,00:00:01/23:56,5006) python /omd/sites/test_modes/bin/cmk --keepalive
(test_modes,97412,30568,00:00:01/23:56,5007) python /omd/sites/test_modes/bin/cmk --keepalive
(test_modes,97308,30680,00:00:01/23:56,5008) python /omd/sites/test_modes/bin/cmk --keepalive
(test_modes,97444,30708,00:00:01/23:55,5014) python /omd/sites/test_modes/bin/cmk --keepalive
(test_modes,99388,30532,00:00:01/23:55,5015) python /omd/sites/test_modes/bin/cmk --keepalive --real-time-checks
(root,12612,1048,00:00:00/23:55,5016) icmpsender 8 0 1000
(root,11756,872,00:00:00/23:55,5017) icmpreceiver
(test_modes,101524,10648,00:00:00/23:46,5763) /usr/sbin/apache2 -f /omd/sites/test_modes/etc/apache/apache.conf
(test_modes,101484,4220,00:00:00/23:46,5774) /usr/sbin/apache2 -f /omd/sites/test_modes/etc/apache/apache.conf
(test_modes,313392,70500,00:00:01/23:46,5775) /usr/sbin/apache2 -f /omd/sites/test_modes/etc/apache/apache.conf
(test_modes,15056,2172,00:00:00/23:46,5776) /omd/sites/test_modes/var/tmp/xinetd -pidfile /omd/sites/test_modes/tmp/run/xinetd.pid -filelog /omd/sites/test_modes/var/log/xinetd.log -f /omd/sites/test_modes/etc/xinetd.conf
(test_modes,313392,70520,00:00:01/23:16,6881) /usr/sbin/apache2 -f /omd/sites/test_modes/etc/apache/apache.conf
(test1,316088,70808,00:00:01/23:16,7013) /usr/sbin/apache2 -f /omd/sites/test1/etc/apache/apache.conf
(heute2,313420,70432,00:00:01/23:16,7014) /usr/sbin/apache2 -f /omd/sites/heute2/etc/apache/apache.conf
(test2,316032,71012,00:00:01/23:16,7023) /usr/sbin/apache2 -f /omd/sites/test2/etc/apache/apache.conf
(root,0,0,00:00:00/09:45,7836) [kworker/3:0]
(root,0,0,00:00:00/09:22,8288) [kworker/0:0]
(heute,221660,11908,00:00:00/02:07,9604) python /omd/sites/heute/bin/mkeventd
(heute,251392,3376,00:00:01/02:04,9785) /omd/sites/heute/bin/rrdcached -w 300 -z 10 -f 7200 -s heute -m 660 -l unix:/omd/sites/heute/tmp/run/rrdcached.sock -p /omd/sites/heute/tmp/rrdcached.pid -j /omd/sites/heute/var/rrdcached
(root,22668,2276,00:00:00/00:29,10284) /bin/bash /usr/bin/check_mk_agent
(heute,1529448,24964,00:00:00/00:27,10434) /omd/sites/heute/bin/cmc /omd/sites/heute/var/check_mk/core/config
(heute,218612,38580,00:00:01/00:27,10456) python /omd/sites/heute/share/check_mk/modules/check_mk.py --create-rrd --keepalive
(heute,100444,27424,00:00:00/00:27,10457) python /omd/sites/heute/share/check_mk/modules/check_mk.py --handle-alerts --keepalive
(heute,100388,27436,00:00:00/00:27,10458) python /omd/sites/heute/share/check_mk/modules/check_mk.py --notify --keepalive
(heute,7396,888,00:00:00/00:27,10459) checkhelper
(heute,7396,788,00:00:00/00:27,10460) checkhelper
(heute,7396,832,00:00:00/00:27,10461) checkhelper
(heute,7396,780,00:00:00/00:27,10462) checkhelper
(heute,7396,832,00:00:00/00:27,10463) checkhelper
(heute,100412,27500,00:00:00/00:27,10464) python /omd/sites/heute/share/check_mk/modules/check_mk.py --keepalive
(heute,100420,27440,00:00:00/00:27,10465) python /omd/sites/heute/share/check_mk/modules/check_mk.py --keepalive
(heute,100420,27532,00:00:00/00:27,10466) python /omd/sites/heute/share/check_mk/modules/check_mk.py --keepalive
(heute,100416,27468,00:00:00/00:27,10467) python /omd/sites/heute/share/check_mk/modules/check_mk.py --keepalive
(heute,100420,27360,00:00:00/00:27,10468) python /omd/sites/heute/share/check_mk/modules/check_mk.py --keepalive
(heute,100456,27572,00:00:00/00:27,10469) python /omd/sites/heute/share/check_mk/modules/check_mk.py --keepalive
(heute,100452,27428,00:00:00/00:27,10470) python /omd/sites/heute/share/check_mk/modules/check_mk.py --keepalive
(heute,100452,27488,00:00:00/00:27,10471) python /omd/sites/heute/share/check_mk/modules/check_mk.py --keepalive
(heute,100428,27608,00:00:00/00:27,10472) python /omd/sites/heute/share/check_mk/modules/check_mk.py --keepalive
(heute,100420,27320,00:00:00/00:27,10473) python /omd/sites/heute/share/check_mk/modules/check_mk.py --keepalive
(heute,100456,27272,00:00:00/00:27,10474) python /omd/sites/heute/share/check_mk/modules/check_mk.py --keepalive
(heute,100420,27492,00:00:00/00:27,10475) python /omd/sites/heute/share/check_mk/modules/check_mk.py --keepalive
(heute,100420,27364,00:00:00/00:27,10476) python /omd/sites/heute/share/check_mk/modules/check_mk.py --keepalive
(heute,100452,27512,00:00:00/00:27,10477) python /omd/sites/heute/share/check_mk/modules/check_mk.py --keepalive
(heute,100452,27344,00:00:00/00:27,10478) python /omd/sites/heute/share/check_mk/modules/check_mk.py --keepalive
(heute,100416,27540,00:00:00/00:27,10479) python /omd/sites/heute/share/check_mk/modules/check_mk.py --keepalive
(heute,100456,27584,00:00:00/00:27,10480) python /omd/sites/heute/share/check_mk/modules/check_mk.py --keepalive
(heute,100440,27344,00:00:00/00:27,10481) python /omd/sites/heute/share/check_mk/modules/check_mk.py --keepalive
(heute,100416,27628,00:00:00/00:27,10482) python /omd/sites/heute/share/check_mk/modules/check_mk.py --keepalive
(heute,100412,27448,00:00:00/00:27,10483) python /omd/sites/heute/share/check_mk/modules/check_mk.py --keepalive
(heute,102500,27776,00:00:00/00:27,10484) python /omd/sites/heute/share/check_mk/modules/check_mk.py --keepalive --real-time-checks
(root,12612,1092,00:00:00/00:27,10485) icmpsender 8 0 1000
(root,11756,864,00:00:00/00:27,10486) icmpreceiver
(root,0,0,00:00:00/18:03,15867) [kworker/0:2]
(root,0,0,00:00:00/05:28,17253) [kworker/u16:3]
(root,0,0,00:00:01/14:39,22323) [kworker/u16:0]
(root,0,0,00:00:00/14:39,22324) [kworker/u16:9]
(root,0,0,00:00:00/13:46,26453) [kworker/7:0]
<<<mem>>>
MemTotal:       16305836 kB
MemFree:        10181860 kB
MemAvailable:   11599688 kB
Buffers:          176928 kB
Cached:          1645288 kB
SwapCached:            0 kB
Active:          5181520 kB
Inactive:         560520 kB
Active(anon):    3923808 kB
Inactive(anon):   210564 kB
Active(file):    1257712 kB
Inactive(file):   349956 kB
Unevictable:           0 kB
Mlocked:               0 kB
SwapTotal:      16650236 kB
SwapFree:       16650236 kB
Dirty:             40948 kB
Writeback:             0 kB
AnonPages:       3919412 kB
Mapped:           330372 kB
Shmem:            214552 kB
Slab:             199928 kB
SReclaimable:     136136 kB
SUnreclaim:        63792 kB
KernelStack:       14832 kB
PageTables:        70708 kB
NFS_Unstable:          0 kB
Bounce:                0 kB
WritebackTmp:          0 kB
CommitLimit:    24803152 kB
Committed_AS:    7579556 kB
VmallocTotal:   34359738367 kB
VmallocUsed:           0 kB
VmallocChunk:          0 kB
HardwareCorrupted:     0 kB
AnonHugePages:   1013760 kB
CmaTotal:              0 kB
CmaFree:               0 kB
HugePages_Total:       0
HugePages_Free:        0
HugePages_Rsvd:        0
HugePages_Surp:        0
Hugepagesize:       2048 kB
DirectMap4k:      195228 kB
DirectMap2M:     8067072 kB
DirectMap1G:     8388608 kB
<<<cpu>>>
2.05 2.31 1.62 1/920 11110 8
<<<uptime>>>
1468.80 10506.46
<<<lnx_if:sep(58)>>>
    lo:462522167348 423391761    0    0    0     0          0         0 462522167348 423391761    0    0    0     0       0          0
  eth0:9663614968657 4226434136    0    0    0     0          0         0 41120440285551 1482439946    0    0    0     0       0          0
  eth1:9706218845482 3826133052    0    0    0     0          0         0 1746479361684 3786292851    0    0    0     0       0          0
  eth2:       0       0    0    0    0     0          0         0        0       0    0    0    0     0       0          0
  eth3:       0       0    0    0    0     0          0         0        0       0    0    0    0     0       0          0
  eth4:       0       0    0    0    0     0          0         0        0       0    0    0    0     0       0          0
  eth5:349900483789 5212016032    0    0    0     0          0         0 185966695259985 5288220641    0    0    0     0       0          0
 bond0:19369833814139 8052567188    0    0    0     0          0         0 42866919647235 5268732797    0    0    0     0       0          0
[lo]
	Link detected: yes
	Address: 00:00:00:00:00:00
[eth0]
	Speed: 1000Mb/s
	Duplex: Full
	Auto-negotiation: on
	Link detected: yes
	Address: e8:39:35:be:5d:90
[eth1]
	Speed: 1000Mb/s
	Duplex: Full
	Auto-negotiation: on
	Link detected: yes
	Address: e8:39:35:be:5d:90
[eth2]
	Speed: Unknown!
	Duplex: Unknown! (255)
	Auto-negotiation: off
	Link detected: no
	Address: e8:39:35:be:5d:92
[eth3]
	Speed: Unknown!
	Duplex: Unknown! (255)
	Auto-negotiation: off
	Link detected: no
	Address: e8:39:35:be:5d:93
[eth4]
	Speed: Unknown!
	Duplex: Unknown! (255)
	Auto-negotiation: off
	Link detected: no
	Address: 80:c1:6e:a9:22:38
[eth5]
	Speed: 10000Mb/s
	Duplex: Full
	Auto-negotiation: off
	Link detected: yes
	Address: 80:c1:6e:a9:22:3c
[bond0]
	Speed: 2000Mb/s
	Duplex: Full
	Auto-negotiation: off
	Link detected: yes
	Address: e8:39:35:be:5d:90
<<<lnx_bonding:sep(58)>>>
==> bond0 <==
Ethernet Channel Bonding Driver: v3.6.0 (September 26, 2009)

Bonding Mode: IEEE 802.3ad Dynamic link aggregation
Transmit Hash Policy: layer2 (0)
MII Status: up
MII Polling Interval (ms): 100
Up Delay (ms): 0
Down Delay (ms): 0

802.3ad info
LACP rate: slow
Aggregator selection policy (ad_select): stable
Active Aggregator Info:
	Aggregator ID: 1
	Number of ports: 2
	Actor Key: 17
	Partner Key: 203
	Partner Mac Address: 00:23:04:ee:be:0b

Slave Interface: eth0
MII Status: up
Speed: 1000 Mbps
Duplex: full
Link Failure Count: 0
Permanent HW addr: e8:39:35:be:5d:90
Aggregator ID: 1
Slave queue ID: 0

Slave Interface: eth1
MII Status: up
Speed: 1000 Mbps
Duplex: full
Link Failure Count: 0
Permanent HW addr: e8:39:35:be:5d:91
Aggregator ID: 1
Slave queue ID: 0
~
<<<tcp_conn_stats>>>
08 3
01 4
0A 34
05 1
06 194
<<<multipath>>>
mpathr (36006016067702d001acda7c3695ae211) dm-18 DGC,RAID 5
size=1.0T features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
|-+- policy='round-robin 0' prio=0 status=active
| |- 1:0:1:14 sdaq 66:160 active undef running
| `- 3:0:1:14 sdbq 68:64  active undef running
`-+- policy='round-robin 0' prio=0 status=enabled
  |- 2:0:0:14 sdq  65:0   active undef running
  `- 4:0:1:14 sdcq 69:224 active undef running
mpathe (36006016067702d00fa66b0e3675ae211) dm-27 DGC,RAID 5
size=1.0T features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
|-+- policy='round-robin 0' prio=0 status=active
| |- 2:0:0:1  sdd  8:48   active undef running
| `- 4:0:1:1  sdcd 69:16  active undef running
`-+- policy='round-robin 0' prio=0 status=enabled
  |- 1:0:1:1  sdad 65:208 active undef running
  `- 3:0:1:1  sdbd 67:112 active undef running
mpathq (36006016067702d00a46b8eaa695ae211) dm-13 DGC,RAID 5
size=1.0T features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
|-+- policy='round-robin 0' prio=0 status=active
| |- 2:0:0:13 sdp  8:240  active undef running
| `- 4:0:1:13 sdcp 69:208 active undef running
`-+- policy='round-robin 0' prio=0 status=enabled
  |- 1:0:1:13 sdap 66:144 active undef running
  `- 3:0:1:13 sdbp 68:48  active undef running
mpathd (36006016067702d00c0eb668d675ae211) dm-2 DGC,RAID 5
size=1.0T features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
|-+- policy='round-robin 0' prio=0 status=active
| |- 1:0:1:0  sdac 65:192 active undef running
| `- 3:0:1:0  sdbc 67:96  active undef running
`-+- policy='round-robin 0' prio=0 status=enabled
  |- 2:0:0:0  sdc  8:32   active undef running
  `- 4:0:1:0  sdcc 69:0   active undef running
mpathp (36006016067702d0088203c96695ae211) dm-26 DGC,RAID 5
size=1.0T features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
|-+- policy='round-robin 0' prio=0 status=active
| |- 1:0:1:12 sdao 66:128 active undef running
| `- 3:0:1:12 sdbo 68:32  active undef running
`-+- policy='round-robin 0' prio=0 status=enabled
  |- 2:0:0:12 sdo  8:224  active undef running
  `- 4:0:1:12 sdco 69:192 active undef running
mpathc (36006016006b032002ec7d90f537be211) dm-9 DGC,RAID 5
size=2.3T features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
|-+- policy='round-robin 0' prio=0 status=active
| |- 2:0:1:0  sdab 65:176 active undef running
| `- 4:0:0:0  sdcb 68:240 active undef running
`-+- policy='round-robin 0' prio=0 status=enabled
  |- 1:0:0:0  sdb  8:16   active undef running
  `- 3:0:0:0  sdbb 67:80  active undef running
mpatho (36006016067702d00b85c207d695ae211) dm-21 DGC,RAID 5
size=1.0T features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
|-+- policy='round-robin 0' prio=0 status=active
| |- 2:0:0:11 sdn  8:208  active undef running
| `- 4:0:1:11 sdcn 69:176 active undef running
`-+- policy='round-robin 0' prio=0 status=enabled
  |- 1:0:1:11 sdan 66:112 active undef running
  `- 3:0:1:11 sdbn 68:16  active undef running
mpathb (36006016067702d00203a0733275be211) dm-17 DGC,RAID 5
size=2.3T features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
|-+- policy='round-robin 0' prio=0 status=active
| |- 1:0:1:17 sdat 66:208 active undef running
| `- 3:0:1:17 sdbt 68:112 active undef running
`-+- policy='round-robin 0' prio=0 status=enabled
  |- 2:0:0:17 sdt  65:48  active undef running
  `- 4:0:1:17 sdct 70:16  active undef running
mpathn (36006016067702d0006a5fe67695ae211) dm-8 DGC,RAID 5
size=1.0T features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
|-+- policy='round-robin 0' prio=0 status=active
| |- 1:0:1:10 sdam 66:96  active undef running
| `- 3:0:1:10 sdbm 68:0   active undef running
`-+- policy='round-robin 0' prio=0 status=enabled
  |- 2:0:0:10 sdm  8:192  active undef running
  `- 4:0:1:10 sdcm 69:160 active undef running
mpathz (3600601600cd031009e4a05bc2022e411) dm-20 DGC,RAID 5
size=1.0T features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
|-+- policy='round-robin 0' prio=0 status=active
| |- 1:0:1:24 sdba 67:64  active undef running
| `- 3:0:1:24 sdca 68:224 active undef running
`-+- policy='round-robin 0' prio=0 status=enabled
  |- 2:0:0:24 sdaa 65:160 active undef running
  `- 4:0:1:24 sdda 70:128 active undef running
mpathm (36006016067702d00fc1e5141695ae211) dm-10 DGC,RAID 5
size=1.0T features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
|-+- policy='round-robin 0' prio=0 status=active
| |- 2:0:0:9  sdl  8:176  active undef running
| `- 4:0:1:9  sdcl 69:144 active undef running
`-+- policy='round-robin 0' prio=0 status=enabled
  |- 1:0:1:9  sdal 66:80  active undef running
  `- 3:0:1:9  sdbl 67:240 active undef running
mpathy (3600601600cd03100d88010822022e411) dm-25 DGC,RAID 5
size=1.0T features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
|-+- policy='round-robin 0' prio=0 status=active
| |- 2:0:0:23 sdz  65:144 active undef running
| `- 4:0:1:23 sdcz 70:112 active undef running
`-+- policy='round-robin 0' prio=0 status=enabled
  |- 1:0:1:23 sdaz 67:48  active undef running
  `- 3:0:1:23 sdbz 68:208 active undef running
mpathl (36006016067702d00c646981c695ae211) dm-11 DGC,RAID 5
size=1.0T features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
|-+- policy='round-robin 0' prio=0 status=active
| |- 1:0:1:8  sdak 66:64  active undef running
| `- 3:0:1:8  sdbk 67:224 active undef running
`-+- policy='round-robin 0' prio=0 status=enabled
  |- 2:0:0:8  sdk  8:160  active undef running
  `- 4:0:1:8  sdck 69:128 active undef running
mpathx (3600601600cd03100ae96394fd111e411) dm-19 DGC,RAID 5
size=1.0T features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
|-+- policy='round-robin 0' prio=0 status=active
| |- 1:0:1:22 sday 67:32  active undef running
| `- 3:0:1:22 sdby 68:192 active undef running
`-+- policy='round-robin 0' prio=0 status=enabled
  |- 2:0:0:22 sdy  65:128 active undef running
  `- 4:0:1:22 sdcy 70:96  active undef running
mpathk (36006016067702d00d416f905695ae211) dm-5 DGC,RAID 5
size=1.0T features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
|-+- policy='round-robin 0' prio=0 status=active
| |- 2:0:0:7  sdj  8:144  active undef running
| `- 4:0:1:7  sdcj 69:112 active undef running
`-+- policy='round-robin 0' prio=0 status=enabled
  |- 1:0:1:7  sdaj 66:48  active undef running
  `- 3:0:1:7  sdbj 67:208 active undef running
mpathw (3600601600cd0310046b0242476fbe311) dm-23 DGC,RAID 5
size=1.0T features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
|-+- policy='round-robin 0' prio=0 status=active
| |- 2:0:0:21 sdx  65:112 active undef running
| `- 4:0:1:21 sdcx 70:80  active undef running
`-+- policy='round-robin 0' prio=0 status=enabled
  |- 1:0:1:21 sdax 67:16  active undef running
  `- 3:0:1:21 sdbx 68:176 active undef running
mpathj (36006016067702d001a8376ef685ae211) dm-6 DGC,RAID 5
size=1.0T features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
|-+- policy='round-robin 0' prio=0 status=active
| |- 1:0:1:6  sdai 66:32  active undef running
| `- 3:0:1:6  sdbi 67:192 active undef running
`-+- policy='round-robin 0' prio=0 status=enabled
  |- 2:0:0:6  sdi  8:128  active undef running
  `- 4:0:1:6  sdci 69:96  active undef running
mpathv (3600601600cd0310058dc725dd9c3e311) dm-22 DGC,RAID 5
size=1.0T features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
|-+- policy='round-robin 0' prio=0 status=active
| |- 1:0:1:20 sdaw 67:0   active undef running
| `- 3:0:1:20 sdbw 68:160 active undef running
`-+- policy='round-robin 0' prio=0 status=enabled
  |- 2:0:0:20 sdw  65:96  active undef running
  `- 4:0:1:20 sdcw 70:64  active undef running
mpathi (36006016067702d00a0b3acd5685ae211) dm-12 DGC,RAID 5
size=1.0T features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
|-+- policy='round-robin 0' prio=0 status=active
| |- 2:0:0:5  sdh  8:112  active undef running
| `- 4:0:1:5  sdch 69:80  active undef running
`-+- policy='round-robin 0' prio=0 status=enabled
  |- 1:0:1:5  sdah 66:16  active undef running
  `- 3:0:1:5  sdbh 67:176 active undef running
mpathu (3600601600cd0310042f4ce68f26ee311) dm-24 DGC,RAID 5
size=1.0T features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
|-+- policy='round-robin 0' prio=0 status=active
| |- 2:0:0:19 sdv  65:80  active undef running
| `- 4:0:1:19 sdcv 70:48  active undef running
`-+- policy='round-robin 0' prio=0 status=enabled
  |- 1:0:1:19 sdav 66:240 active undef running
  `- 3:0:1:19 sdbv 68:144 active undef running
mpathaa (3600601600cd03100f87503f8fb63e411) dm-14 DGC,RAID 5
size=1.0T features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
|-+- policy='round-robin 0' prio=0 status=active
| |- 2:0:0:16 sdas 66:192 active undef running
| `- 4:0:1:16 sdcs 70:0   active undef running
`-+- policy='round-robin 0' prio=0 status=enabled
  |- 1:0:1:16 sds  65:32  active undef running
  `- 3:0:1:16 sdbs 68:96  active undef running
mpathh (36006016067702d0068238084685ae211) dm-4 DGC,RAID 5
size=1.0T features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
|-+- policy='round-robin 0' prio=0 status=active
| |- 1:0:1:4  sdag 66:0   active undef running
| `- 3:0:1:4  sdbg 67:160 active undef running
`-+- policy='round-robin 0' prio=0 status=enabled
  |- 2:0:0:4  sdg  8:96   active undef running
  `- 4:0:1:4  sdcg 69:64  active undef running
mpatht (3600601600cd03100d6ef6cdfb034e311) dm-16 DGC,RAID 5
size=1.0T features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
|-+- policy='round-robin 0' prio=0 status=active
| |- 2:0:0:18 sdu  65:64  active undef running
| `- 4:0:1:18 sdcu 70:32  active undef running
`-+- policy='round-robin 0' prio=0 status=enabled
  |- 1:0:1:18 sdau 66:224 active undef running
  `- 3:0:1:18 sdbu 68:128 active undef running
mpathg (36006016067702d00607ac85b685ae211) dm-7 DGC,RAID 5
size=1.0T features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
|-+- policy='round-robin 0' prio=0 status=active
| |- 2:0:0:3  sdf  8:80   active undef running
| `- 4:0:1:3  sdcf 69:48  active undef running
`-+- policy='round-robin 0' prio=0 status=enabled
  |- 1:0:1:3  sdaf 65:240 active undef running
  `- 3:0:1:3  sdbf 67:144 active undef running
mpaths (36006016067702d007222c7d8695ae211) dm-15 DGC,RAID 5
size=1.0T features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
|-+- policy='round-robin 0' prio=0 status=active
| |- 2:0:0:15 sdr  65:16  active undef running
| `- 4:0:1:15 sdcr 69:240 active undef running
`-+- policy='round-robin 0' prio=0 status=enabled
  |- 1:0:1:15 sdar 66:176 active undef running
  `- 3:0:1:15 sdbr 68:80  active undef running
mpathf (36006016067702d00fc530441685ae211) dm-3 DGC,RAID 5
size=1.0T features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
|-+- policy='round-robin 0' prio=0 status=active
| |- 1:0:1:2  sdae 65:224 active undef running
| `- 3:0:1:2  sdbe 67:128 active undef running
`-+- policy='round-robin 0' prio=0 status=enabled
  |- 2:0:0:2  sde  8:64   active undef running
  `- 4:0:1:2  sdce 69:32  active undef running
<<<md>>>
Personalities : [raid6] [raid5] [raid4] [raid1] [raid10] 
unused devices: <none>
<<<diskstat>>>
1416329024
   8      32 sdc 180321 12 1442664 1251715 0 0 0 0 0 1251539 1251539
   8      64 sde 180756 12 1446144 1041714 0 0 0 0 0 1041539 1041539
   8      96 sdg 180510 12 1444176 1107684 0 0 0 0 0 1107504 1107504
   8     144 sdj 169783423 12 43843540411 632255890 33607604 0 6267004625 272659047 0 419839234 904894095
   8     128 sdi 180646 12 1445264 1256029 0 0 0 0 0 1255842 1255842
   8      80 sdf 174080499 12 44161773254 876550093 33367613 0 6226195383 273863266 0 515985152 1150393431
   8     192 sdm 181055 12 1448536 476885 0 0 0 0 0 476709 476710
   8      16 sdb 181462 0 1451744 165494 0 0 0 0 0 165300 165300
   8     176 sdl 174430992 12 44043387762 474107972 33729801 0 6238082381 269983859 0 371355442 744068918
   8     160 sdk 180924 12 1447488 496736 0 0 0 0 0 496561 496561
   8     112 sdh 168369724 12 43780869363 739540124 34057534 0 6274171212 271958724 0 485493467 1011496108
   8     240 sdp 179081916 12 44237537945 479923725 34337443 0 6262719427 259427653 0 375627878 739328253
  65      16 sdr 172927164 12 43985653571 478338619 33530107 0 6260343800 265087257 0 373386477 743402893
   8       0 sda 763833 257293 20424501 1287709 93635159 72414348 1329396911 54406558 0 19533773 55678513
  65      64 sdu 172005076 12 43879459000 910813851 33484118 0 6271441545 265358933 0 529413152 1176144435
  65      48 sdt 182792 0 1462384 85775 0 0 0 0 0 85585 85585
  65       0 sdq 180988 12 1448000 489333 0 0 0 0 0 489149 489149
  65     128 sdy 181078 12 1448720 733531 0 0 0 0 0 733360 733360
  65     160 sdaa 182901 12 1463304 186498 0 0 0 0 0 186301 186301
   8     208 sdn 165811096 12 43489751870 463351380 33253088 0 6240545446 264381349 0 364249884 727730789
  65      96 sdw 180766 12 1446224 809284 0 0 0 0 0 809122 809122
  65     112 sdx 176288987 12 44186946417 835657359 33809994 0 6305896725 274596304 0 490712014 1110230675
  65      80 sdv 170318962 12 43977833942 944226323 33124746 0 6267302867 271187425 1 540600702 1215395619
  65     144 sdz 29327803 12 19834748713 94500294 70967487 0 37480100425 204162893 0 151405638 298652879
   8     224 sdo 181023 12 1448280 490418 0 0 0 0 0 490256 490256
  65     224 sdae 186309648 12 44878658934 885812511 33130641 0 6216997054 209754505 1 516642113 1095557121
  65     192 sdac 177995876 12 44049002280 883203293 33725709 0 6272541739 207590675 0 521909280 1090774650
  65     240 sdaf 180295 12 1442456 1045278 0 0 0 0 0 1045113 1045113
  65     208 sdad 179668 12 1437440 1338595 0 0 0 0 0 1338405 1338405
  66      16 sdah 180383 12 1443160 1102987 0 0 0 0 0 1102829 1102829
  66       0 sdag 170872794 12 43784837168 894290613 33545944 0 6246324421 210908020 0 521557389 1105174735
  66      32 sdai 181504782 12 44363628577 702399830 34081997 0 6305912018 204797792 0 474274130 907174012
  65     176 sdab 247172 0 37664418 350507 79274383 0 4091088125 183615953 0 18743294 183962880
  66      48 sdaj 180787 12 1446392 661241 0 0 0 0 0 661074 661075
  66      64 sdak 174245864 12 44036523847 447450546 33295568 0 6232289036 199808108 0 358725196 647235404
  66      80 sdal 180914 12 1447408 442235 0 0 0 0 0 442088 442088
  66      96 sdam 175041996 12 44170381939 446980535 33904064 0 6300198883 202889881 0 359058644 649849681
  66     112 sdan 180801 12 1446504 446759 0 0 0 0 0 446585 446585
  66     128 sdao 175192675 12 44022223946 452434848 33429281 0 6238432664 196552202 0 360931362 648967634
  66     160 sdaq 170071547 12 44008413266 443712383 33245032 0 6243941408 196678855 0 357874991 640365722
  66     144 sdap 180925 12 1447496 448385 0 0 0 0 0 448202 448202
  66     176 sdar 180962 12 1447792 447692 0 0 0 0 0 447521 447521
  66     208 sdat 11178914 0 1640585660 6508172 82750574 0 4075968015 183318258 0 20202802 189821198
  66     224 sdau 180608 12 1444960 828058 0 0 0 0 0 827863 827863
  66     240 sdav 180630 12 1445136 759952 0 0 0 0 0 759789 759789
  67      16 sdax 180698 12 1445680 754536 0 0 0 0 0 754381 754381
  67      32 sday 194834036 12 44230490997 1012668689 46147716 0 6579113969 219466311 0 503675456 1232105499
  67      48 sdaz 182986 12 1463984 171884 0 0 0 0 0 171680 171680
  67      64 sdba 29430660 12 19837422627 75988180 71865148 0 37489089073 205210368 0 146192791 281186694
  67      96 sdbc 177985736 12 44048994151 870445301 33735830 0 6277865722 202870095 0 516959883 1073296447
  67     160 sdbg 170861314 12 43783159047 882462595 33557397 0 6250557844 205498363 0 517056458 1087934877
  67     128 sdbe 186299662 12 44879578681 873038598 33140610 0 6222094799 204080767 0 512156350 1077111943
  67     112 sdbd 179674 12 1437488 1199756 0 0 0 0 0 1199607 1199607
  67     176 sdbh 180443 12 1443640 1165030 0 0 0 0 0 1164887 1164887
  67     144 sdbf 180334 12 1442768 1054992 0 0 0 0 0 1054844 1054844
  67       0 sdaw 172237941 12 44028533511 856018770 34063767 0 6277722377 209140501 0 502708439 1065138181
  67     192 sdbi 181487264 12 44359849697 692539799 34099488 0 6310822859 200192713 0 470207303 892710232
  67     208 sdbj 180820 12 1446656 658080 0 0 0 0 0 657912 657912
  68       0 sdbm 175030539 12 44165002768 440432812 33915514 0 6305875984 198048850 0 355051155 638462305
  68      16 sdbn 180842 12 1446832 445901 0 0 0 0 0 445737 445737
  67     240 sdbl 180959 12 1447768 440393 0 0 0 0 0 440250 440250
  68      48 sdbp 180971 12 1447864 427733 0 0 0 0 0 427571 427572
  68      32 sdbo 175188646 12 44022836016 444765923 33433320 0 6241705141 191938838 0 356473225 636685224
  68      80 sdbr 180987 12 1447992 435237 0 0 0 0 0 435084 435084
  67     224 sdbk 174242489 12 44035838106 440562259 33298959 0 6237545717 195136228 0 354761607 635676055
  67      80 sdbb 181695 0 1453608 151168 0 0 0 0 0 151011 151011
   8      48 sdd 170164487 12 43977422611 960984288 33095410 0 6270416349 274433282 0 547728889 1235392096
  68     128 sdbu 180635 12 1445176 816342 0 0 0 0 0 816157 816157
  68     112 sdbt 11180535 0 1641029565 6213475 82748952 0 4085426928 170733161 0 19624286 176941541
  68     160 sdbw 172226000 12 44026020943 843800787 34075671 0 6282424561 204058656 0 498054068 1047841683
  68     176 sdbx 180712 12 1445792 739235 0 0 0 0 0 739064 739064
  68     144 sdbv 180652 12 1445312 762070 0 0 0 0 0 761917 761917
  68     224 sdca 29434530 12 19847199942 74974270 71861288 0 37458821939 198452110 0 143345319 273418621
  68     208 sdbz 183017 12 1464232 167450 0 0 0 0 0 167281 167281
  68     192 sdby 194831142 12 44231144674 998352237 46150568 0 6580853462 214406449 0 498958973 1212736143
  68     240 sdcb 246959 0 38282366 346591 79274503 0 4070306818 197825372 0 18800649 198167887
  68      64 sdbq 170052822 12 44003342545 437435356 33263765 0 6252193096 192273318 0 353842472 629689046
  69       0 sdcc 180393 12 1443240 1191260 0 0 0 0 0 1191085 1191085
  69      16 sdcd 170155375 12 43977233214 954615083 33104521 0 6272932550 269379914 0 544438833 1223974580
  69      48 sdcf 174074108 12 44162468086 871606773 33374045 0 6228157343 268761461 0 512858811 1140349762
  69      64 sdcg 180582 12 1444752 1039199 0 0 0 0 0 1039018 1039018
  69      80 sdch 168364891 12 43781635102 733940875 34062438 0 6276678783 267117653 0 482424460 1001040414
  69      96 sdci 180742 12 1446032 1461326 0 0 0 0 0 1461153 1461153
  69     112 sdcj 169773690 12 43844470437 627177003 33617434 0 6272108509 267741754 0 416887613 894898535
  69     128 sdck 181022 12 1448272 397470 0 0 0 0 0 397318 397318
  69     144 sdcl 174425550 12 44043877357 470651620 33735361 0 6244608112 265091242 0 368338790 735731711
  69     160 sdcm 181179 12 1449528 399820 0 0 0 0 0 399670 399670
  69     176 sdcn 165805336 12 43489139065 459852314 33258939 0 6243539101 259542207 0 361118394 719391696
  69     208 sdcp 179075893 12 44235613446 476218679 34343594 0 6266874925 255091995 0 372410513 731286674
  69     224 sdcq 181077 12 1448712 388206 0 0 0 0 0 388040 388040
  69     240 sdcr 172925679 12 43986766379 474740104 33531707 0 6264954943 260378041 0 370276347 735093414
  70      16 sdct 182949 0 1463616 75280 0 0 0 0 0 75092 75092
  70      32 sdcu 172000515 12 43881551438 905360852 33488708 0 6274106221 260708386 0 525817139 1166049486
  70      48 sdcv 170317533 12 43980327184 937140947 33126218 0 6270338043 266645186 1 536720565 1203765605
  69     192 sdco 181121 12 1449064 420705 0 0 0 0 0 420569 420569
  70      64 sdcw 180810 12 1446576 737315 0 0 0 0 0 737144 737144
  70      80 sdcx 176285484 12 44187992047 830577294 33813549 0 6306223350 269392964 0 487508723 1099948790
  70      96 sdcy 181139 12 1449208 661518 0 0 0 0 0 661371 661371
  70     112 sdcz 29333492 12 19847258821 93679318 70961810 0 37452505355 199080400 0 148342124 292747890
  70     128 sdda 183031 12 1464344 172563 0 0 0 0 0 172380 172380
  69      32 sdce 180899 12 1447288 1018988 0 0 0 0 0 1018827 1018827
 253       0 dm-0 6993 0 184474 9111 1983314 0 15866512 917081 0 398390 926192
 253       1 dm-1 463 0 3704 663 0 0 0 0 0 504 663
 253       2 dm-2 355620743 181965 88095109287 1759320629 67461539 3586607 12550407461 530974294 0 685031846 2290275532
 253       3 dm-3 372247534 436932 89755343215 1766692941 66271251 3544279 12439091853 527944431 1 675870267 2294673597
 253       4 dm-4 341372887 289582 87565106255 1781914829 67103341 3541184 12496882265 546852230 0 682399766 2328802843
 253       5 dm-5 339195464 232151 87685117464 1266056490 67225038 3783221 12539113134 727059471 0 557041987 1993461010
 253       6 dm-6 362630521 188683 88720585882 1399068217 68181485 3622520 12616734877 516187962 0 629261336 1915204532
 253       7 dm-7 347793899 175304 88321355484 1755141077 66741658 3757966 12454352726 722314944 0 673325478 2477635984
 253       8 dm-8 349710138 189053 88332485339 893322661 67819578 3597465 12606074867 511580436 0 486480633 1404828342
 253       9 dm-9 129834 78285 73032240 423928 158548886 741875294 8161394943 408946368 0 23830227 409308176
 253      10 dm-10 348494628 170780 88084369615 951086974 67465162 3725637 12482690493 722886661 0 500387744 1674151941
 253      11 dm-11 348126225 197713 88069464737 894205421 66594527 3531631 12469834753 494562916 0 485887657 1388688948
 253      12 dm-12 336373719 258381 87559617105 1479076747 68119972 3785392 12550849995 711168313 0 640129098 2190434308
 253      13 dm-13 357795868 318210 88470255671 962221766 68681037 3708504 12529594352 658825438 0 507403974 1621115021
 253      15 dm-15 345490863 351487 87969523918 958962221 67061814 3784906 12525298743 700214690 0 503183769 1659323009
 253      16 dm-16 343644279 385533 87758119750 1825583470 66972826 3789906 12545547766 686023443 0 692735006 2511833548
 253      17 dm-17 21993029 4900505 3278683697 12928618 165499526 734924775 8161394943 384910692 0 27793966 397757231
 253      18 dm-18 339762144 337181 88008857819 886732799 66508797 3612637 12496134504 489654118 0 484152024 1376430941
 253      19 dm-19 389302830 222475 88458736695 2071186275 92298284 3552796 13159967431 561316323 0 668125162 2632451255
 253      20 dm-20 58498764 29179 39681690969 152895143 143726436 72262 74947911012 413516717 0 216996879 566145937
 253      21 dm-21 331254727 170920 86975997103 928940402 66512027 3743173 12484084547 692613143 0 491301422 1621766524
 253      22 dm-22 344102290 184997 88051661054 1706014307 68139438 3614308 12560146938 539611589 0 660738746 2245602928
 253      23 dm-23 352213003 214942 88372046528 1673950329 67623543 3853219 12612120075 737318454 0 645355899 2411542268
 253      24 dm-24 340275155 191741 87955270438 1890414127 66250964 3757863 12537640910 715552595 0 705992255 2606116574
 253      25 dm-25 58295032 29854 39679077238 189811590 141929297 73756 74932605780 416742838 0 220730140 606290020
 253      26 dm-26 350019080 169919 88042161842 902756087 66862601 3589118 12480137805 484464019 0 488945127 1387255496
 253      27 dm-27 339960452 402252 87951780353 1924302229 66199931 3786607 12543348899 730221212 0 713953512 2654743738
 253      58 dm-58 720 0 8426 1999 25091973 0 200735784 17886301 0 6577787 17915588
 253      59 dm-59 4585 0 189858 5496 6154916 0 49239328 69346841 0 1272814 69368241
 253      60 dm-60 29663 0 789962 38937 145652 0 1165216 1182859 0 31882 1221797
 253      61 dm-61 2483 0 41106 3499 1956 0 15648 1147 0 1740 4646
 253      62 dm-62 21884 0 615442 256846 93364571 0 746916568 44572422 0 3956584 44963459
 253      63 dm-63 2209 0 105266 3958 1433474 0 11467792 1691185 0 1659598 1695145
 253      64 dm-64 718300 0 10951069 1036294 13592348 0 108736055 10488518 0 1655581 11530820
 253      65 dm-65 180487 0 4235138 253796 9522619 0 76180952 5364362 0 1668153 5620801
 253      66 dm-66 18539 0 148306 22529 12763 0 102104 17661 0 2445 40189
 253      67 dm-67 20260 0 2865066 120881 4234727 0 33877816 202788643 0 251795 202954806
 253      68 dm-68 14328 0 275370 16886 10636477 0 85091816 12990462 0 2123205 13012289
 253      33 dm-33 2 0 2 1 2967567 0 19943214 3147629 0 2894996 3148970
 253      34 dm-34 260316 0 129544458 915203 383884750 0 3084082800 1982485129 0 5735608 1986345045
 253      35 dm-35 185 0 1440 113 2967567 0 19943214 4422825 0 3979685 4424014
 253      36 dm-36 17520 0 4984056 235445 383884750 0 3084082800 1610406883 0 5364824 1614013947
 253      37 dm-37 277836 0 134528514 1151916 383884750 0 3084082800 1397670470 0 8330660 1408574433
 253      38 dm-38 1 0 1 0 3157984 0 20801070 2919595 0 2842159 2919793
 253      39 dm-39 104873 0 50389434 271337 453554156 0 3628433248 1014232621 0 9291791 1016214459
 253      40 dm-40 18 0 128 20 3157984 0 20801070 4085395 0 3975849 4085581
 253      41 dm-41 28217 0 14144840 106309 453554156 0 3628433248 1488862180 0 11578686 1490928857
 253      42 dm-42 133090 0 64534274 378094 453554156 0 3628433248 3425594752 0 14281221 3432140713
 253      43 dm-43 1 0 1 0 2606 0 2606 1010 0 996 1010
 253      44 dm-44 364 0 10226 671 1167 0 9336 595 0 897 1266
 253      45 dm-45 3 0 3 2 2606 0 2606 3198 0 3161 3200
 253      46 dm-46 9 0 72 53 1167 0 9336 1191 0 658 1244
 253      47 dm-47 373 0 10298 724 1167 0 9336 4128 0 3343 4857
 253      48 dm-48 1 0 1 0 739701 0 5357469 406983 0 400044 406995
 253      49 dm-49 26485153 0 3093436614 19432378 20956882 0 1121648960 29607242 0 9955477 49070920
 253      50 dm-50 6 0 33 2 739701 0 5357469 827032 0 810277 827051
 253      51 dm-51 152139 0 52525212 305231 20956882 0 1121648960 45509459 0 2504061 45842266
 253      52 dm-52 24437304 0 3145961826 16458127 20193935 0 1121648960 70742718 0 11637871 87245526
 253      53 dm-53 1 0 1 0 157723 0 1096362 210158 0 204798 210189
 253      54 dm-54 41912 0 5292578 107227 35002449 0 280019592 106698529 0 467307 107031318
 253      55 dm-55 6 0 28 9 157723 0 1096362 298267 0 287825 298318
 253      56 dm-56 9385 0 1369448 55267 35002449 0 280019592 131355158 0 465576 131636552
 253      57 dm-57 51297 0 6662026 162742 35002449 0 280019592 404252302 0 638924 405270113
  65      32 sds 60166 0 481376 360919 0 0 0 0 0 360739 360841
 253      14 dm-14 129396173 83059 27104014468 680283380 34011478 1140542 5943777444 750978405 0 210345105 1431327505
  66     192 sdas 64759279 0 13552125660 325631056 17004589 0 2972152776 245107384 0 159785149 570730478
  68      96 sdbs 59991 0 479928 319839 0 0 0 0 0 319792 319793
  70       0 sdcs 64756720 0 13552847368 323976097 17006889 0 2971624668 240668581 0 158802441 564637339
[dmsetup_info]
mpathr 253:18  
dwhvg-lv_dingdata_rimage_0 253:39 dwhvg lv_dingdata_rimage_0
mpathe 253:27  
dwhvg-lv_backup_rimage_1 253:51 dwhvg lv_backup_rimage_1
rootvg-lv_swap 253:1 rootvg lv_swap
rootvg-lv_usr 253:60 rootvg lv_usr
rootvg-lv_root 253:0 rootvg lv_root
rootvg-lv_var 253:62 rootvg lv_var
mpathq 253:13  
mpathd 253:2  
dwhvg-lv_backup_rimage_0 253:49 dwhvg lv_backup_rimage_0
dwhvg-lv_conndir 253:47 dwhvg lv_conndir
mpathp 253:26  
mpathc 253:9  
mpatho 253:21  
mpathb 253:17  
rootvg-lv_install 253:66 rootvg lv_install
rootvg-lv_local 253:61 rootvg lv_local
mpathn 253:8  
dwhvg-lv_backup 253:52 dwhvg lv_backup
mpathz 253:20  
mpathm 253:10  
dwhvg-lv_opt_uniserv 253:57 dwhvg lv_opt_uniserv
rootvg-lv_opt 253:63 rootvg lv_opt
mpathy 253:25  
mpathl 253:11  
rootvg-lv_oracle11 253:64 rootvg lv_oracle11
dwhvg-lv_backup_rmeta_1 253:50 dwhvg lv_backup_rmeta_1
mpathx 253:19  
mpathk 253:5  
dwhvg-lv_opt_uniserv_rmeta_1 253:55 dwhvg lv_opt_uniserv_rmeta_1
dwhvg-lv_dingdata 253:42 dwhvg lv_dingdata
dwhvg-lv_backup_rmeta_0 253:48 dwhvg lv_backup_rmeta_0
mpathw 253:23  
mpathj 253:6  
dwhvg-lv_opt_uniserv_rmeta_0 253:53 dwhvg lv_opt_uniserv_rmeta_0
mpathv 253:22  
dwhvg-lv_conndir_rimage_1 253:46 dwhvg lv_conndir_rimage_1
mpathi 253:12  
dwhvg-lv_opt_uniserv_rimage_1 253:56 dwhvg lv_opt_uniserv_rimage_1
dwhvg-lv_dingdata_rmeta_1 253:40 dwhvg lv_dingdata_rmeta_1
mpathu 253:24  
mpathaa 253:14  
dwhvg-lv_conndir_rimage_0 253:44 dwhvg lv_conndir_rimage_0
rootvg-lv_home 253:59 rootvg lv_home
mpathh 253:4  
dwhvg-lv_opt_uniserv_rimage_0 253:54 dwhvg lv_opt_uniserv_rimage_0
dwhvg-lv_conndir_rmeta_1 253:45 dwhvg lv_conndir_rmeta_1
rootvg-lv_tmp 253:58 rootvg lv_tmp
dwhvg-lv_dingdata_rmeta_0 253:38 dwhvg lv_dingdata_rmeta_0
rootvg-lv_agent 253:68 rootvg lv_agent
mpatht 253:16  
rootvg-lv_oragrid 253:65 rootvg lv_oragrid
mpathg 253:7  
dwhvg-lv_conndir_rmeta_0 253:43 dwhvg lv_conndir_rmeta_0
mpaths 253:15  
dwhvg-lv_dingdata_rimage_1 253:41 dwhvg lv_dingdata_rimage_1
mpathf 253:3  
<<<md>>>
Personalities : 
unused devices: <none>
<<<ntp>>>
* 172.33.44.49    29.244.104.45    2 u    2   64  377    0.302   -0.353   0.483
+ 172.44.33.19     29.244.104.33    2 u   18   64  377    0.408    0.051   0.319
<<<postfix_mailq>>>
QUEUE_deferred 4 0
QUEUE_active 4 0
<<<postfix_mailq>>>
/var/spool/mqueue is empty
		Total requests: 0
<<<lnx_thermal>>>
thermal_zone0 - x86_pkg_temp 59000 0 passive 0 passive 
thermal_zone1 - BAT0 35000 
<<<logwatch>>>
[[[/var/log/messages:missing]]]
[[[/var/log/kern.log]]]
[[[/var/log/auth.log]]]
[[[/var/log/syslog]]]
"""

#.
#   .--Test Cases----------------------------------------------------------.
#   |            _____         _      ____                                 |
#   |           |_   _|__  ___| |_   / ___|__ _ ___  ___  ___              |
#   |             | |/ _ \/ __| __| | |   / _` / __|/ _ \/ __|             |
#   |             | |  __/\__ \ |_  | |__| (_| \__ \  __/\__ \             |
#   |             |_|\___||___/\__|  \____\__,_|___/\___||___/             |
#   |                                                                      |
#   +----------------------------------------------------------------------+
#   | The test cases implement different kinds of tests                    |
#   '----------------------------------------------------------------------'

class TestCase(object):
    STATE_INITIALIZING = "initializing"
    STATE_RUNNING      = "running"
    STATE_STOPPED      = "stopped"

    RESULT_COMPLETED   = "completed"
    RESULT_SKIPPED     = "skipped"
    RESULT_ABORTED     = "aborted"
    RESULT_FAILED      = "failed"

    def __init__(self):
        self.started_at  = None
        self.finished_at = None
        self.state       = None
        self.result      = None
        self.result_msg  = None


    # Return a meaningful name to identify this test case
    def name(self):
        return "%s()" % self.__class__.__name__


    # The run() method can be implemented as generator and yield several intermediate steps
    # to the Test() object. In case a Wait(duration) object is passed, the Test() object will
    # wait for the given duration and then call the run() method again.
    def run(self):
        raise NotImplementedError()


    def duration(self):
        if self.started_at == None:
            return 0
        elif self.finished_at == None:
            return time.time() - self.started_at
        else:
            return self.finished_at - self.started_at


    def serialize(self):
        d = dict([ (k, v) for k, v in self.__dict__.items() if k[0] != "_" ])
        d["__class_name__"] = self.__class__.__name__
        return d


    def deserialize(self, raw):
        self.__dict__.update(raw)


    def set_state(self, state, result=None, result_msg=None):
        self.state      = state
        self.result     = result
        self.result_msg = result_msg

        if self.state == TestCase.STATE_RUNNING:
            self.started_at = time.time()
        elif self.state == TestCase.STATE_STOPPED:
            self.finished_at = time.time()


    def state_text(self):
        return self.state



class TestAgentBasedStandardHosts(TestCase):
    """A test that creates the specified number of "standard" hosts

    It adds the hosts to Check_MK and starts the monitoring of these
    hosts. The hosts will be configured with a standard set of services
    which is assumed to be a good configuration for the average Check_MK
    agent based host.

    The test will simulate an average network delay and process dynamic
    agent data that will lead to increasing counters in the checks.
    """

    host_name_template = "std-agent-host-%05d"

    def __init__(self, runner=None, num_hosts=None, planned_duration=None,
                       num_cmk_helpers=20):
        super(TestAgentBasedStandardHosts, self).__init__()
        self._runner = runner
        self._test   = runner.test if runner else None

        self.num_hosts        = num_hosts
        self.planned_duration = planned_duration
        self.num_cmk_helpers  = num_cmk_helpers

        self._listeners          = []
        self._hosts_per_listener = 100


    def name(self):
        return "%s(hosts=%d, duration=%d, helpers=%d)" % \
                   (self.__class__.__name__,
                    self.num_hosts,
                    self.planned_duration,
                    self.num_cmk_helpers)


    def run(self):
        try:
            self._spawn_agent_listeners()
            self._write_global_settings()
            self._create_hosts()
            self._discover_services()
            self._activate_changes()
            self._create_rrds()
            self._wait_for_cmk_helpers_ready()
            yield Start()
            yield Wait(self.planned_duration)
        finally:
            self._cleanup_hosts()
            self._activate_changes()
            self._runner._cleanup_host_files()
            self._end_agent_listeners()


    def _spawn_agent_listeners(self):
        self._test.info("  Starting agent listeners")
        for first_host_id in range(0, self.num_hosts, self._hosts_per_listener):
            host_ids = range(first_host_id, first_host_id + self._hosts_per_listener)
            listener = AgentListener(host_ids)
            listener.start()
            self._listeners.append(listener)


    def _end_agent_listeners(self):
        self._test.info("  Stopping agent listeners")
        for listener in self._listeners:
            listener.terminate()

        for listener in self._listeners:
            listener.join()
        self._test.info("  Stopped listeners")


    def _write_global_settings(self):
        self._test.info("  Configuring site")
        config_dir = self._cmk_config_dir()
        try:
            os.makedirs(config_dir)
        except OSError, e:
            if e.errno == 17: # file exists
                pass
            else:
                raise

        with open("%s/global_settings.mk" % config_dir, "w") as f:
            f.write(
                "# Created by mkbench\n"
                "cmc_cmk_helpers = %d\n"
                "cmc_log_limit = 209715200\n"
                "cmc_initial_scheduling = {\n"
                "    'burst': 100,\n"
                "    'spread_cmk': 10,\n"
                "    'spread_generic': 150\n"
                "}\n"
                "\n" % self.num_cmk_helpers
            )

        with open("%s/rules.mk" % config_dir, "w") as f:
            f.write(
                "agent_ports = [\n"
                "    ( 6559, [], ALL_HOSTS, {} ),\n"
                "] + agent_ports\n"
            )

            # Ensure RRDs are created with Check_MK default config
            f.write(
                "cmc_host_rrd_config = [\n"
                "      ({'rras': [\n"
                "          (50.0, 1, 2880), (50.0, 5, 2880),\n"
                "          (50.0, 30, 4320), (50.0, 360, 5840)\n"
                "         ],\n"
                "         'step': 60,\n"
                "         'cfs': ['MIN', 'MAX', 'AVERAGE'],\n"
                "         'format': 'cmc_single'\n"
                "      },\n"
                "      [], ALL_HOSTS, {\n"
                "       'description': u'Default RRD configuration, using new single RRD format'\n"
                "      }),\n"
                "] + cmc_host_rrd_config\n"
                "\n"
                "\n"
                "cmc_service_rrd_config = [\n"
                "   ( {'rras': [\n"
                "       (50.0, 1, 2880), (50.0, 5, 2880),\n"
                "       (50.0, 30, 4320), (50.0, 360, 5840)\n"
                "      ],\n"
                "      'step': 60,\n"
                "      'cfs': ['MIN', 'MAX', 'AVERAGE'],\n"
                "      'format': 'cmc_single'\n"
                "   },\n"
                "   [], ALL_HOSTS, ALL_SERVICES ),\n"
                "] + cmc_service_rrd_config\n"
            )


    def _create_hosts(self):
        self._test.info("  Creating hosts")
        config_dir = self._cmk_config_dir()
        try:
            os.makedirs(config_dir)
        except OSError, e:
            if e.errno == 17: # file exists
                pass
            else:
                raise

        all_hosts, ipaddresses, host_attributes = self._create_host_data()

        with open("%s/hosts.mk" % config_dir, "w") as f:
            f.write(
                "# Created by mkbench\n"
                "\n"
                "all_hosts += %s\n"
                "\n"
                "ipaddresses.update(%s)\n"
                "\n"
                "host_attributes.update(%s)\n" %
                    (pprint.pformat(all_hosts),
                     pprint.pformat(ipaddresses),
                     pprint.pformat(host_attributes))
            )

        with open("%s/rules.mk" % config_dir, "w") as f:
            f.write(
                "agent_ports = [\n"
                "    ( 6559, [], ALL_HOSTS, {} ),\n"
                "] + agent_ports\n"
            )


    def _create_host_data(self):
        all_hosts, ipaddresses, host_attributes = [], {}, {}

        for num in range(self.num_hosts):
            host_name = self.host_name_template % num
            ipaddress = get_host_ip(num)

            all_hosts.append("%s|lan|ip-v4|cmk-agent|tcp|site:%s|ip-v4-only|prod|bench" %
                                (host_name, site_id()))

            ipaddresses[host_name] = ipaddress

            host_attributes[host_name] = {
                "ipaddress": ipaddress,
            }

        return all_hosts, ipaddresses, host_attributes


    def _discover_services(self):
        self._test.info("  Discovering services")

        # Discver first host and copy the rest (all hosts are equal)
        first_host_name = self.host_name_template % 0
        self._discover_services_of_host(first_host_name)

        autochecks_dir = "%s/var/check_mk/autochecks" % omd_root()

        src_file = "%s/%s.mk" % (autochecks_dir, first_host_name)
        for num in range(1, self.num_hosts):
            host_name = self.host_name_template % num
            shutil.copy(src_file, "%s/%s.mk" % (autochecks_dir, host_name))


    def _discover_services_of_host(self, host_name):
        p = subprocess.Popen(["cmk", "-II", host_name], stdin=open(os.devnull),
                close_fds=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)

        try:
            stdout = p.communicate()[0]
        except KeyboardInterrupt:
            try:
                p.terminate()
            except OSError:
                pass
            raise

        if p.returncode != 0:
            raise MKGeneralException("'cmk -II %s' failed: %s" % (host_name, stdout))


    #def _discover_services(self):
    #    self._test.info("  Discovering services")
    #    p = subprocess.Popen(["cmk", "-II"], stdin=open(os.devnull),
    #            close_fds=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)

    #    try:
    #        stdout = p.communicate()[0]
    #    except KeyboardInterrupt:
    #        try:
    #            p.terminate()
    #        except OSError:
    #            pass
    #        raise

    #    if p.returncode != 0:
    #        raise MKGeneralException("'cmk -II' failed: %s" % stdout)


    def _create_rrds(self):
        try:
            # The execution of service checks needs to be stopped to prevent
            # races with the core.
            LocalConnection().command("[%d] STOP_EXECUTING_SVC_CHECKS" % time.time())

            self._test.info("  Creating RRDs (stopping core during this step)")
            first_host_name = self.host_name_template % 0
            self._create_rrds_of_host(first_host_name)
            self._copy_rrds_of(first_host_name)
            self._test.info("  Finished creating RRDs")
        finally:
            LocalConnection().command("[%d] START_EXECUTING_SVC_CHECKS" % time.time())


    def _copy_rrds_of(self, first_host_name):
        rrd_dir = "%s/var/check_mk/rrd" % omd_root()

        try:
            # We have to stop and start the CMC during manual RRD copy.
            # a) No races with the core
            # b) Cached RRD info held by the core during runtime
            subprocess.call(["omd", "stop", "cmc"], stdout=open(os.devnull, "w"),
                            close_fds=True)

            src_dir = "%s/%s" % (rrd_dir, first_host_name)
            for num in range(1, self.num_hosts):
                if self._runner.shall_stop():
                    break

                if num % 100 == 0:
                    self._test.info("  Created RRDs for %d hosts" % num)

                host_name = self.host_name_template % num
                target_dir = "%s/%s" % (rrd_dir, host_name)

                try:
                    shutil.copytree(src_dir, target_dir)
                except OSError, e:
                    if e.errno == 17: # file exists
                        pass
                    else:
                        raise

                # Replace the host name in the just copied info files
                for f in glob.glob("%s/%s/*.info" % (rrd_dir, host_name)):
                    new_content = open(f).readlines()
                    for index, line in enumerate(new_content):
                        if line.startswith("HOST "):
                            new_content[index] = "HOST %s\n" % host_name

                    open(f, "w").write("".join(new_content))
        finally:
            self._activate_changes()


    # To create the RRDs, simply perform one check and wait for the RRDs to be created
    def _create_rrds_of_host(self, host_name):
        # Do 2 iterations to initialize the rrds that base on counter values
        for i in [ 0, 1]:
            p = subprocess.Popen(["cmk", "-v", host_name], stdin=open(os.devnull),
                    close_fds=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)

            try:
                stdout = p.communicate()[0]
            except KeyboardInterrupt:
                try:
                    p.terminate()
                except OSError:
                    pass
                raise

            if p.returncode != 0:
                raise MKGeneralException("'cmk -v %s' failed: %s" % (host_name, stdout))

            time.sleep(2)

        num_rrds, wait_sec = None, 10
        while num_rrds is None or num_rrds < 30:
            if wait_sec == 0:
                raise MKGeneralException("Did not create RRDs for '%s' "
                                         "after 10 seconds" % host_name)
            wait_sec -= 1

            if self._runner.shall_stop():
                break
            num_rrds = len(glob.glob("%s/var/check_mk/rrd/%s/*.rrd" % (omd_root(), host_name)))
            time.sleep(1)
        self._test.info("  Found %d RRDs per host" % num_rrds)



    def _cmk_config_dir(self):
        return "%s/etc/check_mk/conf.d/wato/mkbench" % omd_root()


    def _activate_changes(self):
        if self._test:
            self._test.info("  Activating changes")
        p = subprocess.Popen(["cmk", "-R"],
                close_fds=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)

        try:
            stdout = p.communicate()[0]
        except KeyboardInterrupt:
            try:
                p.terminate()
            except OSError:
                pass
            raise

        if p.returncode != 0:
            raise MKGeneralException("Failed to execute 'cmk -R': %s" % stdout)

        # Wait for livestatus socket to be available
        while (self._runner and not self._runner.shall_stop()) \
              and not os.path.exists("%s/tmp/run/live" % omd_root()):
            time.sleep(1)


    # To deal with the load peak while initializing the Check_MK helpers we
    # need to wait till all helpers are initialized. Since we have no one
    # to tell us the state of the helpers, we need to detect this somehow.
    # -> Find all helper processes and check the open file descriptors.
    def _wait_for_cmk_helpers_ready(self):
        num_not_ready, loadavg_5_before = None, os.getloadavg()[1]
        while num_not_ready == None or num_not_ready > 0:
            if self._runner.shall_stop():
                break

            if num_not_ready == None:
                self._test.info("  Waiting for helper initialization")
            else:
                self._test.info("  Waiting for helper initialization (%d not ready)" %
                                                                        num_not_ready)

            try:
                matched_pids = subprocess.check_output([
                               "pgrep", "-u", site_id(), "-f", "python.*check_mk.py --keepalive$"]).split("\n")
            except subprocess.CalledProcessError, e:
                self._test.error("  Found no helper yet (%s)" % e.returncode)
                continue

            num_not_ready = 0
            for p in matched_pids:
                if p and not os.path.exists("/proc/%s/fd/3" % p):
                    num_not_ready += 1
            time.sleep(1)

        self._test.info("  Finished helper initialization")


    def _cleanup_hosts(self):
        if self._test:
            self._test.info("  Cleaning up test configs")
        for path in glob.glob("%s/*" % self._cmk_config_dir()):
            os.unlink(path)


def run_cleanup_data():
    runner = TestRunner()
    test_case = TestAgentBasedStandardHosts()
    test_case._cleanup_hosts()
    test_case._activate_changes()
    runner._cleanup_host_files(cleanup_rrds=True)
    test_case._activate_changes()


#.
#   .--Reports-------------------------------------------------------------.
#   |                 ____                       _                         |
#   |                |  _ \ ___ _ __   ___  _ __| |_ ___                   |
#   |                | |_) / _ \ '_ \ / _ \| '__| __/ __|                  |
#   |                |  _ <  __/ |_) | (_) | |  | |_\__ \                  |
#   |                |_| \_\___| .__/ \___/|_|   \__|___/                  |
#   |                          |_|                                         |
#   +----------------------------------------------------------------------+
#   | Create some reports out of the test results                          |
#   '----------------------------------------------------------------------'

def run_report_list():
    for entry in sorted(os.listdir(Test.data_dir())):
        sys.stdout.write(entry + "\n")


def run_report(test_id):
    if not Test.exists(test_id):
        raise MKGeneralException("The test '%s' does not exist." % test_id)

    test = Test(test_id)

    if test.state != Test.STATE_STOPPED:
        sys.stdout.write("WARNING: This test is not complete yet.\n")
        sys.stdout.write("\n")

    sys.stdout.write("= SUMMARY ==================================\n")
    sys.stdout.write("\n")
    sys.stdout.write(" Test version : %s\n" % test.version)
    sys.stdout.write(" Date         : %s\n" %
        fmt_datetime(test.first_measurement().time))
    sys.stdout.write(" State        : %s\n" % test.state_text())

    result_txt = test.result
    if test.result == "failed":
        result_txt += " (%s)" % test.result_msg

    sys.stdout.write(" Result       : %s\n" % result_txt)
    sys.stdout.write(" Duration     : %s\n" % fmt_timespan(test.duration()))
    sys.stdout.write("\n")
    sys.stdout.write(" Version      : %s\n" % test.system_info.omd_version),
    sys.stdout.write(" Site         : %s\n" % test.system_info.omd_site),
    sys.stdout.write(" Core         : %s\n" % test.system_info.core),
    sys.stdout.write("\n")

    test_cases_by_name = {}
    for test_case in test.test_cases:
        test_cases_by_name[test_case.name()] = test_case

    for test_case_name, measurements in test.get_measurements_by_test_case().items():
        test_case = test_cases_by_name[test_case_name]

        sys.stdout.write("= %s =================\n" % test_case_name)
        sys.stdout.write("\n")

        values = Measurement.summary(measurements)

        result_txt = test_case.result
        if test_case.result == "failed":
            result_txt += " (%s)" % test_case.result_msg

        data = [
            "State       : %s" % test_case.state_text(),
            "Result      : %s" % result_txt,
            "Duration    : %s" % fmt_timespan(test_case.duration()),
            "",
            "CPU load    : %0.2f %0.2f %0.2f" % values.cpu_load,
            "Memory      : %0.2f %%" % values.memory.percent,
            "Disk %-6s : %s free" %
            (os.path.basename(test.system_info.block_device),
             fmt_bytes(values.disk_usage.free)),
            "Disk IO     : read %s/s, write %s/s" %
            (fmt_bytes(values.disk_io.read_bytes),
             fmt_bytes(values.disk_io.write_bytes)),
        ]

        if values.site_stats:
            data += [
                "",
                "Objects     : %d Hosts, %d Services" % (
                    values.site_stats["num_hosts"],
                    values.site_stats["num_services"]),
                "Check lat.  : %0.2f sec" % values.site_stats["average_latency_cmk"],
                "Checks      : %0.2f Hosts/s %0.2f Services/s" % (
                    values.site_stats["host_checks_rate"],
                    values.site_stats["service_checks_rate"]),
                "CMK helpers : %0.2f %%" % (values.site_stats["helper_usage_cmk"]*100),
            ]
        else:
            data += [
                "ERROR: Got no site statistics"
            ]

        for value in data:
            sys.stdout.write("  %s\n" % value)

        sys.stdout.write("\n")



#.
#   .--GUI-----------------------------------------------------------------.
#   |                            ____ _   _ ___                            |
#   |                           / ___| | | |_ _|                           |
#   |                          | |  _| | | || |                            |
#   |                          | |_| | |_| || |                            |
#   |                           \____|\___/|___|                           |
#   |                                                                      |
#   +----------------------------------------------------------------------+
#   | The GUI rendering code                                               |
#   '----------------------------------------------------------------------'

class Window(npyscreen.FormBaseNew):
    pass



class GeneralInformation(npyscreen.BoxTitle):
    def __init__(self, *args, **kwargs):
        super(GeneralInformation, self).__init__(*args, **kwargs)
        self.name     = "General Information"
        self.editable = False
        self._runner  = kwargs["runner"]


    def update(self, *args, **kwargs):
        self.values = [
            "State      : %s" % self._runner.test.state_text(),
        ]

        if self._runner.test.has_measurements():
            values = self._runner.test.current_measurement()

            self.values += [
                "Duration   : %s" % fmt_timespan(self._runner.test.duration()),
                "",
                "CPU load   : %0.2f %0.2f %0.2f" % values.cpu_load,
                "Memory     : %0.2f %%" % values.memory.percent,
                "Disk %-6s: %s free" %
                (os.path.basename(self._runner.test.system_info.block_device),
                 fmt_bytes(values.disk_usage.free)),
                "Disk IO    : r %s/s, w %s/s" %
                (fmt_bytes(values.disk_io.read_bytes),
                 fmt_bytes(values.disk_io.write_bytes)),
            ]
        super(GeneralInformation, self).update(*args, **kwargs)



class SiteInformation(npyscreen.BoxTitle):
    def __init__(self, *args, **kwargs):
        super(SiteInformation, self).__init__(*args, **kwargs)
        self.name     = "Site"
        self.editable = False
        self._runner  = kwargs["runner"]


    def update(self, *args, **kwargs):
        self._update_info()
        super(SiteInformation, self).update(*args, **kwargs)


    def _update_info(self):
        system_info = self._runner.test.system_info

        self.values = [
            "Version     : %s" % system_info.omd_version,
            "Core        : %s" % system_info.core,
        ]

        if not self._runner.test.has_measurements():
            return

        values = self._runner.test.current_measurement()
        self.values += [
            "State       : %s" % self._site_state_text(values.site_state),
        ]

        if not values.site_stats:
            return

        self.values += [
            "Objects     : %d H, %d S" % (
                values.site_stats["num_hosts"],
                values.site_stats["num_services"]),
            "Check lat.  : %0.2f s" % values.site_stats["average_latency_cmk"],
            "Checks      : %0.2f H/s %0.2f S/s" % (
                values.site_stats["host_checks_rate"],
                values.site_stats["service_checks_rate"]),
            "CMK helpers : %0.2f %%" % (values.site_stats["helper_usage_cmk"]*100),
        ]


    def _site_state_text(self, state):
        return {
            0: "running",
            1: "stopped",
            2: "partially running",
        }.get(state, "UNKNOWN (%d)" % state)



class Log(npyscreen.BoxTitle):
    _contained_widget = npyscreen.BufferPager
    def __init__(self, *args, **kwargs):
        kwargs["contained_widget_arguments"] = {
            "autowrap": True,
        }
        super(Log, self).__init__(*args, **kwargs)
        self.name     = "Log"
        self.editable = False
        self._runner  = kwargs["runner"]


    def update(self, *args, **kwargs):
        entries = []
        for timestamp, level, entry in self._runner.test.get_log():
            if "\n" not in entry:
                lines = [entry]
            else:
                lines = entry.split("\n")

            for line in lines:
                entries.append("%s %s" %
                    (time.strftime("%H:%M:%S", time.localtime(timestamp)),
                     line))

        self.entry_widget.clearBuffer()
        self.entry_widget.buffer(entries, scroll_end=True)
        super(Log, self).update(*args, **kwargs)



class HelpText(npyscreen.FixedText):
    def __init__(self, *args, **kwargs):
        super(HelpText, self).__init__(*args, **kwargs)
        self.value = "^C: quit"


class ConsoleRenderer(npyscreen.NPSApp):
    def __init__(self, runner):
        super(ConsoleRenderer, self).__init__()

        self._runner = runner

        # GUI widgets
        self._window  = None
        self._help    = None

    def main(self):
        npyscreen.setTheme(npyscreen.Themes.ElegantTheme)
        self.keypress_timeout_default = 10

        self._window = Window(
            parentApp=self,
            name="Check_MK System Benchmark %s" % __version__
        )

        max_y, max_x = self._window.curses_pad.getmaxyx()
        form_border     = 1*2
        info_box_width  = (max_x - form_border) / 2
        info_box_height = 10
        help_height     = 2
        log_height      = max_y - info_box_height - form_border - help_height

        self._general = self._window.add(
            GeneralInformation,
            relx=1,
            rely=1,
            max_height=info_box_height,
            width=info_box_width,
            runner=self._runner
        )

        self._site = self._window.add(
            SiteInformation,
            relx=info_box_width + 1,
            rely=1,
            max_height=info_box_height,
            width=info_box_width,
            runner=self._runner
        )

        self._log = self._window.add(
            Log,
            relx=1,
            rely=info_box_height + 1,
            height=log_height,
            width=self._window.curses_pad.getmaxyx()[1] - 2,
            runner=self._runner,
        )

        self._help = self._window.add(
            HelpText,
            height=1,
            relx=1,
            rely=-3,
        )

        self._window.edit()


    def while_waiting(self):
        self._general.update()
        self._window.display(clear=True)


#.
#   .--Main----------------------------------------------------------------.
#   |                        __  __       _                                |
#   |                       |  \/  | __ _(_)_ __                           |
#   |                       | |\/| |/ _` | | '_ \                          |
#   |                       | |  | | (_| | | | | |                         |
#   |                       |_|  |_|\__,_|_|_| |_|                         |
#   |                                                                      |
#   +----------------------------------------------------------------------+
#   |                                                                      |
#   '----------------------------------------------------------------------'

opt_debug = False

def main():
    global opt_debug
    register_signal_handlers()

    set_cmdline(" ".join(["mkbench"] + sys.argv[1:]))

    verify_is_site()

    num_hosts, num_helpers = 100, 20

    short_options = "h"
    long_options = [ "help", "version", "debug", "report", "cleanup-data",
                     "num-hosts=", "num-helpers=" ]

    try:
        opts, args = getopt.getopt(sys.argv[1:], short_options, long_options)
    except getopt.GetoptError, e:
        usage("%s" % e)

    mode = "run"

    for o, a in opts:
        if o in [ "-h", "--help" ]:
            usage()
        elif o == "--version":
            sys.stdout.write("mkbench %s\n" % __version__)
            sys.exit(0)
        elif o == "--debug":
            opt_debug = True

        elif o == "--report":
            mode = "report"

        elif o == "--cleanup-data":
            mode = "cleanup-data"

        elif o == "--num-hosts":
            try:
                num_hosts = int(a)
            except ValueError:
                raise MKGeneralException("--num-hosts must be given as number")

            if num_hosts not in TestRunner.host_steps:
                raise MKGeneralException("--num-hosts must be one of: %s" % ", ".join(map(str, TestRunner.host_steps)))

        elif o == "--num-helpers":
            try:
                num_helpers = int(a)
            except ValueError:
                raise MKGeneralException("--num-helpers must be given as number")


    if mode == "run":
        run_performance_test(num_hosts, num_helpers)
    elif mode == "report":
        if args:
            run_report(args[0])
        else:
            run_report_list()
    elif mode == "cleanup-data":
        run_cleanup_data()
    else:
        raise NotImplementedError()


def verify_is_site():
    if "OMD_ROOT" not in os.environ:
        raise MKGeneralException("mkbench can only be run as Check_MK site user")


def verify_site_is_running():
    site_state = subprocess.call(["omd", "status", "--bare"],
                  stdout=file(os.devnull, "w"), close_fds=True)
    if site_state != 0:
        raise MKGeneralException("Site needs to be running")


def verify_site_is_empty():
    hosts = LocalConnection().query_column("GET hosts\nColumns: host_name")
    # Allow sites:
    # - empty
    # - created with "omd-vonheute"
    # - previously terminated mkbench runs
    if not hosts or hosts == ["heute"] or all([ h.startswith("std-agent-host-") for h in hosts ]):
        return

    raise MKGeneralException("Site needs to be empty")


if __name__ == "__main__":
    try:
        main()
    except KeyboardInterrupt:
        sys.stderr.write("Terminated.\n")
        sys.exit(0)

    except MKGeneralException, e:
        sys.stderr.write("%s\n" % e)
        if opt_debug:
            raise
        sys.exit(3)
