# Functions to control the nodes' operations.

from collections import namedtuple
import glob
import os
import shutil
import time
import logging

from BroControl import execute
from BroControl import events
from BroControl import util
from BroControl import config
from BroControl import install
from BroControl import cron
from BroControl import node as node_mod
from BroControl import cmdresult


# Waits for the nodes' Bro processes to reach the given status.
# Build the Bro parameters for the given node. Include
# script for live operation if live is true.
def _make_bro_params(node, live):
    args = []

    if live and node.interface:
        try:
            # Interface name needs quotes so that shell doesn't interpret any
            # potential metacharacters in the name.
            args += ["-i", "'%s'" % node.interface]
        except AttributeError:
            pass

        if config.Config.savetraces == "1":
            args += ["-w", "trace.pcap"]

    args += ["-U", ".status"]
    args += ["-p", "broctl"]

    if live:
        args += ["-p", "broctl-live"]

    if node.type == "standalone":
        args += ["-p", "standalone"]

    for prefix in config.Config.prefixes.split(":"):
        args += ["-p", "%s" % prefix]

    args += ["-p", "%s" % node.name]

    # The order of loaded scripts is as follows:
    # 1) local.bro gives a common set of loaded scripts for all nodes.
    # 2) The common configuration of broctl is loaded via the broctl package.
    # 3) The distribution's default settings for node configuration are loaded
    #    from either the cluster framework or standalone scripts.  This also
    #    involves loading local-<node>.bro scripts.  At this point anything
    #    in the distribution's default per-node is overridable and any
    #    identifiers in local.bro are able to be used (e.g. in defining
    #    a notice policy).
    # 4) Autogenerated broctl scripts are loaded, which may contain
    #    settings that override the previously loaded node-specific scripts.
    #    (e.g. Log::default_rotation_interval is set in manager.bro,
    #    but overrided by broctl.cfg)
    args += config.Config.sitepolicystandalone.split()
    args += ["broctl"]
    if node.type == "standalone":
        args += ["broctl/standalone"]
    else:
        args += ["base/frameworks/cluster"]
        if node.type == "manager":
            args += config.Config.sitepolicymanager.split()
        elif node.type == "proxy":
            args += ["local-proxy"]
        elif node.type == "worker":
            args += config.Config.sitepolicyworker.split()
    args += ["broctl/auto"]

    if getattr(node, "aux_scripts", None):
        args += [node.aux_scripts]

    if config.Config.broargs:
        # Some args in broargs might contain spaces, so we cannot split it.
        args += [config.Config.broargs]

    return args

# Build the environment variables for the given node.
def _make_env_params(node, returnlist=False):
    envs = []
    if node.type != "standalone":
        envs.append("CLUSTER_NODE=%s" % node.name)

    envs += ["%s=%s" % (key, val) for (key, val) in sorted(node.env_vars.items())]

    if returnlist:
        envlist = [("-v", i) for i in envs]
        return [j for i in envlist for j in i]

    return " ".join(envs)


class Controller:
    def __init__(self, config, ui, executor, pluginregistry):
        self.config = config
        self.ui = ui
        self.executor = executor
        self.pluginregistry = pluginregistry

    def start(self, nodes):
        results = cmdresult.CmdResult()
        manager = []
        proxies = []
        workers = []

        for n in nodes:
            n.setExpectRunning(True)

            if n.type == "worker":
                workers += [n]
            elif n.type == "proxy":
                proxies += [n]
            else:
                manager += [n]

        # Start nodes. Do it in the order manager, proxies, workers.
        if manager:
            self._start_nodes(manager, results)

            if not results.ok:
                for n in (proxies + workers):
                    results.set_node_fail(n)
                return results

        if proxies:
            self._start_nodes(proxies, results)

            if not results.ok:
                for n in workers:
                    results.set_node_fail(n)
                return results

        if workers:
            self._start_nodes(workers, results)

        return results


    # Starts the given nodes.
    def _start_nodes(self, nodes, results):
        filtered = []
        # Ignore nodes which are still running.
        for (node, isrunning) in self._isrunning(nodes):
            if not isrunning:
                filtered += [node]
                if node.hasCrashed():
                    self.ui.info("starting %s (was crashed) ..." % node.name)
                else:
                    self.ui.info("starting %s ..." % node.name)
            else:
                self.ui.info("%s still running" % node.name)

        nodes = filtered

        # Generate crash report for any crashed nodes.
        crashed = [node for node in nodes if node.hasCrashed()]
        self._make_crash_reports(crashed)

        # Make working directories.
        dirs = [(node, node.cwd()) for node in nodes]
        nodes = []
        for (node, success) in self.executor.mkdirs(dirs):
            if success:
                nodes += [node]
            else:
                self.ui.error("cannot create working directory for %s" % node.name)
                results.set_node_fail(node)

        # Start Bro process.
        cmds = []
        for node in nodes:
            envs = []
            pin_cpu = node.pin_cpus

            # If this node isn't using CPU pinning, then use a placeholder value
            if pin_cpu == "":
                pin_cpu = -1

            envs = _make_env_params(node, True)
            cmds += [(node, "start", envs + [node.cwd(), str(pin_cpu)] + _make_bro_params(node, True))]

        nodes = []
        # Note: the shell is used to interpret the command because broargs
        # might contain quoted arguments.
        for (node, success, output) in self.executor.run_helper(cmds, shell=True):
            if success:
                nodes += [node]
                node.setPID(int(output[0]))
            else:
                self.ui.error("cannot start %s; check output of \"diag\"" % node.name)
                results.set_node_fail(node)
                if output:
                    self.ui.error("\n".join(output))

        # Check whether processes did indeed start up.
        hanging = []
        running = []

        for (node, success) in self._waitforbros(nodes, "RUNNING", 3, True):
            if success:
                running += [node]
            else:
                hanging += [node]

        # It can happen that Bro hangs in DNS lookups at startup
        # which can take a while. At this point we already know
        # that the process has been started (_waitforbros ensures that).
        # If by now there is not a TERMINATED status, we assume that it
        # is doing fine and will move on to RUNNING once DNS is done.
        for (node, success) in self._waitforbros(hanging, "TERMINATED", 0, False):
            if success:
                self.ui.info("%s terminated immediately after starting; check output with \"diag\"" % node.name)
                node.clearPID()
                results.set_node_fail(node)
            else:
                self.ui.info("(%s still initializing)" % node.name)
                running += [node]

        for node in running:
            self._log_action(node, "started")
            results.set_node_success(node)

        return results

    def _isrunning(self, nodes, setcrashed=True):

        results = []
        cmds = []

        for node in nodes:
            pid = node.getPID()
            if not pid:
                results += [(node, False)]
                continue

            cmds += [(node, "check-pid", [str(pid)])]

        for (node, success, output) in self.executor.run_helper(cmds):
            # If we cannot run the helper script, then we ignore this node
            # because the process might actually be running but we can't tell.
            if not success:
                if self.config.cron == "0":
                    self.ui.error("cannot connect to %s" % node.name)
                continue

            running = output[0] == "running" and True or False

            results += [(node, running)]

            if not running:
                if setcrashed:
                    # Grmpf. It crashed.
                    node.clearPID()
                    node.setCrashed()

        return results

    def _waitforbros(self, nodes, status, timeout, ensurerunning):
        # If ensurerunning is true, process must still be running.
        if ensurerunning:
            running = self._isrunning(nodes)
        else:
            running = [(node, True) for node in nodes]

        results = []

        # Determine set of nodes still to check.
        todo = {}
        for (node, isrunning) in running:
            if isrunning:
                todo[node.name] = node
            else:
                results += [(node, False)]

        while True:
            # Determine  whether process is still running. We need to do this
            # before we get the state to avoid a race condition.
            running = self._isrunning(todo.values(), setcrashed=False)

            # Check nodes' .status file
            cmds = []
            for node in todo.values():
                cmds += [(node, "first-line", ["%s/.status" % node.cwd()])]

            for (node, success, output) in self.executor.run_helper(cmds):
                if not success or not output[0]:
                    continue

                fields = output[0].split()
                if len(fields) == 2:
                    if status in fields[0]:
                        # Status reached. Cool.
                        del todo[node.name]
                        results += [(node, True)]
                else:
                    # Something's wrong. We give up on that node.
                    del todo[node.name]
                    results += [(node, False)]

            for (node, isrunning) in running:
                if node.name in todo and not isrunning:
                    # Alright, a dead node's status will not change anymore.
                    del todo[node.name]
                    results += [(node, False)]

            if not todo:
                # All done.
                break

            # Wait a bit before we start over.
            time.sleep(1)

            # Timeout reached?
            timeout -= 1
            if timeout <= 0:
                break

            logging.debug("Waiting for %d node(s)...", len(todo))

        for node in todo.values():
            # These did time-out.
            results += [(node, False)]

        if todo:
            logging.debug("Timeout while waiting for %d node(s)", len(todo))

        return results

    def _log_action(self, node, action):
        if self.config.statslogenable == "0":
            return
        t = time.time()
        with open(self.config.statslog, "a") as out:
            out.write("%s %s action %s\n" % (t, node, action))

    # Do a "post-terminate crash" for the given nodes.
    def _make_crash_reports(self, nodes):
        for n in nodes:
            self.pluginregistry.broProcessDied(n)

        msg = "If you want to help us debug this problem, then please forward\nthis mail to reports@bro.org\n"
        postterminate = os.path.join(self.config.scriptsdir, "post-terminate")
        cmds = [(node, postterminate, [node.cwd(), "crash"]) for node in nodes]

        for (node, success, output) in self.executor.run_cmds(cmds):
            if success:
                msuccess, moutput = self._sendmail("Crash report from %s" % node.name, msg + "\n".join(output))
                if not msuccess:
                    self.ui.error("error occurred while trying to send mail: %s" % moutput[0])
            else:
                self.ui.error("error running post-terminate for %s: %s" % (node.name, output[0]))

            node.clearCrashed()

    def _sendmail(self, subject, body):
        if not self.config.sendmail:
            return True, ""

        cmd = "%s '%s'" % (os.path.join(self.config.scriptsdir, "send-mail"), subject)
        (success, output) = execute.run_localcmd(cmd, "", body)
        return success, output

    # Stop Bro processes on nodes.
    def stop(self, nodes):
        results = cmdresult.CmdResult()
        manager = []
        proxies = []
        workers = []

        for n in nodes:
            n.setExpectRunning(False)

            if n.type == "worker":
                workers += [n]
            elif n.type == "proxy":
                proxies += [n]
            else:
                manager += [n]


        # Stop nodes. Do it in the order workers, proxies, manager
        # (the reverse of "start").
        if workers:
            self._stop_nodes(workers, results)

            if not results.ok:
                for n in (proxies + manager):
                    results.set_node_fail(n)
                return results

        if proxies:
            self._stop_nodes(proxies, results)

            if not results.ok:
                for n in manager:
                    results.set_node_fail(n)
                return results

        if manager:
            self._stop_nodes(manager, results)

        return results

    def _stop_nodes(self, nodes, results):

        running = []

        # Check for crashed nodes.
        for (node, isrunning) in self._isrunning(nodes):
            if isrunning:
                running += [node]
                self.ui.info("stopping %s ..." % node.name)
            else:
                results.set_node_success(node)

                if node.hasCrashed():
                    self.ui.info("%s not running (was crashed)" % node.name)
                    self._make_crash_reports([node])
                else:
                    self.ui.info("%s not running" % node.name)

        # Helper function to stop nodes with given signal.
        def stop(nodes, signal):
            cmds = []
            for node in nodes:
                cmds += [(node, "stop", [str(node.getPID()), str(signal)])]

            return self.executor.run_helper(cmds)

        # Stop nodes.
        for (node, success, output) in stop(running, 15):
            if not success:
                self.ui.error("failed to send stop signal to %s" % node.name)

        if running:
            time.sleep(1)

        # Check whether they terminated.
        terminated = []
        kill = []
        for (node, success) in self._waitforbros(running, "TERMINATED", int(self.config.stoptimeout), False):
            if not success:
                # Check whether it crashed during shutdown ...
                result = self._isrunning([node])
                for (node, isrunning) in result:
                    if isrunning:
                        self.ui.info("%s did not terminate ... killing ..." % node.name)
                        kill += [node]
                    else:
                        # crashed flag is set by _isrunning().
                        self.ui.info("%s crashed during shutdown" % node.name)

        if kill:
            # Kill those which did not terminate gracefully.
            stop(kill, 9)
            # Give them a bit to disappear.
            time.sleep(5)

        # Check which are still running. We check all nodes to be on the safe
        # side and give them a bit more time to finally disappear.
        timeout = 10

        todo = {}
        for node in running:
            todo[node.name] = node

        while True:

            running = self._isrunning(todo.values(), setcrashed=False)

            for (node, isrunning) in running:
                if node.name in todo and not isrunning:
                    # Alright, it's gone.
                    del todo[node.name]
                    terminated += [node]
                    results.set_node_success(node)

            if not todo:
                # All done.
                break

            # Wait a bit before we start over.

            if timeout <= 0:
                break

            time.sleep(1)
            timeout -= 1

        for node in todo:
            results.set_node_fail(node)

        # Do post-terminate cleanup for those which terminated gracefully.
        cleanup = [node for node in terminated if not node.hasCrashed()]

        cmds = []
        postterminate = os.path.join(self.config.scriptsdir, "post-terminate")
        for node in cleanup:
            crashflag = ""
            if node in kill:
                crashflag = "killed"

            cmds += [(node, postterminate, [node.cwd(), crashflag])]

        for (node, success, output) in self.executor.run_cmds(cmds):
            if success:
                self._log_action(node, "stopped")
            else:
                self.ui.error("error running post-terminate for %s: %s" % (node.name, output[0]))
                self._log_action(node, "stopped (failed)")

            node.clearPID()
            node.clearCrashed()

        return results


    # Output status summary for nodes.
    def status(self, nodes):
        results = cmdresult.CmdResult()

        showall = self.config.statuscmdshowall != "0"

        if showall:
            self.ui.info("Getting process status ...")

        nodestatus = self._isrunning(nodes)
        running = []

        cmds = []
        for (node, isrunning) in nodestatus:
            if isrunning:
                running += [node]
                cmds += [(node, "first-line", ["%s/.startup" % node.cwd(), "%s/.status" % node.cwd()])]

        startups = {}
        statuses = {}
        for (n, success, output) in self.executor.run_helper(cmds):
            startups[n.name] = (success and output[0]) and util.fmttime(output[0]) or "???"
            statuses[n.name] = (success and output[1]) and output[1].split()[0].lower() or "???"

        if showall:
            self.ui.info("Getting peer status ...")
            peers = {}
            nodes = [n for n in running if statuses[n.name] == "running"]
            for (node, success, args) in self._query_peerstatus(nodes):
                if success:
                    peers[node.name] = []
                    for f in args[0].split():
                        keyval = f.split("=")
                        if len(keyval) > 1:
                            (key, val) = keyval
                            if key == "peer" and val != "":
                                peers[node.name] += [val]

        for (node, isrunning) in nodestatus:
            node_info = {
                "name": node.name,
                "type": node.type,
                "host": node.host,
                "status": "stopped",
                "pid": None,
                "started": None,
            }
            if showall:
                node_info["peers"] = None

            if isrunning:
                node_info["status"] = statuses[node.name]
            elif node.hasCrashed():
                node_info["status"] = "crashed"

            if isrunning:
                node_info["pid"] = node.getPID()

                if showall:
                    if node.name in peers:
                        node_info["peers"] = len(peers[node.name])
                    else:
                        node_info["peers"] = "???"

                node_info["started"] = startups[node.name]

            results.set_node_data(node, True, node_info)

        return results

    # Check the configuration for nodes without installing first.
    def check(self, nodes):
        return self._check_config(nodes, False, False)

    # Print the loaded_scripts.log for either the installed scripts
    # (if "check" is false), or the original scripts (if "check" is true).
    def scripts(self, nodes, check):
        return self._check_config(nodes, not check, True)


    def _check_config(self, nodes, installed, list_scripts):
        results = cmdresult.CmdResult()

        nodetmpdirs = [(node, os.path.join(self.config.tmpdir, "check-config-%s" % node.name)) for node in nodes]

        nodes = []
        for (node, cwd) in nodetmpdirs:
            if os.path.isdir(cwd):
                try:
                    shutil.rmtree(cwd)
                except OSError as err:
                    self.ui.error("cannot remove directory: %s" % err)
                    results.ok = False
                    return results

            try:
                os.makedirs(cwd)
            except OSError as err:
                self.ui.error("cannot create temporary directory: %s" % err)
                results.ok = False
                return results

            nodes += [(node, cwd)]

        cmds = []
        for (node, cwd) in nodes:

            env = _make_env_params(node)

            installed_policies = installed and "1" or "0"
            print_scripts = list_scripts and "1" or "0"

            install.make_layout(cwd, self.ui, True)
            if not install.make_local_networks(cwd, self.ui, True):
                results.ok = False
                return results

            install.make_broctl_config_policy(cwd, self.ui, True)

            cmd = os.path.join(self.config.scriptsdir, "check-config") + " %s %s %s %s" % (installed_policies, print_scripts, cwd, " ".join(_make_bro_params(node, False)))
            cmd += " broctl/check"

            cmds += [((node, cwd), cmd, env, None)]

        for ((node, cwd), success, output) in execute.run_localcmds(cmds):
            results.set_node_output(node, success, output)
            shutil.rmtree(cwd)

        return results

    def _query_peerstatus(self, nodes):
        running = self._isrunning(nodes)

        eventlist = []
        for (node, isrunning) in running:
            if isrunning:
                eventlist += [(node, "Control::peer_status_request", [], "Control::peer_status_response")]

        return events.send_events_parallel(eventlist)

    def execute_cmd(self, nodes, cmd):
        results = cmdresult.CmdResult()

        for node, success, out in self.executor.run_shell_cmds([(n, cmd) for n in nodes]):
            results.set_node_output(node, success, out)

        return results

    # Clean up the working directory for nodes (flushes state).
    # If cleantmp is true, also wipes ${tmpdir}; this is done
    # even when the node is still running.
    def cleanup(self, nodes, cleantmp=False):
        def addfailed(orig, res):
            for (n, status) in res:
                if not status:
                    orig.add(n.name)

            return orig


        results = cmdresult.CmdResult()

        result = self._isrunning(nodes)
        running    = [node for (node, on) in result if on]
        notrunning = [node for (node, on) in result if not on]

        for node in running:
            self.ui.info("   %s is still running, not cleaning work directory" % node)

        results1 = self.executor.rmdirs([(n, n.cwd()) for n in notrunning])
        results2 = self.executor.mkdirs([(n, n.cwd()) for n in notrunning])
        failed = set()
        failed = addfailed(failed, results1)
        failed = addfailed(failed, results2)

        for node in notrunning:
            node.clearCrashed()

        if cleantmp:
            results3 = self.executor.rmdirs([(n, self.config.tmpdir) for n in running + notrunning])
            results4 = self.executor.mkdirs([(n, self.config.tmpdir) for n in running + notrunning])
            failed = addfailed(failed, results3)
            failed = addfailed(failed, results4)

        for node in nodes:
            if node.name in failed:
                results.set_node_fail(node)
            else:
                results.set_node_success(node)

        return results

    # Report diagnostics for nodes (e.g., stderr output).
    def diag(self, nodes):
        results = cmdresult.CmdResult()

        crashdiag = os.path.join(self.config.scriptsdir, "crash-diag")
        cmds = [(node, crashdiag, [node.cwd()]) for node in nodes]

        for (node, success, output) in self.executor.run_cmds(cmds):
            if not success:
                errmsgs = ["error running crash-diag for %s" % node.name]
                errmsgs += output
                results.set_node_output(node, False, errmsgs)
                continue

            results.set_node_output(node, True, output)

        return results

    def capstats(self, nodes, interval):
        results = cmdresult.CmdResult()

        if self.config.capstatspath:
            for (node, netif, success, vals) in self.get_capstats_output(nodes, interval):
                if not success:
                    vals = {"output": vals}
                results.set_node_data(node, success, vals)

        return results

    # Gather capstats from interfaces.
    #
    # Returns a list of tuples of the form (node, netif, success, vals)
    # where 'netif' is the network interface name used by capstats on
    # the 'node', and 'success' is a boolean indicating whether or not
    # we were able to get the data; in case there's no error, 'vals' maps
    # tags to their values (otherwise, 'vals' is an error message).
    #
    # Tags are those as returned by capstats on the command-line.
    #
    # If there is more than one node, then the results will also contain
    # one "pseudo-node" of the name "$total" with the sum of all individual
    # values.
    def get_capstats_output(self, nodes, interval):
        results = []

        # Construct a list of (node, interface) tuples, one tuple for each
        # unique (host, interface) pair.
        nodenetifs = []
        hosts = {}
        for node in nodes:
            if not node.interface:
                continue

            netif = self._capstats_interface(node)

            if hosts.setdefault((node.addr, netif), node) == node:
                nodenetifs.append((node, netif))

        capstats = self.config.capstatspath
        cmds = [(node, capstats, ["-I", str(interval), "-n", "1", "-i", interface]) for (node, interface) in nodenetifs]

        outputs = self.executor.run_cmds(cmds)

        totals = {}

        for (node, success, output) in outputs:
            netif = self._capstats_interface(node)

            if not success:
                if output:
                    results += [(node, netif, False, "%s: capstats failed (%s)" % (node.name, output[0]))]
                else:
                    results += [(node, netif, False, "%s: cannot execute capstats" % node.name)]
                continue

            if not output:
                results += [(node, netif, False, "%s: no capstats output" % node.name)]
                continue

            fields = output[0].split()[1:]

            if not fields:
                results += [(node, netif, False, "%s: unexpected capstats output: %s" % (node.name, output[0]))]
                continue

            vals = {}

            try:
                for field in fields:
                    (key, val) = field.split("=")
                    val = float(val)
                    vals[key] = val

                    if key in totals:
                        totals[key] += val
                    else:
                        totals[key] = val

                results += [(node, netif, True, vals)]

            except ValueError:
                results += [(node, netif, False, "%s: unexpected capstats output: %s" % (node.name, output[0]))]

        # Add pseudo-node for totals
        if len(nodes) > 1:
            results += [(node_mod.Node(self.config, "$total"), None, True, totals)]

        return results


    def _capstats_interface(self, node):
        netif = node.interface

        # If PF_RING+DNA with pfdnacluster_master is being used, then this hack
        # is needed to prevent capstats from trying to use the same interface
        # name as Bro.
        if netif.startswith("dnacl") and netif.count("@") == 1:
            netif = netif.split("@", 1)[0]

        return netif


    # Get current statistics from cFlow.
    #
    # Returns dict of the form port->(cum-pkts, cum-bytes).
    #
    # Returns None if we can't run the helper sucessfully.
    def getCFlowStatus(self):
        (success, output) = execute.run_localcmd(os.path.join(self.config.scriptsdir, "cflow-stats"))
        if not success or not output:
            self.ui.error("failed to run cflow-stats")
            return None

        vals = {}

        for line in output:
            try:
                (port, pps, bps, pkts, bytes) = line.split()
                vals[port] = (float(pkts), float(bytes))
            except ValueError:
                # Probably an error message because we can't connect.
                self.ui.error("failed to get cFlow statistics: %s" % line)
                return None

        return vals

    # Calculates the differences between two getCFlowStatus() calls.
    # Returns a list of tuples in the same form as get_capstats_output() does.
    def calculateCFlowRate(self, start, stop, interval):
        diffs = [(port, stop[port][0] - start[port][0], (stop[port][1] - start[port][1])) for port in start.keys() if port in stop]

        rates = []
        for (port, pkts, bytes) in diffs:
            vals = {"kpps": "%.1f" % (pkts / 1e3 / interval)}
            if start[port][1] >= 0:
                vals["mbps"] = "%.1f" % (bytes * 8 / 1e6 / interval)

            rates += [(port, None, vals)]

        return rates

    # Update the configuration of a running instance on the fly.
    def update(self, nodes):
        results = cmdresult.CmdResult()

        running = self._isrunning(nodes)
        zone = self.config.zoneid
        if not zone:
            zone = "NOZONE"

        cmds = []
        for (node, isrunning) in running:
            if isrunning:
                env = _make_env_params(node)
                env += " BRO_DNS_FAKE=1"
                args = " ".join(_make_bro_params(node, False))
                cmds += [(node.name, os.path.join(self.config.scriptsdir, "update") + " %s %s %s/tcp %s" % (util.format_bro_addr(node.addr), zone, node.getPort(), args), env, None)]
                self.ui.info("updating %s ..." % node.name)

        res = execute.run_localcmds(cmds)

        for (tag, success, output) in res:
            node = self.config.nodes(tag=tag)[0]
            if not success:
                self.ui.info("failed to update %s: %s" % (tag, output[0]))
                results.set_node_fail(node)
            else:
                self.ui.info("%s: %s" % (tag, output[0]))
                results.set_node_success(node)

        return results

    # Gets disk space on all volumes relevant to broctl installation.
    # Returns a list of the form:  [ (host, diskinfo), ...]
    # where diskinfo is a list of the form DiskInfo named tuple objects (fs,
    # total, used, avail, percent) or ["FAIL", <error message>] if an error
    # is encountered.
    def df(self, nodes):
        results = cmdresult.CmdResult()

        DiskInfo = namedtuple("DiskInfo", ("fs", "total", "used", "available", "percent"))
        dirs = ("logdir", "bindir", "helperdir", "cfgdir", "spooldir",
                "policydir", "libdir", "tmpdir", "staticdir", "scriptsdir")

        df = {}
        for node in nodes:
            df[node.name] = {}

        cmds = []
        for node in nodes:
            for key in dirs:
                if key == "logdir" and node.type not in ("manager", "standalone"):
                    # Don't need this on the workers/proxies.
                    continue

                path = self.config.config[key]

                cmds += [(node, "df", [path])]

        res = self.executor.run_helper(cmds)

        for (node, success, output) in res:
            if success:
                if not output:
                    df[node.name]["FAIL"] = "no output from df helper"
                    continue

                fields = output[0].split()

                fs = fields[0]
                # Ignore NFS mounted volumes.
                if ":" in fs:
                    continue

                total = float(fields[1])
                used = float(fields[2])
                avail = float(fields[3])
                perc = used * 100.0 / (used + avail)
                df[node.name][fs] = DiskInfo(fs, total, used, avail, perc)
            else:
                if output:
                    msg = output[0]
                else:
                    msg = "unknown failure"
                df[node.name]["FAIL"] = msg

        for node in nodes:
            success = "FAIL" not in df[node.name]
            results.set_node_data(node, success, df[node.name])

        return results

    # Returns a list of tuples of the form (node, error, vals) where 'error' is
    # an error message string, or None if there was no error.  'vals' is a list
    # of dicts which map tags to their values.  Tags are "pid", "proc", "vsize",
    # "rss", "cpu", and "cmd".
    def get_top_output(self, nodes):

        results = []
        cmds = []

        running = self._isrunning(nodes)

        # Get all the PIDs first.

        pids = {}
        parents = {}

        for (node, isrunning) in running:
            if isrunning:
                pid = node.getPID()
                pids[node.name] = [pid]
                parents[node.name] = str(pid)

                cmds += [(node, "get-childs", [str(pid)])]
            else:
                results += [(node, "not running", [{}])]
                continue

        if not cmds:
            return results

        for (node, success, output) in self.executor.run_helper(cmds):

            if not success:
                results += [(node, "cannot get child pids", [{}])]
                continue

            pids[node.name] += [int(line) for line in output]

        cmds = []
        hosts = {}

        # Now run top once per host.
        for node in nodes:   # Do the loop again to keep the order.
            if node.name not in pids:
                continue

            if node.host in hosts:
                continue

            hosts[node.host] = 1

            cmds += [(node, "top", [])]

        if not cmds:
            return results

        res = {}
        for (node, success, output) in self.executor.run_helper(cmds):
            res[node.host] = (success, output)

        # Gather results for all the nodes that are running
        for node in nodes:
            if node.name not in pids:
                continue

            success, output = res[node.host]

            if not success:
                results += [(node, "top failed: %s" % output[0], [{}])]
                continue

            if not output:
                results += [(node, "no output from top", [{}])]
                continue

            procs = [line.split() for line in output if int(line.split()[0]) in pids[node.name]]

            if not procs:
                # It's possible that the process is no longer there.
                results += [(node, "not running", [{}])]
                continue

            vals = []

            try:
                for p in procs:
                    d = {}
                    d["pid"] = int(p[0])
                    d["proc"] = (p[0] == parents[node.name] and "parent" or "child")
                    d["vsize"] = int(float(p[1])) #May be something like 2.17684e+9
                    d["rss"] = int(float(p[2]))
                    d["cpu"] = p[3]
                    d["cmd"] = " ".join(p[4:])
                    vals += [d]
            except ValueError as err:
                results += [(node, "unexpected top output: %s" % err, [{}])]
                continue

            results += [(node, None, vals)]

        return results

    # Produce a top-like output for node's processes.
    def top(self, nodes):
        results = cmdresult.CmdResult()

        for (node, error, vals) in self.get_top_output(nodes):
            top_info = {"name": node.name, "type": node.type,
                        "host": node.host, "pid": None, "proc": None,
                        "vsize": None, "rss": None, "cpu": None,
                        "cmd": None, "error": None}
            if error:
                top_info["error"] = error
                results.set_node_data(node, False, {"procs": [top_info]})
                continue

            proclist = []
            for d in vals:
                top_info2 = top_info.copy()
                top_info2.update(d)
                proclist.append(top_info2)

            results.set_node_data(node, True, {"procs": proclist})

        return results

    def print_id(self, nodes, id):
        running = self._isrunning(nodes)

        eventlist = []
        for (node, isrunning) in running:
            if isrunning:
                eventlist += [(node, "Control::id_value_request", [id], "Control::id_value_response")]

        results = cmdresult.CmdResult()
        for (node, success, args) in events.send_events_parallel(eventlist):
            results.set_node_output(node, success, args)

        return results


    def _query_netstats(self, nodes):
        running = self._isrunning(nodes)

        eventlist = []
        for (node, isrunning) in running:
            if isrunning:
                eventlist += [(node, "Control::net_stats_request", [], "Control::net_stats_response")]

        return events.send_events_parallel(eventlist)

    def peerstatus(self, nodes):
        results = cmdresult.CmdResult()
        for (node, success, args) in self._query_peerstatus(nodes):
            if success:
                out = args[0]
            else:
                out = args
            results.set_node_output(node, success, out)

        return results

    def netstats(self, nodes):
        results = cmdresult.CmdResult()
        for (node, success, args) in self._query_netstats(nodes):
            if success:
                out = args[0].strip()
            else:
                out = args
            results.set_node_output(node, success, out)

        return results

    def process(self, trace, bro_options, bro_scripts):
        results = cmdresult.CmdResult()

        if not os.path.isfile(trace):
            self.ui.error("trace file not found: %s" % trace)
            results.ok = False
            return results

        standalone = (self.config.standalone == "1")
        if standalone:
            tag = "standalone"
        else:
            tag = "workers"

        node = self.config.nodes(tag=tag)[0]

        cwd = os.path.join(self.config.tmpdir, "testing")

        if os.path.isdir(cwd):
            try:
                shutil.rmtree(cwd)
            except OSError as err:
                self.ui.error("cannot remove directory: %s" % err)
                results.ok = False
                return results

        try:
            os.makedirs(cwd)
        except OSError as err:
            self.ui.error("cannot create directory: %s" % err)
            results.ok = False
            return results

        env = _make_env_params(node)

        bro_args =  " ".join(bro_options + _make_bro_params(node, False))
        bro_args += " broctl/process-trace"

        if bro_scripts:
            bro_args += " " + " ".join(bro_scripts)

        cmd = os.path.join(self.config.scriptsdir, "run-bro-on-trace") + " %s %s %s %s" % (0, cwd, trace, bro_args)

        self.ui.info(cmd)

        (success, output) = execute.run_localcmd(cmd, env, donotcaptureoutput=True)

        if not success:
            results.ok = False

        for line in output:
            self.ui.info(line)

        self.ui.info("\n### Bro output in %s" % cwd)

        return results

    def install(self, local_only):
        results = cmdresult.CmdResult()

        try:
            self.config.record_bro_version()
        except config.ConfigurationError as err:
            self.ui.error("%s" % err)
            results.ok = False
            return results

        manager = self.config.manager()

        # Delete previously installed policy files to not mix things up.
        policies = [self.config.policydirsiteinstall, self.config.policydirsiteinstallauto]

        for dirpath in policies:
            if os.path.isdir(dirpath):
                self.ui.info("removing old policies in %s ..." % dirpath)
                try:
                    shutil.rmtree(dirpath)
                except OSError as err:
                    self.ui.error("failed to remove directory: %s" % err)
                    results.ok = False
                    return results

        self.ui.info("creating policy directories ...")
        for dirpath in policies:
            try:
                os.makedirs(dirpath)
            except OSError as err:
                self.ui.error("failed to create directory: %s" % err)
                results.ok = False
                return results

        # Install local site policy.

        if self.config.sitepolicypath:
            self.ui.info("installing site policies ...")
            dst = self.config.policydirsiteinstall
            for dir in self.config.sitepolicypath.split(":"):
                dirpath = self.config.subst(dir)
                for pathname in glob.glob(os.path.join(dirpath, "*")):
                    if not execute.install(pathname, dst, self.ui):
                        results.ok = False
                        return results

        install.make_layout(self.config.policydirsiteinstallauto, self.ui)

        self.ui.info("generating local-networks.bro ...")
        if not install.make_local_networks(self.config.policydirsiteinstallauto, self.ui):
            results.ok = False
            return results

        self.ui.info("generating broctl-config.bro ...")
        install.make_broctl_config_policy(self.config.policydirsiteinstallauto, self.ui)

        current = self.config.subst(os.path.join(self.config.logdir, "current"))
        try:
            util.force_symlink(manager.cwd(), current)
        except (IOError, OSError) as err:
            results.ok = False
            self.ui.error("failed to update symlink '%s': %s" % (current, err))
            return results

        self.ui.info("generating broctl-config.sh ...")
        if not install.make_broctl_config_sh(self.ui):
            results.ok = False
            return results

        if local_only:
            return results

        # Sync to clients.
        self.ui.info("updating nodes ...")

        # Make sure we install each remote host only once.
        nodes = self.config.hosts(nolocal=True)

        dirs = []

        if self.config.havenfs != "1":
            # Non-NFS, need to explicitly synchronize.
            syncs = install.get_syncs()
        else:
            # NFS. We only need to take care of the spool/log directories.

            # We need this only on the manager.
            dirs.append((manager, self.config.logdir))

            syncs = install.get_nfssyncs()

        createdirs = [self.config.subst(dir) for (dir, mirror) in syncs if not mirror]
        for n in nodes:
            for dir in createdirs:
                dirs.append((n, dir))

        for (node, success) in self.executor.mkdirs(dirs):
            if not success:
                self.ui.error("cannot create (some of the) directories %s on node %s" % (",".join(createdirs), node.name))
                results.ok = False
                return results

        paths = [self.config.subst(dir) for (dir, mirror) in syncs if mirror]
        if not execute.sync(nodes, paths, self.ui):
            results.ok = False
            return results

        # Save current node configuration state.
        self.config.update_nodecfg_hash()

        # Save current configuration state.
        self.config.update_broctlcfg_hash()

        return results


    # Triggers all activity which is to be done regularly via cron.
    def cron(self, watch):
        if not self.config.has_attr("cronenabled"):
            self.config.set_state("cronenabled", True)

        if not self.config.cronenabled:
            return

        # Check if "broctl install" has been run.
        if not os.path.exists(os.path.join(self.config.scriptsdir, "broctl-config.sh")):
            # Don't output anything here, otherwise the cron job may generate
            # emails before the user has a chance to do "broctl install".
            return

        # Flag to indicate that we're running from cron.
        self.config.config["cron"] = "1"

        cronui = cron.CronUI()
        tasks = cron.CronTasks(cronui, self.config, self, self.executor, self.pluginregistry)

        cronui.buffer_output()

        if watch:
            # Check if node state matches expected state, and start/stop if
            # necessary.
            startlist = []
            stoplist = []
            for (node, isrunning) in self._isrunning(self.config.nodes()):
                expectrunning = node.getExpectRunning()

                if not isrunning and expectrunning:
                    startlist.append(node)
                elif isrunning and not expectrunning:
                    stoplist.append(node)

            if startlist:
                results = self.start(startlist)
            if stoplist:
                results = self.stop(stoplist)

        # Check for dead hosts.
        tasks.check_hosts()

        # Generate statistics.
        tasks.log_stats(5)

        # Check available disk space.
        tasks.check_disk_space()

        # Expire old log files.
        tasks.expire_logs()

        # Update the HTTP stats directory.
        tasks.update_http_stats()

        # Run external command if we have one.
        tasks.run_cron_cmd()

        # Mail potential output.
        output = cronui.get_buffered_output()
        if output:
            success, out = self._sendmail("cron: " + output.splitlines()[0], output)
            if not success:
                self.ui.error("broctl cron failed to send mail: %s" % out[0])
                self.ui.info("\nOutput of broctl cron:\n%s" % output)

        self.config.config["cron"] = "0"
        logging.debug("cron done")

