modules.processing.suricata

Package modules :: Package processing :: Module suricata |
[hide private]
[frames] | no frames]

# Source Code for Module modules.processing.suricata

      1  # Copyright (C) 2010-2013 Claudio Guarnieri.
      2  # Copyright (C) 2014-2016 Cuckoo Foundation.
      3  # This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org
      4  # See the file 'docs/LICENSE' for copying permission.
      5
      6  import json
      7  import logging
      8  import os
      9  import shutil
     10  import subprocess
     11  import time
     12
     13  from lib.cuckoo.common.abstracts import Processing
     14  from lib.cuckoo.common.exceptions import CuckooProcessingError
     15  from lib.cuckoo.common.utils import md5_file, sha1_file
     16
     17  try:
     18      import suricatasc
     19      HAVE_SURICATASC = True
     20  except ImportError:
     21      HAVE_SURICATASC = False
     22
     23  log = logging.getLogger(__name__)
     24
     25 -class Suricata(Processing):
     26      """Suricata processing module."""
     27
     28      # List of Suricata Signatures IDs that should be ignored.
     29      sid_blacklist = [
     30          # SURICATA FRAG IPv6 Fragmentation overlap
     31          2200074,
     32
     33          # ET INFO InetSim Response from External Source Possible SinkHole
     34          2017363,
     35
     36          # SURICATA UDPv4 invalid checksum
     37          2200075,
     38      ]
     39
     40 -    def process_pcap_socket(self):
     41          """Process a PCAP file with Suricata in socket mode."""
     42          if not HAVE_SURICATASC:
     43              raise CuckooProcessingError(
     44                  "Suricata has been configured to run in socket mode but "
     45                  "suricatasc has not been installed, please re-install "
     46                  "Suricata or SuricataSC"
     47              )
     48
     49          if not os.path.exists(self.socket):
     50              raise CuckooProcessingError(
     51                  "Suricata has been configured to run in socket mode "
     52                  "but the socket is unavailable"
     53              )
     54
     55          suri = suricatasc.SuricataSC(self.socket)
     56
     57          try:
     58              suri.connect()
     59          except suricatasc.SuricataException as e:
     60              raise CuckooProcessingError(
     61                  "Error connecting to Suricata in socket mode: %s" % e
     62              )
     63
     64          # Submit the PCAP file.
     65          ret = suri.send_command("pcap-file", {
     66              "filename": self.pcap_path,
     67              "output-dir": self.suricata_path,
     68          })
     69
     70          if not ret or ret["return"] != "OK":
     71              raise CuckooProcessingError(
     72                  "Error submitting PCAP file to Suricata in socket mode, "
     73                  "return value: %s" % ret
     74              )
     75
     76          # TODO Should we add a timeout here? If we do so we should also add
     77          # timeout logic to the binary mode.
     78          while True:
     79              ret = suri.send_command("pcap-current")
     80
     81              # When the pcap file has been processed the "current pcap" file
     82              # will be none.
     83              if ret and ret["message"] == "None":
     84                  break
     85
     86              time.sleep(1)
     87
     88 -    def process_pcap_binary(self):
     89          """Process a PCAP file with Suricata by running Suricata.
     90
     91          Using the socket mode is preferred as the plain binary mode requires
     92          Suricata to load all its rules for every PCAP file and thus takes a
     93          couple of performance heavy seconds to set itself up.
     94          """
     95          if not os.path.isfile(self.suricata):
     96              raise CuckooProcessingError("Unable to locate Suricata binary")
     97
     98          if not os.path.isfile(self.config_path):
     99              raise CuckooProcessingError(
    100                  "Unable to locate Suricata configuration"
    101              )
    102
    103          args = [
    104              self.suricata,
    105              "-c", self.config_path,
    106              "-k", "none",
    107              "-l", self.suricata_path,
    108              "-r", self.pcap_path,
    109          ]
    110
    111          try:
    112              subprocess.check_call(args)
    113          except subprocess.CalledProcessError as e:
    114              raise CuckooProcessingError(
    115                  "Suricata returned an error processing this pcap: %s" % e
    116              )
    117
    118 -    def parse_eve_json(self):
    119          """Parse the eve.json file."""
    120          eve_log = os.path.join(self.suricata_path, self.eve_log)
    121          if not os.path.isfile(eve_log):
    122              log.warning("Unable to find the eve.json log file")
    123              return
    124
    125          for line in open(eve_log, "rb"):
    126              event = json.loads(line)
    127
    128              if event["event_type"] == "alert":
    129                  alert = event["alert"]
    130
    131                  if alert["signature_id"] in self.sid_blacklist:
    132                      log.debug(
    133                          "Ignoring alert with sid=%d, signature=%s",
    134                          alert["signature_id"], alert["signature"]
    135                      )
    136                      continue
    137
    138                  if alert["signature"].startswith("SURICATA STREAM"):
    139                      log.debug(
    140                          "Ignoring alert starting with \"SURICATA STREAM\""
    141                      )
    142                      continue
    143
    144                  self.results["alerts"].append({
    145                      "sid": alert["signature_id"],
    146                      "src_ip": event.get("src_ip"),
    147                      "src_port": event.get("src_port"),
    148                      "dst_ip": event["dest_ip"],
    149                      "dst_port": event.get("dest_port"),
    150                      "protocol": event.get("proto"),
    151                      "timestamp": event.get("timestamp"),
    152                      "category": alert.get("category") or "undefined",
    153                      "signature": alert["signature"],
    154                  })
    155
    156              elif event["event_type"] == "http":
    157                  http = event["http"]
    158
    159                  referer = http.get("http_referer")
    160                  if referer == "<unknown>":
    161                      referer = None
    162
    163                  user_agent = http.get("http_user_agent")
    164                  if user_agent == "<unknown>":
    165                      user_agent = None
    166
    167                  self.results["http"].append({
    168                      "src_ip": event.get("src_ip"),
    169                      "src_port": event.get("src_port"),
    170                      "dst_ip": event.get("dest_ip"),
    171                      "dst_port": event.get("dest_port"),
    172                      "timestamp": event.get("timestamp"),
    173                      "method": http.get("http_method"),
    174                      "hostname": http.get("hostname"),
    175                      "url": http.get("url"),
    176                      "status": "%s" % http.get("status"),
    177                      "content_type": http.get("http_content_type"),
    178                      "user_agent": user_agent,
    179                      "referer": referer,
    180                      "length": http.get("length"),
    181                  })
    182
    183              elif event["event_type"] == "tls":
    184                  tls = event["tls"]
    185
    186                  self.results["tls"].append({
    187                      "src_ip": event.get("src_ip"),
    188                      "src_port": event.get("src_port"),
    189                      "dst_ip": event.get("dest_ip"),
    190                      "dst_port": event.get("dest_port"),
    191                      "timestamp": event.get("timestamp"),
    192                      "fingerprint": tls.get("fingerprint"),
    193                      "issuer": tls.get("issuerdn"),
    194                      "version": tls.get("version"),
    195                      "subject": tls.get("subject"),
    196                  })
    197
    198 -    def parse_files(self):
    199          """Parse the files-json.log file and its associated files."""
    200          files_log = os.path.join(self.suricata_path, self.files_log)
    201          if not os.path.isfile(files_log):
    202              log.warning("Unable to find the files-json.log log file")
    203              return
    204
    205          files = {}
    206
    207          # Index all the available files.
    208          files_dir = os.path.join(self.suricata_path, self.files_dir)
    209          if not os.path.exists(files_dir):
    210              log.warning("Suricata files dir is not available. Maybe you forgot to enable Suricata file-store ?")
    211              return
    212
    213          for filename in os.listdir(files_dir):
    214              filepath = os.path.join(files_dir, filename)
    215              files[md5_file(filepath)] = filepath
    216
    217          for line in open(files_log, "rb"):
    218              event = json.loads(line)
    219
    220              # Not entirely sure what's up, but some files are given just an
    221              # ID, some files are given just an md5 hash (and maybe some get
    222              # neither?) So take care of these situations.
    223              if "id" in event:
    224                  filepath = os.path.join(files_dir, "file.%s" % event["id"])
    225              elif "md5" in event:
    226                  filepath = files.get(event["md5"])
    227              else:
    228                  filepath = None
    229
    230              if not filepath or not os.path.isfile(filepath):
    231                  log.warning(
    232                      "Suricata dropped file with id=%s and md5=%s not found, "
    233                      "skipping it..", event.get("id"), event.get("md5")
    234                  )
    235                  continue
    236
    237              referer = event.get("http_referer")
    238              if referer == "<unknown>":
    239                  referer = None
    240
    241              self.results["files"].append({
    242                  "id": int(filepath.split(".", 1)[-1]),
    243                  "filesize": event["size"],
    244                  "filename": os.path.basename(event["filename"]),
    245                  "hostname": event.get("http_host"),
    246                  "uri": event.get("http_uri"),
    247                  "md5": md5_file(filepath),
    248                  "sha1": sha1_file(filepath),
    249                  "magic": event.get("magic"),
    250                  "referer": referer,
    251              })
    252
    253 -    def run(self):
    254          self.key = "suricata"
    255
    256          self.results = {
    257              "alerts": [],
    258              "tls": [],
    259              "files": [],
    260              "http": [],
    261          }
    262
    263          self.suricata = self.options.get("suricata", "/usr/bin/suricata")
    264          self.config_path = self.options.get("conf", "/etc/suricata/suricata.yaml")
    265          self.eve_log = self.options.get("eve_log", "eve.json")
    266          self.files_log = self.options.get("files_log", "files-json.log")
    267          self.files_dir = self.options.get("files_dir", "files")
    268
    269          # Determines whether we're in socket more or binary mode.
    270          self.socket = self.options.get("socket")
    271
    272          if not os.path.isfile(self.pcap_path):
    273              log.warning("Unable to run Suricata as no pcap is available")
    274              return self.results
    275
    276          # Remove any existing Suricata related log-files before we
    277          # run Suricata again. I.e., prevent reprocessing an analysis from
    278          # generating duplicate results.
    279          if os.path.isdir(self.suricata_path):
    280              shutil.rmtree(self.suricata_path)
    281
    282          os.mkdir(self.suricata_path)
    283
    284          if self.socket:
    285              self.process_pcap_socket()
    286          else:
    287              self.process_pcap_binary()
    288
    289          self.parse_eve_json()
    290          self.parse_files()
    291
    292          return self.results
    293