lib.cuckoo.core.resultserver

Package lib :: Package cuckoo :: Package core :: Module resultserver |
[hide private]
[frames] | no frames]

# Source Code for Module lib.cuckoo.core.resultserver

      1  # Copyright (C) 2010-2013 Claudio Guarnieri.
      2  # Copyright (C) 2014-2016 Cuckoo Foundation.
      3  # This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org
      4  # See the file 'docs/LICENSE' for copying permission.
      5
      6  import errno
      7  import json
      8  import os
      9  import socket
     10  import select
     11  import logging
     12  import datetime
     13  import SocketServer
     14  import threading
     15
     16  from lib.cuckoo.common.abstracts import ProtocolHandler
     17  from lib.cuckoo.common.config import Config
     18  from lib.cuckoo.common.constants import CUCKOO_ROOT
     19  from lib.cuckoo.common.exceptions import CuckooOperationalError
     20  from lib.cuckoo.common.exceptions import CuckooCriticalError
     21  from lib.cuckoo.common.exceptions import CuckooResultError
     22  from lib.cuckoo.common.netlog import BsonParser
     23  from lib.cuckoo.common.utils import create_folder, Singleton
     24  from lib.cuckoo.core.log import task_log_start, task_log_stop
     25
     26  log = logging.getLogger(__name__)
     27
     28  BUFSIZE = 16 * 1024
     29
     30 -class Disconnect(Exception):
     31      pass
     32
     33 -class ResultServer(SocketServer.ThreadingTCPServer, object):
     34      """Result server. Singleton!
     35
     36      This class handles results coming back from the analysis machines.
     37      """
     38
     39      __metaclass__ = Singleton
     40
     41      allow_reuse_address = True
     42      daemon_threads = True
     43
     44 -    def __init__(self, *args, **kwargs):
     45          self.cfg = Config()
     46          self.analysistasks = {}
     47          self.analysishandlers = {}
     48
     49          ip = self.cfg.resultserver.ip
     50          self.port = int(self.cfg.resultserver.port)
     51          while True:
     52              try:
     53                  server_addr = ip, self.port
     54                  SocketServer.ThreadingTCPServer.__init__(
     55                      self, server_addr, ResultHandler, *args, **kwargs
     56                  )
     57              except Exception as e:
     58                  if e.errno == errno.EADDRINUSE:
     59                      if self.cfg.resultserver.get("force_port"):
     60                          raise CuckooCriticalError(
     61                              "Cannot bind ResultServer on port %d, "
     62                              "bailing." % self.port
     63                          )
     64                      else:
     65                          log.warning("Cannot bind ResultServer on port %s, "
     66                                      "trying another port.", self.port)
     67                          self.port += 1
     68                  elif e.errno == errno.EADDRNOTAVAIL:
     69                      raise CuckooCriticalError(
     70                          "Unable to bind ResultServer on %s:%s %s. This "
     71                          "usually happens when you start Cuckoo without "
     72                          "bringing up the virtual interface associated with "
     73                          "the ResultServer IP address. Please refer to "
     74                          "http://docs.cuckoosandbox.org/en/latest/faq/#troubles-problem"
     75                          " for more information." % (ip, self.port, e)
     76                      )
     77                  else:
     78                      raise CuckooCriticalError(
     79                          "Unable to bind ResultServer on %s:%s: %s" %
     80                          (ip, self.port, e)
     81                      )
     82              else:
     83                  log.debug("ResultServer running on %s:%s.", ip, self.port)
     84                  self.servethread = threading.Thread(target=self.serve_forever)
     85                  self.servethread.setDaemon(True)
     86                  self.servethread.start()
     87                  break
     88
     89 -    def add_task(self, task, machine):
     90          """Register a task/machine with the ResultServer."""
     91          self.analysistasks[machine.ip] = task, machine
     92          self.analysishandlers[task.id] = []
     93
     94 -    def del_task(self, task, machine):
     95          """Delete ResultServer state and wait for pending RequestHandlers."""
     96          x = self.analysistasks.pop(machine.ip, None)
     97          if not x:
     98              log.warning("ResultServer did not have %s in its task info.",
     99                          machine.ip)
    100          handlers = self.analysishandlers.pop(task.id, None)
    101          for h in handlers:
    102              h.end_request.set()
    103              h.done_event.wait()
    104
    105 -    def register_handler(self, handler):
    106          """Register a RequestHandler so that we can later wait for it."""
    107          task, machine = self.get_ctx_for_ip(handler.client_address[0])
    108          if not task or not machine:
    109              return False
    110
    111          self.analysishandlers[task.id].append(handler)
    112
    113 -    def get_ctx_for_ip(self, ip):
    114          """Return state for this IP's task."""
    115          x = self.analysistasks.get(ip)
    116          if not x:
    117              log.critical("ResultServer unable to map ip to context: %s.", ip)
    118              return None, None
    119
    120          return x
    121
    122 -    def build_storage_path(self, ip):
    123          """Initialize analysis storage folder."""
    124          task, machine = self.get_ctx_for_ip(ip)
    125          if not task or not machine:
    126              return
    127
    128          return os.path.join(CUCKOO_ROOT, "storage", "analyses", str(task.id))
    129
    130 -class ResultHandler(SocketServer.BaseRequestHandler):
    131      """Result handler.
    132
    133      This handler speaks our analysis log network protocol.
    134      """
    135
    136 -    def setup(self):
    137          self.rawlogfd = None
    138          self.protocol = None
    139          self.startbuf = ""
    140          self.end_request = threading.Event()
    141          self.done_event = threading.Event()
    142          self.server.register_handler(self)
    143
    144          if hasattr(select, "poll"):
    145              self.poll = select.poll()
    146              self.poll.register(self.request, select.POLLIN)
    147          else:
    148              self.poll = None
    149
    150 -    def finish(self):
    151          self.done_event.set()
    152
    153          if self.protocol:
    154              self.protocol.close()
    155          if self.rawlogfd:
    156              self.rawlogfd.close()
    157
    158 -    def wait_sock_or_end(self):
    159          while True:
    160              if self.end_request.isSet():
    161                  return False
    162
    163              if self.poll:
    164                  if self.poll.poll(1000):
    165                      return True
    166              else:
    167                  rs, _, _ = select.select([self.request], [], [], 1)
    168                  if rs:
    169                      return True
    170
    171 -    def seek(self, pos):
    172          pass
    173
    174 -    def read(self, length):
    175          buf = ""
    176          while len(buf) < length:
    177              if not self.wait_sock_or_end():
    178                  raise Disconnect()
    179              tmp = self.request.recv(length-len(buf))
    180              if not tmp:
    181                  raise Disconnect()
    182              buf += tmp
    183
    184          if isinstance(self.protocol, BsonParser):
    185              if self.rawlogfd:
    186                  self.rawlogfd.write(buf)
    187              else:
    188                  self.startbuf += buf
    189
    190                  if len(self.startbuf) > 0x10000:
    191                      raise CuckooResultError(
    192                          "Somebody is knowingly overflowing the startbuf "
    193                          "buffer, possibly to use excessive amounts of memory."
    194                      )
    195
    196          return buf
    197
    198 -    def read_any(self):
    199          if not self.wait_sock_or_end():
    200              raise Disconnect()
    201          tmp = self.request.recv(BUFSIZE)
    202          if not tmp:
    203              raise Disconnect()
    204          return tmp
    205
    206 -    def read_newline(self, strip=False):
    207          buf = ""
    208          while "\n" not in buf:
    209              buf += self.read(1)
    210
    211          if strip:
    212              buf = buf.strip()
    213
    214          return buf
    215
    216 -    def negotiate_protocol(self):
    217          protocol = self.read_newline(strip=True)
    218
    219          # Command with version number.
    220          if " " in protocol:
    221              command, version = protocol.split()
    222              version = int(version)
    223          else:
    224              command, version = protocol, None
    225
    226          if command == "BSON":
    227              self.protocol = BsonParser(self, version)
    228          elif command == "FILE":
    229              self.protocol = FileUpload(self, version)
    230          elif command == "LOG":
    231              self.protocol = LogHandler(self, version)
    232          else:
    233              raise CuckooOperationalError(
    234                  "Netlog failure, unknown protocol requested."
    235              )
    236
    237          self.protocol.init()
    238
    239 -    def handle(self):
    240          ip, port = self.client_address
    241          self.connect_time = datetime.datetime.now()
    242
    243          self.storagepath = self.server.build_storage_path(ip)
    244          if not self.storagepath:
    245              return
    246
    247          task, _ = self.server.get_ctx_for_ip(ip)
    248          task_log_start(task.id)
    249
    250          # Create all missing folders for this analysis.
    251          self.create_folders()
    252
    253          try:
    254              # Initialize the protocol handler class for this connection.
    255              self.negotiate_protocol()
    256
    257              for event in self.protocol:
    258                  if isinstance(self.protocol, BsonParser) and event["type"] == "process":
    259                      self.open_process_log(event)
    260
    261          except CuckooResultError as e:
    262              log.warning(
    263                  "ResultServer connection stopping because of "
    264                  "CuckooResultError: %s.", e
    265              )
    266          except (Disconnect, socket.error):
    267              pass
    268          except:
    269              log.exception("FIXME - exception in resultserver connection %s",
    270                            self.client_address)
    271
    272          task_log_stop(task.id)
    273
    274 -    def open_process_log(self, event):
    275          pid = event["pid"]
    276          ppid = event["ppid"]
    277          procname = event["process_name"]
    278
    279          if self.rawlogfd:
    280              log.debug(
    281                  "ResultServer got a new process message but already "
    282                  "has pid %d ppid %s procname %s.", pid, ppid, procname
    283              )
    284              raise CuckooResultError(
    285                  "ResultServer connection state inconsistent."
    286              )
    287
    288          if not isinstance(pid, (int, long)):
    289              raise CuckooResultError(
    290                  "An invalid process identifier has been provided, this "
    291                  "could be a potential security hazard."
    292              )
    293
    294          # Only report this process when we're tracking it.
    295          if event["track"]:
    296              log.debug(
    297                  "New process (pid=%s, ppid=%s, name=%s)",
    298                  pid, ppid, procname
    299              )
    300
    301          filepath = os.path.join(self.storagepath, "logs", "%s.bson" % pid)
    302          self.rawlogfd = open(filepath, "wb")
    303          self.rawlogfd.write(self.startbuf)
    304
    305 -    def create_folders(self):
    306          folders = "shots", "files", "logs", "buffer"
    307
    308          for folder in folders:
    309              try:
    310                  create_folder(self.storagepath, folder=folder)
    311              except CuckooOperationalError:
    312                  log.error("Unable to create folder %s" % folder)
    313                  return False
    314
    315 -class FileUpload(ProtocolHandler):
    316      RESTRICTED_DIRECTORIES = "reports/",
    317      lock = threading.Lock()
    318
    319 -    def init(self):
    320          self.upload_max_size = \
    321              self.handler.server.cfg.resultserver.upload_max_size
    322          self.storagepath = self.handler.storagepath
    323          self.fd = None
    324
    325          self.filelog = os.path.join(self.handler.storagepath, "files.json")
    326
    327 -    def __iter__(self):
    328          # Read until newline for file path, e.g.,
    329          # shots/0001.jpg or files/9498687557/libcurl-4.dll.bin
    330
    331          dump_path = self.handler.read_newline(strip=True).replace("\\", "/")
    332
    333          if self.version >= 2:
    334              filepath = self.handler.read_newline(strip=True)
    335              pids = map(int, self.handler.read_newline(strip=True).split())
    336          else:
    337              filepath, pids = None, []
    338
    339          log.debug("File upload request for %s", dump_path)
    340
    341          dir_part, filename = os.path.split(dump_path)
    342
    343          if "./" in dump_path or not dir_part or dump_path.startswith("/"):
    344              raise CuckooOperationalError(
    345                  "FileUpload failure, banned path: %s" % dump_path
    346              )
    347
    348          for restricted in self.RESTRICTED_DIRECTORIES:
    349              if restricted in dir_part:
    350                  raise CuckooOperationalError(
    351                      "FileUpload failure, banned path."
    352                  )
    353
    354          try:
    355              create_folder(self.storagepath, dir_part)
    356          except CuckooOperationalError:
    357              log.error("Unable to create folder %s", dir_part)
    358              return
    359
    360          file_path = os.path.join(self.storagepath, dump_path)
    361
    362          if not file_path.startswith(self.storagepath):
    363              raise CuckooOperationalError(
    364                  "FileUpload failure, path sanitization failed."
    365              )
    366
    367          if os.path.exists(file_path):
    368              log.warning(
    369                  "Analyzer tried to overwrite an existing file, "
    370                  "closing connection."
    371              )
    372              return
    373
    374          self.fd = open(file_path, "wb")
    375          chunk = self.handler.read_any()
    376          while chunk:
    377              self.fd.write(chunk)
    378
    379              if self.fd.tell() >= self.upload_max_size:
    380                  log.warning(
    381                      "Uploaded file length larger than upload_max_size, "
    382                      "stopping upload."
    383                  )
    384                  self.fd.write("... (truncated)")
    385                  break
    386
    387              try:
    388                  chunk = self.handler.read_any()
    389              except:
    390                  break
    391
    392          self.lock.acquire()
    393
    394          with open(self.filelog, "a+b") as f:
    395              f.write("%s\n" % json.dumps({
    396                  "path": dump_path,
    397                  "filepath": filepath,
    398                  "pids": pids,
    399              }))
    400
    401          self.lock.release()
    402
    403          log.debug("Uploaded file length: %s", self.fd.tell())
    404          return
    405          yield
    406
    407 -    def close(self):
    408          if self.fd:
    409              self.fd.close()
    410
    411 -class LogHandler(ProtocolHandler):
    412 -    def init(self):
    413          self.logpath = os.path.join(self.handler.storagepath, "analysis.log")
    414          self.fd = self._open()
    415          log.debug("LogHandler for live analysis.log initialized.")
    416
    417 -    def __iter__(self):
    418          if not self.fd:
    419              return
    420
    421          while True:
    422              try:
    423                  buf = self.handler.read_newline(strip=False)
    424              except Disconnect:
    425                  break
    426
    427              if not buf:
    428                  break
    429
    430              self.fd.write(buf)
    431              self.fd.flush()
    432
    433          return
    434          yield
    435
    436 -    def close(self):
    437          if self.fd:
    438              self.fd.close()
    439
    440 -    def _open(self):
    441          if not os.path.exists(self.logpath):
    442              return open(self.logpath, "wb")
    443
    444          log.debug("Log analysis.log already existing, appending data.")
    445          fd = open(self.logpath, "ab")
    446
    447          # add a fake log entry, saying this had to be re-opened
    448          #  use the same format as the default logger, in case anyone wants to parse this
    449          #  2015-02-23 12:05:05,092 [lib.api.process] DEBUG: Using QueueUserAPC injection.
    450          now = datetime.datetime.now()
    451          print >>fd, "\n%s,%03.0f [lib.core.resultserver] WARNING: This log file was re-opened, log entries will be appended." % (
    452              now.strftime("%Y-%m-%d %H:%M:%S"), now.microsecond / 1000.0
    453          )
    454
    455          return fd
    456