lib.cuckoo.common.abstracts

Package lib :: Package cuckoo :: Package common :: Module abstracts |
[hide private]
[frames] | no frames]

# Source Code for Module lib.cuckoo.common.abstracts

       1  # Copyright (C) 2010-2013 Claudio Guarnieri.
       2  # Copyright (C) 2014-2016 Cuckoo Foundation.
       3  # This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org
       4  # See the file 'docs/LICENSE' for copying permission.
       5
       6  import os
       7  import re
       8  import logging
       9  import time
      10
      11  import xml.etree.ElementTree as ET
      12
      13  from lib.cuckoo.common.config import Config
      14  from lib.cuckoo.common.constants import CUCKOO_ROOT
      15  from lib.cuckoo.common.exceptions import CuckooCriticalError
      16  from lib.cuckoo.common.exceptions import CuckooMachineError
      17  from lib.cuckoo.common.exceptions import CuckooOperationalError
      18  from lib.cuckoo.common.exceptions import CuckooReportError
      19  from lib.cuckoo.common.exceptions import CuckooDependencyError
      20  from lib.cuckoo.common.objects import Dictionary
      21  from lib.cuckoo.common.utils import create_folder
      22  from lib.cuckoo.core.database import Database
      23
      24  try:
      25      import libvirt
      26      HAVE_LIBVIRT = True
      27  except ImportError:
      28      HAVE_LIBVIRT = False
      29
      30  log = logging.getLogger(__name__)
      31
      32 -class Auxiliary(object):
      33      """Base abstract class for auxiliary modules."""
      34
      35 -    def __init__(self):
      36          self.task = None
      37          self.machine = None
      38          self.guest_manager = None
      39          self.options = None
      40
      41 -    def set_task(self, task):
      42          self.task = task
      43
      44 -    def set_machine(self, machine):
      45          self.machine = machine
      46
      47 -    def set_guest_manager(self, guest_manager):
      48          self.guest_manager = guest_manager
      49
      50 -    def set_options(self, options):
      51          self.options = options
      52
      53 -    def start(self):
      54          raise NotImplementedError
      55
      56 -    def stop(self):
      57          raise NotImplementedError
      58
      59 -class Machinery(object):
      60      """Base abstract class for machinery modules."""
      61
      62      # Default label used in machinery configuration file to supply virtual
      63      # machine name/label/vmx path. Override it if you dubbed it in another
      64      # way.
      65      LABEL = "label"
      66
      67 -    def __init__(self):
      68          self.module_name = ""
      69          self.options = None
      70          self.options_globals = Config()
      71          # Database pointer.
      72          self.db = Database()
      73
      74          # Machine table is cleaned to be filled from configuration file
      75          # at each start.
      76          self.db.clean_machines()
      77
      78 -    def pcap_path(self, task_id):
      79          """Returns the .pcap path for this task id."""
      80          return os.path.join(CUCKOO_ROOT, "storage", "analyses",
      81                              "%s" % task_id, "dump.pcap")
      82
      83 -    def set_options(self, options):
      84          """Set machine manager options.
      85          @param options: machine manager options dict.
      86          """
      87          self.options = options
      88
      89 -    def initialize(self, module_name):
      90          """Read, load, and verify machines configuration.
      91          @param module_name: module name.
      92          """
      93          # Load.
      94          self._initialize(module_name)
      95
      96          # Run initialization checks.
      97          self._initialize_check()
      98
      99 -    def _get_resultserver_port(self):
     100          """Returns the ResultServer port."""
     101          # Avoid import recursion issues by importing ResultServer here.
     102          from lib.cuckoo.core.resultserver import ResultServer
     103          return ResultServer().port
     104
     105 -    def _initialize(self, module_name):
     106          """Read configuration.
     107          @param module_name: module name.
     108          """
     109          self.module_name = module_name
     110          mmanager_opts = self.options.get(module_name)
     111
     112          for machine_id in mmanager_opts["machines"].strip().split(","):
     113              try:
     114                  machine_opts = self.options.get(machine_id.strip())
     115                  machine = Dictionary()
     116                  machine.id = machine_id.strip()
     117                  machine.label = machine_opts[self.LABEL]
     118                  machine.platform = machine_opts["platform"]
     119                  machine.options = machine_opts.get("options", "")
     120                  machine.tags = machine_opts.get("tags")
     121                  machine.ip = machine_opts["ip"]
     122
     123                  # If configured, use specific network interface for this
     124                  # machine, else use the default value.
     125                  if machine_opts.get("interface"):
     126                      machine.interface = machine_opts["interface"]
     127                  else:
     128                      machine.interface = mmanager_opts.get("interface")
     129
     130                  # If configured, use specific snapshot name, else leave it
     131                  # empty and use default behaviour.
     132                  machine.snapshot = machine_opts.get("snapshot")
     133
     134                  # If configured, use specific resultserver IP and port,
     135                  # else use the default value.
     136                  opt_resultserver = self.options_globals.resultserver
     137
     138                  # the resultserver port might have been dynamically changed
     139                  #  -> get the current one from the resultserver singleton
     140                  opt_resultserver.port = self._get_resultserver_port()
     141
     142                  ip = machine_opts.get("resultserver_ip", opt_resultserver.ip)
     143                  port = machine_opts.get("resultserver_port", opt_resultserver.port)
     144
     145                  machine.resultserver_ip = ip
     146                  machine.resultserver_port = port
     147
     148                  # Strip parameters.
     149                  for key, value in machine.items():
     150                      if value and isinstance(value, basestring):
     151                          machine[key] = value.strip()
     152
     153                  self.db.add_machine(name=machine.id,
     154                                      label=machine.label,
     155                                      ip=machine.ip,
     156                                      platform=machine.platform,
     157                                      options=machine.options,
     158                                      tags=machine.tags,
     159                                      interface=machine.interface,
     160                                      snapshot=machine.snapshot,
     161                                      resultserver_ip=ip,
     162                                      resultserver_port=port)
     163              except (AttributeError, CuckooOperationalError) as e:
     164                  log.warning("Configuration details about machine %s "
     165                              "are missing: %s", machine_id.strip(), e)
     166                  continue
     167
     168 -    def _initialize_check(self):
     169          """Runs checks against virtualization software when a machine manager
     170          is initialized.
     171          @note: in machine manager modules you may override or superclass
     172                 his method.
     173          @raise CuckooMachineError: if a misconfiguration or a unkown vm state
     174                                     is found.
     175          """
     176          try:
     177              configured_vms = self._list()
     178          except NotImplementedError:
     179              return
     180
     181          for machine in self.machines():
     182              # If this machine is already in the "correct" state, then we
     183              # go on to the next machine.
     184              if machine.label in configured_vms and \
     185                      self._status(machine.label) in [self.POWEROFF, self.ABORTED]:
     186                  continue
     187
     188              # This machine is currently not in its correct state, we're going
     189              # to try to shut it down. If that works, then the machine is fine.
     190              try:
     191                  self.stop(machine.label)
     192              except CuckooMachineError as e:
     193                  msg = "Please update your configuration. Unable to shut " \
     194                        "'{0}' down or find the machine in its proper state:" \
     195                        " {1}".format(machine.label, e)
     196                  raise CuckooCriticalError(msg)
     197
     198          if not self.options_globals.timeouts.vm_state:
     199              raise CuckooCriticalError("Virtual machine state change timeout "
     200                                        "setting not found, please add it to "
     201                                        "the config file.")
     202
     203 -    def machines(self):
     204          """List virtual machines.
     205          @return: virtual machines list
     206          """
     207          return self.db.list_machines()
     208
     209 -    def availables(self):
     210          """How many machines are free.
     211          @return: free machines count.
     212          """
     213          return self.db.count_machines_available()
     214
     215 -    def acquire(self, machine_id=None, platform=None, tags=None):
     216          """Acquire a machine to start analysis.
     217          @param machine_id: machine ID.
     218          @param platform: machine platform.
     219          @param tags: machine tags
     220          @return: machine or None.
     221          """
     222          if machine_id:
     223              return self.db.lock_machine(label=machine_id)
     224          elif platform:
     225              return self.db.lock_machine(platform=platform, tags=tags)
     226          else:
     227              return self.db.lock_machine(tags=tags)
     228
     229 -    def release(self, label=None):
     230          """Release a machine.
     231          @param label: machine name.
     232          """
     233          self.db.unlock_machine(label)
     234
     235 -    def running(self):
     236          """Returns running virtual machines.
     237          @return: running virtual machines list.
     238          """
     239          return self.db.list_machines(locked=True)
     240
     241 -    def shutdown(self):
     242          """Shutdown the machine manager. Kills all alive machines.
     243          @raise CuckooMachineError: if unable to stop machine.
     244          """
     245          if len(self.running()) > 0:
     246              log.info("Still %s guests alive. Shutting down...",
     247                       len(self.running()))
     248              for machine in self.running():
     249                  try:
     250                      self.stop(machine.label)
     251                  except CuckooMachineError as e:
     252                      log.warning("Unable to shutdown machine %s, please check "
     253                                  "manually. Error: %s", machine.label, e)
     254
     255 -    def set_status(self, label, status):
     256          """Set status for a virtual machine.
     257          @param label: virtual machine label
     258          @param status: new virtual machine status
     259          """
     260          self.db.set_machine_status(label, status)
     261
     262 -    def start(self, label, task):
     263          """Start a machine.
     264          @param label: machine name.
     265          @param task: task object.
     266          @raise NotImplementedError: this method is abstract.
     267          """
     268          raise NotImplementedError
     269
     270 -    def stop(self, label=None):
     271          """Stop a machine.
     272          @param label: machine name.
     273          @raise NotImplementedError: this method is abstract.
     274          """
     275          raise NotImplementedError
     276
     277 -    def _list(self):
     278          """Lists virtual machines configured.
     279          @raise NotImplementedError: this method is abstract.
     280          """
     281          raise NotImplementedError
     282
     283 -    def dump_memory(self, label, path):
     284          """Takes a memory dump of a machine.
     285          @param path: path to where to store the memory dump.
     286          """
     287          raise NotImplementedError
     288
     289 -    def _wait_status(self, label, state):
     290          """Waits for a vm status.
     291          @param label: virtual machine name.
     292          @param state: virtual machine status, accepts multiple states as list.
     293          @raise CuckooMachineError: if default waiting timeout expire.
     294          """
     295          # This block was originally suggested by Loic Jaquemet.
     296          waitme = 0
     297          try:
     298              current = self._status(label)
     299          except NameError:
     300              return
     301
     302          if isinstance(state, str):
     303              state = [state]
     304
     305          while current not in state:
     306              log.debug("Waiting %i cuckooseconds for machine %s to switch "
     307                        "to status %s", waitme, label, state)
     308              if waitme > int(self.options_globals.timeouts.vm_state):
     309                  raise CuckooMachineError("Timeout hit while for machine {0} "
     310                                           "to change status".format(label))
     311              time.sleep(1)
     312              waitme += 1
     313              current = self._status(label)
     314
     315 -class LibVirtMachinery(Machinery):
     316      """Libvirt based machine manager.
     317
     318      If you want to write a custom module for a virtualization software
     319      supported by libvirt you have just to inherit this machine manager and
     320      change the connection string.
     321      """
     322
     323      # VM states.
     324      RUNNING = "running"
     325      PAUSED = "paused"
     326      POWEROFF = "poweroff"
     327      ERROR = "machete"
     328      ABORTED = "abort"
     329
     330 -    def __init__(self):
     331          if not HAVE_LIBVIRT:
     332              raise CuckooDependencyError("Unable to import libvirt")
     333
     334          super(LibVirtMachinery, self).__init__()
     335
     336 -    def initialize(self, module):
     337          """Initialize machine manager module. Override default to set proper
     338          connection string.
     339          @param module:  machine manager module
     340          """
     341          super(LibVirtMachinery, self).initialize(module)
     342
     343 -    def _initialize_check(self):
     344          """Runs all checks when a machine manager is initialized.
     345          @raise CuckooMachineError: if libvirt version is not supported.
     346          """
     347          # Version checks.
     348          if not self._version_check():
     349              raise CuckooMachineError("Libvirt version is not supported, "
     350                                       "please get an updated version")
     351
     352          # Preload VMs
     353          self.vms = self._fetch_machines()
     354
     355          # Base checks. Also attempts to shutdown any machines which are
     356          # currently still active.
     357          super(LibVirtMachinery, self)._initialize_check()
     358
     359 -    def start(self, label, task):
     360          """Starts a virtual machine.
     361          @param label: virtual machine name.
     362          @param task: task object.
     363          @raise CuckooMachineError: if unable to start virtual machine.
     364          """
     365          log.debug("Starting machine %s", label)
     366
     367          if self._status(label) != self.POWEROFF:
     368              msg = "Trying to start a virtual machine that has not " \
     369                    "been turned off {0}".format(label)
     370              raise CuckooMachineError(msg)
     371
     372          conn = self._connect()
     373
     374          vm_info = self.db.view_machine_by_label(label)
     375
     376          snapshot_list = self.vms[label].snapshotListNames(flags=0)
     377
     378          # If a snapshot is configured try to use it.
     379          if vm_info.snapshot and vm_info.snapshot in snapshot_list:
     380              # Revert to desired snapshot, if it exists.
     381              log.debug("Using snapshot {0} for virtual machine "
     382                        "{1}".format(vm_info.snapshot, label))
     383              try:
     384                  vm = self.vms[label]
     385                  snapshot = vm.snapshotLookupByName(vm_info.snapshot, flags=0)
     386                  self.vms[label].revertToSnapshot(snapshot, flags=0)
     387              except libvirt.libvirtError:
     388                  msg = "Unable to restore snapshot {0} on " \
     389                        "virtual machine {1}".format(vm_info.snapshot, label)
     390                  raise CuckooMachineError(msg)
     391              finally:
     392                  self._disconnect(conn)
     393          elif self._get_snapshot(label):
     394              snapshot = self._get_snapshot(label)
     395              log.debug("Using snapshot {0} for virtual machine "
     396                        "{1}".format(snapshot.getName(), label))
     397              try:
     398                  self.vms[label].revertToSnapshot(snapshot, flags=0)
     399              except libvirt.libvirtError:
     400                  raise CuckooMachineError("Unable to restore snapshot on "
     401                                           "virtual machine {0}".format(label))
     402              finally:
     403                  self._disconnect(conn)
     404          else:
     405              self._disconnect(conn)
     406              raise CuckooMachineError("No snapshot found for virtual machine "
     407                                       "{0}".format(label))
     408
     409          # Check state.
     410          self._wait_status(label, self.RUNNING)
     411
     412 -    def stop(self, label):
     413          """Stops a virtual machine. Kill them all.
     414          @param label: virtual machine name.
     415          @raise CuckooMachineError: if unable to stop virtual machine.
     416          """
     417          log.debug("Stopping machine %s", label)
     418
     419          if self._status(label) == self.POWEROFF:
     420              raise CuckooMachineError("Trying to stop an already stopped "
     421                                       "machine {0}".format(label))
     422
     423          # Force virtual machine shutdown.
     424          conn = self._connect()
     425          try:
     426              if not self.vms[label].isActive():
     427                  log.debug("Trying to stop an already stopped machine %s. "
     428                            "Skip", label)
     429              else:
     430                  self.vms[label].destroy()  # Machete's way!
     431          except libvirt.libvirtError as e:
     432              raise CuckooMachineError("Error stopping virtual machine "
     433                                       "{0}: {1}".format(label, e))
     434          finally:
     435              self._disconnect(conn)
     436          # Check state.
     437          self._wait_status(label, self.POWEROFF)
     438
     439 -    def shutdown(self):
     440          """Override shutdown to free libvirt handlers - they print errors."""
     441          super(LibVirtMachinery, self).shutdown()
     442
     443          # Free handlers.
     444          self.vms = None
     445
     446 -    def dump_memory(self, label, path):
     447          """Takes a memory dump.
     448          @param path: path to where to store the memory dump.
     449          """
     450          log.debug("Dumping memory for machine %s", label)
     451
     452          conn = self._connect()
     453          try:
     454              # Resolve permission issue as libvirt creates the file as
     455              # root/root in mode 0600, preventing us from reading it. This
     456              # supposedly still doesn't allow us to remove it, though..
     457              open(path, "wb").close()
     458              self.vms[label].coreDump(path, flags=libvirt.VIR_DUMP_MEMORY_ONLY)
     459          except libvirt.libvirtError as e:
     460              raise CuckooMachineError("Error dumping memory virtual machine "
     461                                       "{0}: {1}".format(label, e))
     462          finally:
     463              self._disconnect(conn)
     464
     465 -    def _status(self, label):
     466          """Gets current status of a vm.
     467          @param label: virtual machine name.
     468          @return: status string.
     469          """
     470          log.debug("Getting status for %s", label)
     471
     472          # Stetes mapping of python-libvirt.
     473          # virDomainState
     474          # VIR_DOMAIN_NOSTATE = 0
     475          # VIR_DOMAIN_RUNNING = 1
     476          # VIR_DOMAIN_BLOCKED = 2
     477          # VIR_DOMAIN_PAUSED = 3
     478          # VIR_DOMAIN_SHUTDOWN = 4
     479          # VIR_DOMAIN_SHUTOFF = 5
     480          # VIR_DOMAIN_CRASHED = 6
     481          # VIR_DOMAIN_PMSUSPENDED = 7
     482
     483          conn = self._connect()
     484          try:
     485              state = self.vms[label].state(flags=0)
     486          except libvirt.libvirtError as e:
     487              raise CuckooMachineError("Error getting status for virtual "
     488                                       "machine {0}: {1}".format(label, e))
     489          finally:
     490              self._disconnect(conn)
     491
     492          if state:
     493              if state[0] == 1:
     494                  status = self.RUNNING
     495              elif state[0] == 3:
     496                  status = self.PAUSED
     497              elif state[0] == 4 or state[0] == 5:
     498                  status = self.POWEROFF
     499              else:
     500                  status = self.ERROR
     501
     502          # Report back status.
     503          if status:
     504              self.set_status(label, status)
     505              return status
     506          else:
     507              raise CuckooMachineError("Unable to get status for "
     508                                       "{0}".format(label))
     509
     510 -    def _connect(self):
     511          """Connects to libvirt subsystem.
     512          @raise CuckooMachineError: when unable to connect to libvirt.
     513          """
     514          # Check if a connection string is available.
     515          if not self.dsn:
     516              raise CuckooMachineError("You must provide a proper "
     517                                       "connection string")
     518
     519          try:
     520              return libvirt.open(self.dsn)
     521          except libvirt.libvirtError:
     522              raise CuckooMachineError("Cannot connect to libvirt")
     523
     524 -    def _disconnect(self, conn):
     525          """Disconnects to libvirt subsystem.
     526          @raise CuckooMachineError: if cannot disconnect from libvirt.
     527          """
     528          try:
     529              conn.close()
     530          except libvirt.libvirtError:
     531              raise CuckooMachineError("Cannot disconnect from libvirt")
     532
     533 -    def _fetch_machines(self):
     534          """Fetch machines handlers.
     535          @return: dict with machine label as key and handle as value.
     536          """
     537          vms = {}
     538          for vm in self.machines():
     539              vms[vm.label] = self._lookup(vm.label)
     540          return vms
     541
     542 -    def _lookup(self, label):
     543          """Search for a virtual machine.
     544          @param conn: libvirt connection handle.
     545          @param label: virtual machine name.
     546          @raise CuckooMachineError: if virtual machine is not found.
     547          """
     548          conn = self._connect()
     549          try:
     550              vm = conn.lookupByName(label)
     551          except libvirt.libvirtError:
     552                  raise CuckooMachineError("Cannot find machine "
     553                                           "{0}".format(label))
     554          finally:
     555              self._disconnect(conn)
     556          return vm
     557
     558 -    def _list(self):
     559          """List available virtual machines.
     560          @raise CuckooMachineError: if unable to list virtual machines.
     561          """
     562          conn = self._connect()
     563          try:
     564              names = conn.listDefinedDomains()
     565          except libvirt.libvirtError:
     566              raise CuckooMachineError("Cannot list domains")
     567          finally:
     568              self._disconnect(conn)
     569          return names
     570
     571 -    def _version_check(self):
     572          """Check if libvirt release supports snapshots.
     573          @return: True or false.
     574          """
     575          if libvirt.getVersion() >= 8000:
     576              return True
     577          else:
     578              return False
     579
     580 -    def _get_snapshot(self, label):
     581          """Get current snapshot for virtual machine
     582          @param label: virtual machine name
     583          @return None or current snapshot
     584          @raise CuckooMachineError: if cannot find current snapshot or
     585                                     when there are too many snapshots available
     586          """
     587          def _extract_creation_time(node):
     588              """Extracts creation time from a KVM vm config file.
     589              @param node: config file node
     590              @return: extracted creation time
     591              """
     592              xml = ET.fromstring(node.getXMLDesc(flags=0))
     593              return xml.findtext("./creationTime")
     594
     595          snapshot = None
     596          conn = self._connect()
     597          try:
     598              vm = self.vms[label]
     599
     600              # Try to get the currrent snapshot, otherwise fallback on the latest
     601              # from config file.
     602              if vm.hasCurrentSnapshot(flags=0):
     603                  snapshot = vm.snapshotCurrent(flags=0)
     604              else:
     605                  log.debug("No current snapshot, using latest snapshot")
     606
     607                  # No current snapshot, try to get the last one from config file.
     608                  snapshot = sorted(vm.listAllSnapshots(flags=0),
     609                                    key=_extract_creation_time,
     610                                    reverse=True)[0]
     611          except libvirt.libvirtError:
     612              raise CuckooMachineError("Unable to get snapshot for "
     613                                       "virtual machine {0}".format(label))
     614          finally:
     615              self._disconnect(conn)
     616
     617          return snapshot
     618
     619 -class Processing(object):
     620      """Base abstract class for processing module."""
     621      order = 1
     622      enabled = True
     623
     624 -    def __init__(self):
     625          self.analysis_path = ""
     626          self.baseline_path = ""
     627          self.logs_path = ""
     628          self.task = None
     629          self.options = None
     630          self.results = {}
     631
     632 -    def set_options(self, options):
     633          """Set report options.
     634          @param options: report options dict.
     635          """
     636          self.options = options
     637
     638 -    def set_task(self, task):
     639          """Add task information.
     640          @param task: task dictionary.
     641          """
     642          self.task = task
     643
     644 -    def set_baseline(self, baseline_path):
     645          """Set the path to the baseline directory."""
     646          self.baseline_path = baseline_path
     647
     648 -    def set_path(self, analysis_path):
     649          """Set paths.
     650          @param analysis_path: analysis folder path.
     651          """
     652          self.analysis_path = analysis_path
     653          self.log_path = os.path.join(self.analysis_path, "analysis.log")
     654          self.cuckoolog_path = os.path.join(self.analysis_path, "cuckoo.log")
     655          self.file_path = os.path.realpath(os.path.join(self.analysis_path,
     656                                                         "binary"))
     657          self.dropped_path = os.path.join(self.analysis_path, "files")
     658          self.dropped_meta_path = os.path.join(self.analysis_path, "files.json")
     659          self.package_files = os.path.join(self.analysis_path, "package_files")
     660          self.buffer_path = os.path.join(self.analysis_path, "buffer")
     661          self.logs_path = os.path.join(self.analysis_path, "logs")
     662          self.shots_path = os.path.join(self.analysis_path, "shots")
     663          self.pcap_path = os.path.join(self.analysis_path, "dump.pcap")
     664          self.pmemory_path = os.path.join(self.analysis_path, "memory")
     665          self.memory_path = os.path.join(self.analysis_path, "memory.dmp")
     666          self.mitmout_path = os.path.join(self.analysis_path, "mitm.log")
     667          self.mitmerr_path = os.path.join(self.analysis_path, "mitm.err")
     668          self.tlsmaster_path = os.path.join(self.analysis_path, "tlsmaster.txt")
     669          self.suricata_path = os.path.join(self.analysis_path, "suricata")
     670          self.network_path = os.path.join(self.analysis_path, "network")
     671          self.taskinfo_path = os.path.join(self.analysis_path, "task.json")
     672
     673 -    def set_results(self, results):
     674          """Set the results - the fat dictionary."""
     675          self.results = results
     676
     677 -    def run(self):
     678          """Start processing.
     679          @raise NotImplementedError: this method is abstract.
     680          """
     681          raise NotImplementedError
     682
     683 -class Signature(object):
     684      """Base class for Cuckoo signatures."""
     685      name = ""
     686      description = ""
     687      severity = 1
     688      order = 1
     689      categories = []
     690      families = []
     691      authors = []
     692      references = []
     693      platform = None
     694      alert = False
     695      enabled = True
     696      minimum = None
     697      maximum = None
     698
     699      # Maximum amount of marks to record.
     700      markcount = 50
     701
     702      # Basic filters to reduce the amount of events sent to this signature.
     703      filter_apinames = []
     704      filter_categories = []
     705
     706      # If no on_call() handler is present and this field has been set, then
     707      # dispatch on a per-API basis to the accompanying API. That is, rather
     708      # than calling the generic on_call(), call, e.g., on_call_CreateFile().
     709      on_call_dispatch = False
     710
     711 -    def __init__(self, caller):
     712          """
     713          @param caller: calling object. Stores results in caller.results
     714          """
     715          self.marks = []
     716          self.matched = False
     717          self._caller = caller
     718
     719          # These are set by the caller, they represent the process identifier
     720          # and call index respectively.
     721          self.pid = None
     722          self.cid = None
     723          self.call = None
     724
     725 -    def _check_value(self, pattern, subject, regex=False, all=False):
     726          """Checks a pattern against a given subject.
     727          @param pattern: string or expression to check for.
     728          @param subject: target of the check.
     729          @param regex: boolean representing if the pattern is a regular
     730                        expression or not and therefore should be compiled.
     731          @return: boolean with the result of the check.
     732          """
     733          ret = set()
     734          if regex:
     735              exp = re.compile(pattern, re.IGNORECASE)
     736              if isinstance(subject, list):
     737                  for item in subject:
     738                      if exp.match(item):
     739                          ret.add(item)
     740              else:
     741                  if exp.match(subject):
     742                      ret.add(subject)
     743          else:
     744              if isinstance(subject, list):
     745                  for item in subject:
     746                      if item.lower() == pattern.lower():
     747                          ret.add(item)
     748              else:
     749                  if subject == pattern:
     750                      ret.add(subject)
     751
     752          # Return all elements.
     753          if all:
     754              return list(ret)
     755          # Return only the first element, if available. Otherwise return None.
     756          elif ret:
     757              return ret.pop()
     758
     759 -    def get_results(self, key=None, default=None):
     760          if key:
     761              return self._caller.results.get(key, default)
     762
     763          return self._caller.results
     764
     765 -    def get_processes(self, name=None):
     766          """Get a list of processes.
     767
     768          @param name: If set only return processes with that name.
     769          @return: List of processes or empty list
     770          """
     771          for item in self.get_results("behavior", {}).get("processes", []):
     772              if name is None or item["process_name"] == name:
     773                  yield item
     774
     775 -    def get_process_by_pid(self, pid=None):
     776          """Get a process by its process identifier.
     777
     778          @param pid: pid to search for.
     779          @return: process.
     780          """
     781          for item in self.get_results("behavior", {}).get("processes", []):
     782              if item["pid"] == pid:
     783                  return item
     784
     785 -    def get_summary(self, key=None, default=[]):
     786          """Get one or all values related to the global summary."""
     787          summary = self.get_results("behavior", {}).get("summary", {})
     788          return summary.get(key, default) if key else summary
     789
     790 -    def get_summary_generic(self, pid, actions):
     791          """Get generic info from summary.
     792
     793          @param pid: pid of the process. None for all
     794          @param actions: A list of actions to get
     795          """
     796          ret = []
     797          for process in self.get_results("behavior", {}).get("generic", []):
     798              if pid is not None and process["pid"] != pid:
     799                  continue
     800
     801              for action in actions:
     802                  if action in process["summary"]:
     803                      ret += process["summary"][action]
     804          return ret
     805
     806 -    def get_files(self, pid=None, actions=None):
     807          """Get files read, queried, or written to optionally by a
     808          specific process.
     809
     810          @param pid: the process or None for all
     811          @param actions: actions to search for. None is all
     812          @return: yields files
     813
     814          """
     815          if actions is None:
     816              actions = [
     817                  "file_opened", "file_written",
     818                  "file_read", "file_deleted",
     819                  "file_exists", "file_failed",
     820              ]
     821
     822          return self.get_summary_generic(pid, actions)
     823
     824 -    def get_dll_loaded(self, pid=None):
     825          """Get DLLs loaded by a specific process.
     826
     827          @param pid: the process or None for all
     828          @return: yields DLLs loaded
     829
     830          """
     831          return self.get_summary_generic(pid, ["dll_loaded"])
     832
     833 -    def get_keys(self, pid=None, actions=None):
     834          """Get registry keys.
     835
     836          @param pid: The pid to look in or None for all.
     837          @param actions: the actions as a list.
     838          @return: yields registry keys
     839
     840          """
     841          if actions is None:
     842              actions = [
     843                  "regkey_opened", "regkey_written",
     844                  "regkey_read", "regkey_deleted",
     845              ]
     846
     847          return self.get_summary_generic(pid, actions)
     848
     849 -    def check_file(self, pattern, regex=False, actions=None, pid=None,
     850                     all=False):
     851          """Checks for a file being opened.
     852          @param pattern: string or expression to check for.
     853          @param regex: boolean representing if the pattern is a regular
     854                        expression or not and therefore should be compiled.
     855          @param actions: a list of key actions to use.
     856          @param pid: The process id to check. If it is set to None, all
     857                      processes will be checked.
     858          @return: boolean with the result of the check.
     859          """
     860          if actions is None:
     861              actions = [
     862                  "file_opened", "file_written",
     863                  "file_read", "file_deleted",
     864                  "file_exists", "file_failed",
     865              ]
     866
     867          return self._check_value(pattern=pattern,
     868                                   subject=self.get_files(pid, actions),
     869                                   regex=regex,
     870                                   all=all)
     871
     872 -    def check_dll_loaded(self, pattern, regex=False, actions=None, pid=None,
     873                           all=False):
     874          """Checks for DLLs being loaded.
     875          @param pattern: string or expression to check for.
     876          @param regex: boolean representing if the pattern is a regular
     877                        expression or not and therefore should be compiled.
     878          @param pid: The process id to check. If it is set to None, all
     879                      processes will be checked.
     880          @return: boolean with the result of the check.
     881          """
     882          return self._check_value(pattern=pattern,
     883                                   subject=self.get_dll_loaded(pid),
     884                                   regex=regex,
     885                                   all=all)
     886
     887 -    def check_key(self, pattern, regex=False, actions=None, pid=None,
     888                    all=False):
     889          """Checks for a registry key being accessed.
     890          @param pattern: string or expression to check for.
     891          @param regex: boolean representing if the pattern is a regular
     892                        expression or not and therefore should be compiled.
     893          @param actions: a list of key actions to use.
     894          @param pid: The process id to check. If it is set to None, all
     895                      processes will be checked.
     896          @return: boolean with the result of the check.
     897          """
     898          if actions is None:
     899              actions = [
     900                  "regkey_written", "regkey_opened",
     901                  "regkey_read", "regkey_deleted",
     902              ]
     903
     904          return self._check_value(pattern=pattern,
     905                                   subject=self.get_keys(pid, actions),
     906                                   regex=regex,
     907                                   all=all)
     908
     909 -    def get_mutexes(self, pid=None):
     910          """
     911          @param pid: Pid to filter for
     912          @return:List of mutexes
     913          """
     914          return self.get_summary_generic(pid, ["mutex"])
     915
     916 -    def check_mutex(self, pattern, regex=False, all=False):
     917          """Checks for a mutex being opened.
     918          @param pattern: string or expression to check for.
     919          @param regex: boolean representing if the pattern is a regular
     920                        expression or not and therefore should be compiled.
     921          @return: boolean with the result of the check.
     922          """
     923          return self._check_value(pattern=pattern,
     924                                   subject=self.get_mutexes(),
     925                                   regex=regex,
     926                                   all=all)
     927
     928 -    def get_command_lines(self):
     929          """Retrieves all command lines used."""
     930          return self.get_summary("command_line")
     931
     932 -    def get_wmi_queries(self):
     933          """Retrieves all executed WMI queries."""
     934          return self.get_summary("wmi_query")
     935
     936 -    def get_net_generic(self, subtype):
     937          """Generic getting network data.
     938
     939          @param subtype: subtype string to search for.
     940          """
     941          return self.get_results("network", {}).get(subtype, [])
     942
     943 -    def get_net_hosts(self):
     944          """Returns a list of all hosts."""
     945          return self.get_net_generic("hosts")
     946
     947 -    def get_net_domains(self):
     948          """Returns a list of all domains."""
     949          return self.get_net_generic("domains")
     950
     951 -    def get_net_http(self):
     952          """Returns a list of all http data."""
     953          return self.get_net_generic("http")
     954
     955 -    def get_net_http_ex(self):
     956          """Returns a list of all http data."""
     957          return \
     958              self.get_net_generic("http_ex") + self.get_net_generic("https_ex")
     959
     960 -    def get_net_udp(self):
     961          """Returns a list of all udp data."""
     962          return self.get_net_generic("udp")
     963
     964 -    def get_net_icmp(self):
     965          """Returns a list of all icmp data."""
     966          return self.get_net_generic("icmp")
     967
     968 -    def get_net_irc(self):
     969          """Returns a list of all irc data."""
     970          return self.get_net_generic("irc")
     971
     972 -    def get_net_smtp(self):
     973          """Returns a list of all smtp data."""
     974          return self.get_net_generic("smtp")
     975
     976 -    def get_virustotal(self):
     977          """Returns the information retrieved from virustotal."""
     978          return self.get_results("virustotal", {})
     979
     980 -    def get_volatility(self, module=None):
     981          """Returns the data that belongs to the given module."""
     982          volatility = self.get_results("memory", {})
     983          return volatility if module is None else volatility.get(module, {})
     984
     985 -    def get_apkinfo(self, section=None, default={}):
     986          """Returns the apkinfo results for this analysis."""
     987          apkinfo = self.get_results("apkinfo", {})
     988          return apkinfo if section is None else apkinfo.get(section, default)
     989
     990 -    def get_droidmon(self, section=None, default={}):
     991          """Returns the droidmon results for this analysis."""
     992          droidmon = self.get_results("droidmon", {})
     993          return droidmon if section is None else droidmon.get(section, default)
     994
     995 -    def get_googleplay(self, section=None, default={}):
     996          """Returns the Google Play results for this analysis."""
     997          googleplay = self.get_results("googleplay", {})
     998          return googleplay if section is None else googleplay.get(section, default)
     999
    1000 -    def check_ip(self, pattern, regex=False, all=False):
    1001          """Checks for an IP address being contacted.
    1002          @param pattern: string or expression to check for.
    1003          @param regex: boolean representing if the pattern is a regular
    1004                        expression or not and therefore should be compiled.
    1005          @return: boolean with the result of the check.
    1006          """
    1007          return self._check_value(pattern=pattern,
    1008                                   subject=self.get_net_hosts(),
    1009                                   regex=regex,
    1010                                   all=all)
    1011
    1012 -    def check_domain(self, pattern, regex=False, all=False):
    1013          """Checks for a domain being contacted.
    1014          @param pattern: string or expression to check for.
    1015          @param regex: boolean representing if the pattern is a regular
    1016                        expression or not and therefore should be compiled.
    1017          @return: boolean with the result of the check.
    1018          """
    1019          domains = set()
    1020          for item in self.get_net_domains():
    1021              domains.add(item["domain"])
    1022
    1023          return self._check_value(pattern=pattern,
    1024                                   subject=list(domains),
    1025                                   regex=regex,
    1026                                   all=all)
    1027
    1028 -    def check_url(self, pattern, regex=False, all=False):
    1029          """Checks for a URL being contacted.
    1030          @param pattern: string or expression to check for.
    1031          @param regex: boolean representing if the pattern is a regular
    1032                        expression or not and therefore should be compiled.
    1033          @return: boolean with the result of the check.
    1034          """
    1035          urls = set()
    1036          for item in self.get_net_http():
    1037              urls.add(item["uri"])
    1038
    1039          return self._check_value(pattern=pattern,
    1040                                   subject=list(urls),
    1041                                   regex=regex,
    1042                                   all=all)
    1043
    1044 -    def init(self):
    1045          """Allow signatures to initialize themselves."""
    1046
    1047 -    def mark_call(self, *args, **kwargs):
    1048          """Mark the current call as explanation as to why this signature
    1049          matched."""
    1050          mark = {
    1051              "type": "call",
    1052              "pid": self.pid,
    1053              "cid": self.cid,
    1054              "call": self.call,
    1055          }
    1056
    1057          if args or kwargs:
    1058              log.warning(
    1059                  "You have provided extra arguments to the mark_call() method "
    1060                  "which no longer supports doing so. Please report explicit "
    1061                  "IOCs through mark_ioc()."
    1062              )
    1063
    1064          self.marks.append(mark)
    1065
    1066 -    def mark_ioc(self, category, ioc, description=None):
    1067          """Mark an IOC as explanation as to why the current signature
    1068          matched."""
    1069          mark = {
    1070              "type": "ioc",
    1071              "category": category,
    1072              "ioc": ioc,
    1073              "description": description,
    1074          }
    1075
    1076          # Prevent duplicates.
    1077          if mark not in self.marks:
    1078              self.marks.append(mark)
    1079
    1080 -    def mark_vol(self, plugin, **kwargs):
    1081          """Mark output of a Volatility plugin as explanation as to why the
    1082          current signature matched."""
    1083          mark = {
    1084              "type": "volatility",
    1085              "plugin": plugin,
    1086          }
    1087          mark.update(kwargs)
    1088          self.marks.append(mark)
    1089
    1090 -    def mark(self, **kwargs):
    1091          """Mark arbitrary data."""
    1092          mark = {
    1093              "type": "generic",
    1094          }
    1095          mark.update(kwargs)
    1096          self.marks.append(mark)
    1097
    1098 -    def has_marks(self, count=None):
    1099          """Returns true if this signature has one or more marks."""
    1100          if count is not None:
    1101              return len(self.marks) >= count
    1102          return not not self.marks
    1103
    1104 -    def on_call(self, call, process):
    1105          """Notify signature about API call. Return value determines
    1106          if this signature is done or could still match.
    1107
    1108          Only called if signature is "active".
    1109
    1110          @param call: logged API call.
    1111          @param process: proc object.
    1112          """
    1113          # Dispatch this call to a per-API specific handler.
    1114          if self.on_call_dispatch:
    1115              return getattr(self, "on_call_%s" % call["api"])(call, process)
    1116
    1117          raise NotImplementedError
    1118
    1119 -    def on_signature(self, signature):
    1120          """Event yielded when another signatures has matched. Some signatures
    1121          only take effect when one or more other signatures have matched as
    1122          well.
    1123
    1124          @param signature: The signature that just matched
    1125          """
    1126
    1127 -    def on_process(self, process):
    1128          """Called on process change.
    1129
    1130          Can be used for cleanup of flags, re-activation of the signature, etc.
    1131
    1132          @param process: dictionary describing this process
    1133          """
    1134
    1135 -    def on_complete(self):
    1136          """Signature is notified when all API calls have been processed."""
    1137
    1138 -    def results(self):
    1139          """Turn this signature into actionable results."""
    1140          return dict(name=self.name,
    1141                      description=self.description,
    1142                      severity=self.severity,
    1143                      families=self.families,
    1144                      references=self.references,
    1145                      marks=self.marks[:self.markcount],
    1146                      markcount=len(self.marks))
    1147
    1148 -class Report(object):
    1149      """Base abstract class for reporting module."""
    1150      order = 1
    1151
    1152 -    def __init__(self):
    1153          self.analysis_path = ""
    1154          self.reports_path = ""
    1155          self.task = None
    1156          self.options = None
    1157
    1158 -    def _get_analysis_path(self, subpath):
    1159          return os.path.join(self.analysis_path, subpath)
    1160
    1161 -    def set_path(self, analysis_path):
    1162          """Set analysis folder path.
    1163          @param analysis_path: analysis folder path.
    1164          """
    1165          self.analysis_path = analysis_path
    1166          self.conf_path = self._get_analysis_path("analysis.conf")
    1167          self.file_path = os.path.realpath(self._get_analysis_path("binary"))
    1168          self.reports_path = self._get_analysis_path("reports")
    1169          self.shots_path = self._get_analysis_path("shots")
    1170          self.pcap_path = self._get_analysis_path("dump.pcap")
    1171
    1172          try:
    1173              create_folder(folder=self.reports_path)
    1174          except CuckooOperationalError as e:
    1175              raise CuckooReportError(e)
    1176
    1177 -    def set_options(self, options):
    1178          """Set report options.
    1179          @param options: report options dict.
    1180          """
    1181          self.options = options
    1182
    1183 -    def set_task(self, task):
    1184          """Add task information.
    1185          @param task: task dictionary.
    1186          """
    1187          self.task = task
    1188
    1189 -    def run(self):
    1190          """Start report processing.
    1191          @raise NotImplementedError: this method is abstract.
    1192          """
    1193          raise NotImplementedError
    1194
    1195 -class BehaviorHandler(object):
    1196      """Base class for behavior handlers inside of BehaviorAnalysis."""
    1197      key = "undefined"
    1198
    1199      # Behavior event types this handler is interested in.
    1200      event_types = []
    1201
    1202 -    def __init__(self, behavior_analysis):
    1203          self.analysis = behavior_analysis
    1204
    1205 -    def handles_path(self, logpath):
    1206          """Needs to return True for the log files this handler wants to
    1207          process."""
    1208          return False
    1209
    1210 -    def parse(self, logpath):
    1211          """Called after handles_path succeeded, should generate behavior
    1212          events."""
    1213          raise NotImplementedError
    1214
    1215 -    def handle_event(self, event):
    1216          """Handle an event that gets passed down the stack."""
    1217          raise NotImplementedError
    1218
    1219 -    def run(self):
    1220          """Return the handler specific structure, gets placed into
    1221          behavior[self.key]."""
    1222          raise NotImplementedError
    1223
    1224 -class ProtocolHandler(object):
    1225      """Abstract class for protocol handlers coming out of the analysis."""
    1226 -    def __init__(self, handler, version=None):
    1227          self.handler = handler
    1228          self.version = version
    1229
    1230 -    def init(self):
    1231          pass
    1232
    1233 -    def close(self):
    1234          pass
    1235