lib.cuckoo.core.startup

Package lib :: Package cuckoo :: Package core :: Module startup |
[hide private]
[frames] | no frames]

# Source Code for Module lib.cuckoo.core.startup

      1  # Copyright (C) 2010-2013 Claudio Guarnieri.
      2  # Copyright (C) 2014-2016 Cuckoo Foundation.
      3  # This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org
      4  # See the file 'docs/LICENSE' for copying permission.
      5
      6  import httplib
      7  import os
      8  import shutil
      9  import sys
     10  import json
     11  import socket
     12  import urllib
     13  import urllib2
     14  import logging
     15  import logging.handlers
     16
     17  from distutils.version import LooseVersion
     18
     19  from lib.cuckoo.common.colors import red, green, yellow
     20  from lib.cuckoo.common.config import Config
     21  from lib.cuckoo.common.constants import CUCKOO_ROOT, CUCKOO_VERSION
     22  from lib.cuckoo.common.exceptions import CuckooStartupError, CuckooDatabaseError
     23  from lib.cuckoo.common.exceptions import CuckooOperationalError
     24  from lib.cuckoo.common.utils import create_folders
     25  from lib.cuckoo.core.database import Database, TASK_RUNNING
     26  from lib.cuckoo.core.database import TASK_FAILED_ANALYSIS, TASK_PENDING
     27  from lib.cuckoo.core.log import DatabaseHandler, ConsoleHandler, TaskHandler
     28  from lib.cuckoo.core.plugins import import_plugin, import_package, list_plugins
     29  from lib.cuckoo.core.rooter import rooter, vpns
     30
     31  try:
     32      import pwd
     33      HAVE_PWD = True
     34  except ImportError:
     35      HAVE_PWD = False
     36
     37  log = logging.getLogger()
     38
     39 -def check_python_version():
     40      """Checks if Python version is supported by Cuckoo.
     41      @raise CuckooStartupError: if version is not supported.
     42      """
     43      if sys.version_info[:2] != (2, 7):
     44          raise CuckooStartupError("You are running an incompatible version "
     45                                   "of Python, please use 2.7")
     46
     47
     48 -def check_working_directory():
     49      """Checks if working directories are ready.
     50      @raise CuckooStartupError: if directories are not properly configured.
     51      """
     52      if not os.path.exists(CUCKOO_ROOT):
     53          raise CuckooStartupError("You specified a non-existing root "
     54                                   "directory: {0}".format(CUCKOO_ROOT))
     55
     56      cwd = os.path.join(os.getcwd(), "cuckoo.py")
     57      if not os.path.exists(cwd):
     58          raise CuckooStartupError("You are not running Cuckoo from it's "
     59                                   "root directory")
     60
     61
     62 -def check_configs():
     63      """Checks if config files exist.
     64      @raise CuckooStartupError: if config files do not exist.
     65      """
     66      configs = (
     67          "auxiliary.conf", "avd.conf", "cuckoo.conf", "esx.conf", "kvm.conf",
     68          "memory.conf", "physical.conf", "processing.conf", "qemu.conf",
     69          "reporting.conf", "virtualbox.conf", "vmware.conf", "vpn.conf",
     70          "vsphere.conf", "xenserver.conf",
     71      )
     72
     73      for config in [os.path.join(CUCKOO_ROOT, "conf", f) for f in configs]:
     74          if not os.path.exists(config):
     75              raise CuckooStartupError("Config file does not exist at "
     76                                       "path: {0}".format(config))
     77
     78      return True
     79
     80 -def create_structure():
     81      """Creates Cuckoo directories."""
     82      folders = [
     83          "log",
     84          "storage",
     85          os.path.join("storage", "analyses"),
     86          os.path.join("storage", "binaries"),
     87          os.path.join("storage", "baseline"),
     88      ]
     89
     90      try:
     91          create_folders(root=CUCKOO_ROOT, folders=folders)
     92      except CuckooOperationalError as e:
     93          raise CuckooStartupError(e)
     94
     95 -def check_version():
     96      """Checks version of Cuckoo."""
     97      cfg = Config()
     98
     99      if not cfg.cuckoo.version_check:
    100          return
    101
    102      print(" Checking for updates...")
    103
    104      url = "http://api.cuckoosandbox.org/checkversion.php"
    105      data = urllib.urlencode({"version": CUCKOO_VERSION})
    106
    107      try:
    108          request = urllib2.Request(url, data)
    109          response = urllib2.urlopen(request)
    110      except (urllib2.URLError, urllib2.HTTPError, httplib.BadStatusLine):
    111          print(red(" Failed! ") + "Unable to establish connection.\n")
    112          return
    113
    114      try:
    115          response_data = json.loads(response.read())
    116      except ValueError:
    117          print(red(" Failed! ") + "Invalid response.\n")
    118          return
    119
    120      stable_version = response_data["current"]
    121
    122      if CUCKOO_VERSION.endswith("-dev"):
    123          print(yellow(" You are running a development version! Current stable is {}.".format(
    124              stable_version)))
    125      else:
    126          if LooseVersion(CUCKOO_VERSION) < LooseVersion(stable_version):
    127              msg = "Cuckoo Sandbox version {} is available now.".format(
    128                  stable_version)
    129
    130              print(red(" Outdated! ") + msg)
    131          else:
    132              print(green(" Good! ") + "You have the latest version "
    133                                       "available.\n")
    134
    135 -def init_logging():
    136      """Initializes logging."""
    137      formatter = logging.Formatter("%(asctime)s [%(name)s] %(levelname)s: %(message)s")
    138
    139      fh = logging.handlers.WatchedFileHandler(os.path.join(CUCKOO_ROOT, "log", "cuckoo.log"))
    140      fh.setFormatter(formatter)
    141      log.addHandler(fh)
    142
    143      ch = ConsoleHandler()
    144      ch.setFormatter(formatter)
    145      log.addHandler(ch)
    146
    147      dh = DatabaseHandler()
    148      dh.setLevel(logging.ERROR)
    149      log.addHandler(dh)
    150
    151      th = TaskHandler()
    152      th.setFormatter(formatter)
    153      log.addHandler(th)
    154
    155      log.setLevel(logging.INFO)
    156
    157 -def init_console_logging():
    158      """Initializes logging only to console."""
    159      formatter = logging.Formatter("%(asctime)s [%(name)s] %(levelname)s: %(message)s")
    160
    161      ch = ConsoleHandler()
    162      ch.setFormatter(formatter)
    163      log.addHandler(ch)
    164
    165      log.setLevel(logging.INFO)
    166
    167 -def init_tasks():
    168      """Check tasks and reschedule uncompleted ones."""
    169      db = Database()
    170      cfg = Config()
    171
    172      log.debug("Checking for locked tasks..")
    173      for task in db.list_tasks(status=TASK_RUNNING):
    174          if cfg.cuckoo.reschedule:
    175              task_id = db.reschedule(task.id)
    176              log.info(
    177                  "Rescheduled task with ID %s and target %s: task #%s",
    178                  task.id, task.target, task_id
    179              )
    180          else:
    181              db.set_status(task.id, TASK_FAILED_ANALYSIS)
    182              log.info("Updated running task ID {0} status to failed_analysis".format(task.id))
    183
    184      log.debug("Checking for pending service tasks..")
    185      for task in db.list_tasks(status=TASK_PENDING, category="service"):
    186          db.set_status(task.id, TASK_FAILED_ANALYSIS)
    187
    188 -def delete_file(*rel_path):
    189      filepath = os.path.join(CUCKOO_ROOT, *rel_path)
    190      if not os.path.exists(filepath):
    191          return
    192
    193      try:
    194          os.unlink(filepath)
    195      except Exception as e:
    196          log.warning(
    197              "Unable to remove old %s leftover file from before you updated "
    198              "your Cuckoo setup to the latest version: %s.",
    199              os.path.join(*rel_path), e
    200          )
    201
    202 -def init_modules(machinery=True):
    203      """Initializes plugins."""
    204      log.debug("Importing modules...")
    205
    206      # Import all auxiliary modules.
    207      import modules.auxiliary
    208      import_package(modules.auxiliary)
    209
    210      # Import all processing modules.
    211      import modules.processing
    212      import_package(modules.processing)
    213
    214      # Import all signatures.
    215      import modules.signatures
    216      import_package(modules.signatures)
    217
    218      delete_file("modules", "reporting", "maec40.pyc")
    219      delete_file("modules", "reporting", "maec41.pyc")
    220      delete_file("modules", "reporting", "mmdef.pyc")
    221
    222      # Import all reporting modules.
    223      import modules.reporting
    224      import_package(modules.reporting)
    225
    226      # Import machine manager.
    227      if machinery:
    228          import_plugin("modules.machinery." + Config().cuckoo.machinery)
    229
    230      for category, entries in list_plugins().items():
    231          log.debug("Imported \"%s\" modules:", category)
    232
    233          for entry in entries:
    234              if entry == entries[-1]:
    235                  log.debug("\t `-- %s", entry.__name__)
    236              else:
    237                  log.debug("\t |-- %s", entry.__name__)
    238
    239 -def init_yara():
    240      """Generates index for yara signatures."""
    241      log.debug("Initializing Yara...")
    242
    243      # Generate root directory for yara rules.
    244      yara_root = os.path.join(CUCKOO_ROOT, "data", "yara")
    245
    246      # We divide yara rules in three categories.
    247      categories = ["binaries", "urls", "memory"]
    248      generated = []
    249      # Loop through all categories.
    250      for category in categories:
    251          # Check if there is a directory for the given category.
    252          category_root = os.path.join(yara_root, category)
    253          if not os.path.exists(category_root):
    254              continue
    255
    256          # Check if the directory contains any rules.
    257          signatures = []
    258          for entry in os.listdir(category_root):
    259              if entry.endswith(".yara") or entry.endswith(".yar"):
    260                  signatures.append(os.path.join(category_root, entry))
    261
    262          if not signatures:
    263              continue
    264
    265          # Generate path for the category's index file.
    266          index_name = "index_{0}.yar".format(category)
    267          index_path = os.path.join(yara_root, index_name)
    268
    269          # Create index file and populate it.
    270          with open(index_path, "w") as index_handle:
    271              for signature in signatures:
    272                  index_handle.write("include \"{0}\"\n".format(signature))
    273
    274          generated.append(index_name)
    275
    276      for entry in generated:
    277          if entry == generated[-1]:
    278              log.debug("\t `-- %s", entry)
    279          else:
    280              log.debug("\t |-- %s", entry)
    281
    282 -def init_binaries():
    283      """Inform the user about the need to periodically look for new analyzer
    284      binaries. These include the Windows monitor etc."""
    285      dirpath = os.path.join(CUCKOO_ROOT, "data", "monitor", "latest")
    286
    287      # Checks whether the "latest" symlink is available as well as whether
    288      # it points to an existing directory.
    289      if not os.path.exists(dirpath):
    290          raise CuckooStartupError(
    291              "The binaries used for Windows analysis are updated regularly, "
    292              "independently from the release line. It appears that you're "
    293              "not up-to-date. This can happen when you've just installed "
    294              "Cuckoo or when you've updated your Cuckoo version by pulling "
    295              "the latest changes from our Git repository. In order to get "
    296              "up-to-date, please run the following "
    297              "command: `./utils/community.py -wafb monitor` or "
    298              "`./utils/community.py -waf` if you'd also like to download "
    299              "over 300 Cuckoo signatures."
    300          )
    301
    302      # If "latest" is a file and not a symbolic link, check if it's destination
    303      # directory is available.
    304      if os.path.isfile(dirpath):
    305          monitor = os.path.basename(open(dirpath, "rb").read().strip())
    306          dirpath = os.path.join(CUCKOO_ROOT, "data", "monitor", monitor)
    307      else:
    308          dirpath = None
    309
    310      if dirpath and not os.path.isdir(dirpath):
    311          raise CuckooStartupError(
    312              "The binaries used for Windows analysis are updated regularly, "
    313              "independently from the release line. It appears that you're "
    314              "not up-to-date. This can happen when you've just installed "
    315              "Cuckoo or when you've updated your Cuckoo version by pulling "
    316              "the latest changes from our Git repository. In order to get "
    317              "up-to-date, please run the following "
    318              "command: `./utils/community.py -wafb monitor` or "
    319              "`./utils/community.py -waf` if you'd also like to download "
    320              "over 300 Cuckoo signatures."
    321          )
    322
    323 -def init_rooter():
    324      """If required, check whether the rooter is running and whether we can
    325      connect to it."""
    326      cuckoo = Config()
    327
    328      # The default configuration doesn't require the rooter to be ran.
    329      if not Config("vpn").vpn.enabled and cuckoo.routing.route == "none":
    330          return
    331
    332      cuckoo = Config()
    333      s = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
    334
    335      try:
    336          s.connect(cuckoo.cuckoo.rooter)
    337      except socket.error as e:
    338          if e.strerror == "No such file or directory":
    339              raise CuckooStartupError(
    340                  "The rooter is required but it is either not running or it "
    341                  "has been configured to a different Unix socket path. "
    342                  "(In order to disable the use of rooter, please set route "
    343                  "and internet to none in cuckoo.conf and enabled to no in "
    344                  "vpn.conf)."
    345              )
    346
    347          if e.strerror == "Connection refused":
    348              raise CuckooStartupError(
    349                  "The rooter is required but we can't connect to it as the "
    350                  "rooter is not actually running. "
    351                  "(In order to disable the use of rooter, please set route "
    352                  "and internet to none in cuckoo.conf and enabled to no in "
    353                  "vpn.conf)."
    354              )
    355
    356          if e.strerror == "Permission denied":
    357              raise CuckooStartupError(
    358                  "The rooter is required but we can't connect to it due to "
    359                  "incorrect permissions. Did you assign it the correct group? "
    360                  "(In order to disable the use of rooter, please set route "
    361                  "and internet to none in cuckoo.conf and enabled to no in "
    362                  "vpn.conf)."
    363              )
    364
    365          raise CuckooStartupError("Unknown rooter error: %s" % e)
    366
    367      # Do not forward any packets unless we have explicitly stated so.
    368      rooter("forward_drop")
    369
    370 -def init_routing():
    371      """Initialize and check whether the routing information is correct."""
    372      cuckoo = Config()
    373      vpn = Config("vpn")
    374
    375      # Check whether all VPNs exist if configured and make their configuration
    376      # available through the vpns variable. Also enable NAT on each interface.
    377      if vpn.vpn.enabled:
    378          for name in vpn.vpn.vpns.split(","):
    379              name = name.strip()
    380              if not name:
    381                  continue
    382
    383              if not hasattr(vpn, name):
    384                  raise CuckooStartupError(
    385                      "Could not find VPN configuration for %s" % name
    386                  )
    387
    388              entry = vpn.get(name)
    389
    390              if not rooter("nic_available", entry.interface):
    391                  raise CuckooStartupError(
    392                      "The network interface that has been configured for "
    393                      "VPN %s is not available." % entry.name
    394                  )
    395
    396              if not rooter("rt_available", entry.rt_table):
    397                  raise CuckooStartupError(
    398                      "The routing table that has been configured for "
    399                      "VPN %s is not available." % entry.name
    400                  )
    401
    402              vpns[entry.name] = entry
    403
    404              # Disable & enable NAT on this network interface. Disable it just
    405              # in case we still had the same rule from a previous run.
    406              rooter("disable_nat", entry.interface)
    407              rooter("enable_nat", entry.interface)
    408
    409              # Populate routing table with entries from main routing table.
    410              if cuckoo.routing.auto_rt:
    411                  rooter("flush_rttable", entry.rt_table)
    412                  rooter("init_rttable", entry.rt_table, entry.interface)
    413
    414      # Check whether the default VPN exists if specified.
    415      if cuckoo.routing.route not in ("none", "internet"):
    416          if not vpn.vpn.enabled:
    417              raise CuckooStartupError(
    418                  "A VPN has been configured as default routing interface for "
    419                  "VMs, but VPNs have not been enabled in vpn.conf"
    420              )
    421
    422          if cuckoo.routing.route not in vpns:
    423              raise CuckooStartupError(
    424                  "The VPN defined as default routing target has not been "
    425                  "configured in vpn.conf."
    426              )
    427
    428      # Check whether the dirty line exists if it has been defined.
    429      if cuckoo.routing.internet != "none":
    430          if not rooter("nic_available", cuckoo.routing.internet):
    431              raise CuckooStartupError(
    432                  "The network interface that has been configured as dirty "
    433                  "line is not available."
    434              )
    435
    436          if not rooter("rt_available", cuckoo.routing.rt_table):
    437              raise CuckooStartupError(
    438                  "The routing table that has been configured for dirty "
    439                  "line interface is not available."
    440              )
    441
    442          # Disable & enable NAT on this network interface. Disable it just
    443          # in case we still had the same rule from a previous run.
    444          rooter("disable_nat", cuckoo.routing.internet)
    445          rooter("enable_nat", cuckoo.routing.internet)
    446
    447          # Populate routing table with entries from main routing table.
    448          if cuckoo.routing.auto_rt:
    449              rooter("flush_rttable", cuckoo.routing.rt_table)
    450              rooter("init_rttable", cuckoo.routing.rt_table,
    451                     cuckoo.routing.internet)
    452
    453 -def cuckoo_clean():
    454      """Clean up cuckoo setup.
    455      It deletes logs, all stored data from file system and configured
    456      databases (SQL and MongoDB).
    457      """
    458      # Init logging.
    459      # This need to init a console logger handler, because the standard
    460      # logger (init_logging()) logs to a file which will be deleted.
    461      create_structure()
    462      init_console_logging()
    463
    464      # Initialize the database connection.
    465      try:
    466          db = Database(schema_check=False)
    467      except CuckooDatabaseError as e:
    468          # If something is screwed due to incorrect database migrations or bad
    469          # database SqlAlchemy would be unable to connect and operate.
    470          log.warning("Error connecting to database: it is suggested to check "
    471                      "the connectivity, apply all migrations if needed or purge "
    472                      "it manually. Error description: %s", e)
    473      else:
    474          # Drop all tables.
    475          db.drop()
    476
    477      # Check if MongoDB reporting is enabled and drop that if it is.
    478      cfg = Config("reporting")
    479      if cfg.mongodb and cfg.mongodb.enabled:
    480          from pymongo import MongoClient
    481          host = cfg.mongodb.get("host", "127.0.0.1")
    482          port = cfg.mongodb.get("port", 27017)
    483          mdb = cfg.mongodb.get("db", "cuckoo")
    484          try:
    485              conn = MongoClient(host, port)
    486              conn.drop_database(mdb)
    487              conn.close()
    488          except:
    489              log.warning("Unable to drop MongoDB database: %s", mdb)
    490
    491      # Paths to clean.
    492      paths = [
    493          os.path.join(CUCKOO_ROOT, "db"),
    494          os.path.join(CUCKOO_ROOT, "log"),
    495          os.path.join(CUCKOO_ROOT, "storage"),
    496      ]
    497
    498      # Delete various directories.
    499      for path in paths:
    500          if os.path.isdir(path):
    501              try:
    502                  shutil.rmtree(path)
    503              except (IOError, OSError) as e:
    504                  log.warning("Error removing directory %s: %s", path, e)
    505
    506      # Delete all compiled Python objects ("*.pyc").
    507      for dirpath, dirnames, filenames in os.walk(CUCKOO_ROOT):
    508          for fname in filenames:
    509              if not fname.endswith(".pyc"):
    510                  continue
    511
    512              # We don't want to delete the Android's Agent .pyc files (as we
    513              # don't ship the original .py files and thus they're critical).
    514              if "agent/android/python_agent" in dirpath.replace("\\", "/"):
    515                  continue
    516
    517              path = os.path.join(dirpath, fname)
    518
    519              try:
    520                  os.unlink(path)
    521              except (IOError, OSError) as e:
    522                  log.warning("Error removing file %s: %s", path, e)
    523
    524 -def drop_privileges(username):
    525      """Drops privileges to selected user.
    526      @param username: drop privileges to this username
    527      """
    528      if not HAVE_PWD:
    529          sys.exit("Unable to import pwd required for dropping "
    530                   "privileges (`pip install pwd`)")
    531
    532      try:
    533          user = pwd.getpwnam(username)
    534          os.setgroups((user.pw_gid,))
    535          os.setgid(user.pw_gid)
    536          os.setuid(user.pw_uid)
    537          os.putenv("HOME", user.pw_dir)
    538      except KeyError:
    539          sys.exit("Invalid user specified to drop privileges to: %s" % user)
    540      except OSError as e:
    541          sys.exit("Failed to drop privileges to %s: %s" % (username, e))
    542