#@+leo-ver=5-thin
#@+node:ekr.20031218072017.3018: * @file leoFileCommands.py
'''Classes relating to reading and writing .leo files.'''
#@+<< imports >>
#@+node:ekr.20050405141130: ** << imports >> (leoFileCommands)
import leo.core.leoGlobals as g
# if g.app and g.app.use_psyco:
    # # g.pr("enabled psyco classes",__file__)
    # try: from psyco.classes import *
    # except ImportError: pass
import leo.core.leoNodes as leoNodes
import binascii
import difflib
import time
if g.isPython3:
    import io # Python 3.x
    StringIO = io.StringIO
    BytesIO = io.BytesIO
else:
    import cStringIO # Python 2.x
    StringIO = cStringIO.StringIO
import os
import pickle
import string
import sys
import tempfile
import zipfile
try:
    # IronPython has problems with this.
    import xml.sax
    import xml.sax.saxutils
except Exception:
    pass
# The following is sometimes used.
# import time
#@-<< imports >>
#@+<< define exception classes >>
#@+node:ekr.20060918164811: ** << define exception classes >>
class BadLeoFile(Exception):

    def __init__(self, message):
        self.message = message
        Exception.__init__(self, message) # Init the base class.

    def __str__(self):
        return "Bad Leo File:" + self.message

class InvalidPaste(Exception):
    pass
#@-<< define exception classes >>
if sys.platform != 'cli':
    #@+<< define sax classes >>
    #@+node:ekr.20060919145406: ** << define sax classes >>
    #@+others
    #@+node:ekr.20060919110638.19: *3* class SaxContentHandler (XMLGenerator)
    class SaxContentHandler(xml.sax.saxutils.XMLGenerator):
        '''A sax content handler class that reads Leo files.'''
        #@+others
        #@+node:ekr.20060919110638.20: *4*  sax.__init__ & helpers
        def __init__(self, c, fileName, silent, inClipboard):
            '''Ctor for SaxContentHandler class.'''
            self.c = c
            self.fileName = fileName
            self.silent = silent
            self.inClipboard = inClipboard
            out = sys.stdout if sys.stdout else g.fileLikeObject()
                # Fix the exceedingly strange problem with Python 3.x and pythonw.exe.
                # The sax ctor throws an exception in Python 3.x if sys.stdout is None.
                # The workaround is use a disposable output stream in that case.
            xml.sax.saxutils.XMLGenerator.__init__(self, out=out)
                # Init the base class.
            #@+<< define dispatch dict >>
            #@+node:ekr.20060919110638.21: *5* << define dispatch dict >>
            #@@nobeautify

            # There is no need for an 'end' method if all info is carried in attributes.
            self.dispatchDict = {
                'change_string':               (None,None),
                'find_panel_settings':         (None,None),
                'find_string':                 (None,None),
                'globals':                     (self.startGlobals,None),
                'global_log_window_position':  (None,None), # The position of the log window is no longer used.
                'global_window_position':      (self.startWinPos,None),
                'leo_file':                    (None,None),
                'leo_header':                  (self.startLeoHeader,None),
                'preferences':                 (None,None),
                't':                           (self.startTnode,self.endTnode),
                'tnodes':                      (None,None),
                'v':                           (self.startVnode,self.endVnode),
                'vh':                          (self.startVH,self.endVH),
                'vnodes':                      (self.startVnodes,None), # Causes window to appear.
            }
            #@-<< define dispatch dict >>
            self.printElements = [] # 'all', 'v'
            # Global attributes of the .leo file...
            # self.body_outline_ratio = '0.5'
            self.global_window_position = {}
            self.encoding = 'utf-8'
            # Semantics...
            self.content = None
            self.elementStack = []
            self.errors = 0
            self.tnxToListDict = {}
                # Keys are tnx's (strings)
                # Values are *lists* of SaxNodeClass objects.
            self.level = 0
            self.node = None
            self.nodeList = [] # List of SaxNodeClass objects with the present VNode.
            self.nodeStack = []
            self.ratio = self.secondary_ratio = 0.5
            self.rootNode = None # a sax node.
            self.trace = False # True and g.unitTesting
        #@+node:ekr.20060919110638.29: *4*  sax.Do nothing
        def endElementNS(self, unused_name, unused_qname):
            g.trace(unused_name)

        def endDocument(self):
            pass

        def ignorableWhitespace(self, unused_whitespace):
            pass

        def skippedEntity(self, name):
            g.trace(name)

        def startElementNS(self, unused_name, unused_qname, unused_attrs):
            g.trace(unused_name)

        def startDocument(self):
            pass
        #@+node:ekr.20060919134313: *4*  sax.Utils
        #@+node:ekr.20060919110638.23: *5* attrsToList
        def attrsToList(self, attrs):
            '''Convert the attributes to a list of g.Bunches.

            attrs: an Attributes item passed to startElement.'''
            return [
                g.Bunch(name=name, val=attrs.getValue(name))
                    for name in attrs.getNames()]
        #@+node:ekr.20060919110638.26: *5* error
        def error(self, message):
            g.pr('\n\nXML error: %s\n' % (message))
            self.errors += 1
        #@+node:ekr.20060919110638.27: *5* inElement
        def inElement(self, name):
            return self.elementStack and name in self.elementStack
        #@+node:ekr.20060919110638.28: *5* printStartElement
        def printStartElement(self, name, attrs):
            indent = '\t' * self.level or ''
            if attrs.getLength() > 0:
                g.pr('%s<%s %s>' % (
                    indent,
                    self.clean(name).strip(),
                    self.attrsToString(attrs, sep=' ')),
                    newline=False)
            else:
                g.pr('%s<%s>' % (
                    indent,
                    self.clean(name).strip()),
                    newline=False)
            if name.lower() in ['v', 't', 'vnodes', 'tnodes',]:
                g.pr('')
        #@+node:ekr.20060919110638.24: *6* attrsToString
        def attrsToString(self, attrs, sep='\n'):
            '''Convert the attributes to a string.

            attrs: an Attributes item passed to startElement.

            sep: the separator charater between attributes.'''
            result = [
                '%s="%s"' % (bunch.name, bunch.val)
                for bunch in self.attrsToList(attrs)
            ]
            return sep.join(result)
        #@+node:ekr.20060919110638.25: *6* clean
        def clean(self, s):
            return g.toEncodedString(s, "ascii")
        #@+node:ekr.20060919110638.30: *4* sax.characters
        def characters(self, content):
            '''Handle the characters element.'''
            if content and not g.isUnicode(content):
                g.trace('Non-unicode content', repr(content))
                content = g.toUnicode(content)
            content = content.replace('\r', '')
            if not content:
                return
            elementName = (self.elementStack and self.elementStack[-1].lower() or
                '<no element name>')
            # if self.trace: g.trace(elementName,content.strip())
            if elementName in ('t', 'vh'):
                # if elementName == 'vh': g.trace(elementName,repr(content))
                self.content.append(content)
            elif content.strip():
                g.pr('unexpected content:', elementName, repr(content))
        #@+node:ekr.20060919110638.31: *4* sax.endElement & helpers
        def endElement(self, name):
            '''Handle the end of any xml element.'''
            name = name.lower()
            if name in self.printElements or 'all' in self.printElements:
                indent = '\t' * (self.level - 1) or ''
                g.pr('%s</%s>' % (indent, self.clean(name).strip()))
            data = self.dispatchDict.get(name)
            if data is None:
                if 1: g.trace('unknown end element', name)
            else:
                junk, func = data
                if func:
                    func()
            name2 = self.elementStack.pop()
            assert name == name2
            # if self.trace: g.trace('** pop',name2)
        #@+node:ekr.20060919110638.32: *5* sax.endTnode
        def endTnode(self):
            '''Handle the end of a <tnode> element.'''
            trace = (False or self.trace) and not g.unitTesting
            # trace = trace and self.fileName.endswith('clone-test.leo')
            for sax_node in self.nodeList:
                sax_node.bodyString = ''.join(self.content)
                if trace: g.trace(repr(sax_node))
            self.content = []
        #@+node:ekr.20060919110638.33: *5* sax.endVnode
        def endVnode(self):
            '''Handle the end of a <vnode> element.'''
            self.level -= 1
            self.node = self.nodeStack.pop()
            # if self.trace: g.trace(repr(self.node))
        #@+node:ekr.20060919110638.34: *5* sax.endVH
        def endVH(self):
            '''Handle the end of a <vh> element.'''
            if self.node:
                self.node.headString = ''.join(self.content)
                # if self.trace: g.trace(repr(self.node))
            self.content = []
        #@+node:ekr.20060919110638.45: *4* sax.getRootNode
        def getRootNode(self):
            if self.trace:
                g.trace()
                self.rootNode.dump()
                for child in self.rootNode.children:
                    child.dump()
            return self.rootNode
        #@+node:ekr.20061004054323: *4* sax.processingInstruction (stylesheet)
        def processingInstruction(self, target, data):
            '''
            sax: handle and xml processing instruction.
            We expect the target to be 'xml-stylesheet'.
            '''
            trace = False and not g.unitTesting
            if target == 'xml-stylesheet':
                # A strange hack.  Don't set this for settings files.
                # This looks like a strange sax interaction.
                sfn = (self.c.shortFileName() or '').strip().lower()
                if sfn.endswith('leosettings.leo') or sfn.endswith('myleosettings.leo'):
                    pass
                else:
                    self.c.frame.stylesheet = data
                    if trace: g.trace(self.c.shortFileName(), repr(data))
                # g.warning('','%s: %s' % (target,data))
            else:
                g.trace(target, data)
        #@+node:ekr.20060919110638.35: *4* sax.startElement & helpers
        def startElement(self, name, attrs):
            name = name.lower()
            if name in self.printElements or 'all' in self.printElements:
                self.printStartElement(name, attrs)
            self.elementStack.append(name)
            # if self.trace: g.trace('**push',name)
            data = self.dispatchDict.get(name)
            if data is None:
                if 1: g.trace('unknown start element', name)
            else:
                func, junk = data
                if func:
                    func(attrs)
        #@+node:ekr.20060919110638.36: *5* sax.getWindowPositionAttributes
        def getWindowPositionAttributes(self, attrs):
            trace = False and not g.unitTesting
            c = self.c
            d = {}
            windowSize = g.app.loadManager.options.get('windowSize')
            if windowSize is not None:
                h, w = windowSize # checked in LM.scanOption.
                d['height'] = h
                d['width'] = w
                for bunch in self.attrsToList(attrs):
                    name = bunch.name; val = bunch.val
                    if name in ('top', 'left'):
                        try:
                            d[name] = int(val)
                        except ValueError:
                            d[name] = 50 # A reasonable default.
                if trace: g.trace(d)
            elif g.enableDB and c.mFileName:
                d = c.cacher.getCachedWindowPositionDict(c.mFileName)
            if not d and c.fixed and c.fixedWindowPosition:
                width, height, left, top = c.fixedWindowPosition
                d = {'top': top, 'left': left, 'width': width, 'height': height}
            if not d:
                for bunch in self.attrsToList(attrs):
                    name = bunch.name; val = bunch.val
                    if name in ('top', 'left', 'width', 'height'):
                        try:
                            d[name] = int(val)
                        except ValueError:
                            d[name] = 100 # A reasonable default.
                    else:
                        g.trace(name, len(val))
            if trace: g.trace(c.mFileName, d)
            return d # Assigned to self.global_window_position
        #@+node:ekr.20060919110638.37: *5* sax.startGlobals
        def startGlobals(self, attrs):
            trace = False and not g.unitTesting
            c = self.c
            if self.inClipboard:
                return
            if trace: g.trace(c.mFileName)
            use_db = g.enableDB and c.mFileName
            if use_db:
                ratio, ratio2 = c.cacher.getCachedGlobalFileRatios()
                if trace: g.trace(ratio, ratio2)
                self.ratio, self.secondary_ratio = ratio, ratio2
            else:
                try:
                    for bunch in self.attrsToList(attrs):
                        name = bunch.name; val = bunch.val
                        if name == 'body_outline_ratio':
                            self.ratio = float(val)
                        elif name == 'body_secondary_ratio':
                            self.secondary_ratio = float(val)
                    if trace: g.trace('** not cached:', '%1.2f %1.2f' % (
                        self.ratio, self.secondary_ratio))
                except Exception:
                    pass
        #@+node:ekr.20060919110638.38: *5* sax.startWinPos
        def startWinPos(self, attrs):
            self.global_window_position = self.getWindowPositionAttributes(attrs)
        #@+node:ekr.20060919110638.39: *5* sax.startLeoHeader
        def startLeoHeader(self, unused_attrs):
            self.tnxToListDict = {}
        #@+node:ekr.20060919110638.40: *5* sax.startVH
        def startVH(self, unused_attrs):
            self.content = []
        #@+node:ekr.20060919112118: *5* sax.startVnodes
        def startVnodes(self, unused_attrs):
            if self.inClipboard:
                return # No need to do anything to the main window.
            c = self.c; d = self.global_window_position
            w = d.get('width', 700)
            h = d.get('height', 500)
            x = d.get('left', 50)
            y = d.get('top', 50)
            # g.trace(d,w,h,x,y)
            # Redraw the window before writing into it.
            # Honor --minimized, --maximized or --fullscreen.
            # 2013/10/25: do set the geometry for minimized windows.
            if g.app.start_minimized:
                c.frame.setTopGeometry(w, h, x, y)
            elif not g.app.start_maximized and not g.app.start_fullscreen:
                c.frame.setTopGeometry(w, h, x, y)
                c.frame.deiconify()
                c.frame.lift()
            # Causes window to appear.
            if c.frame.top:
                c.frame.resizePanesToRatio(self.ratio, self.secondary_ratio)
            if not self.silent and not g.unitTesting:
                g.es("reading:", self.fileName)
        #@+node:ekr.20060919110638.41: *5* sax.startTnode
        def startTnode(self, attrs):
            if not self.inElement('tnodes'):
                self.error('<t> outside <tnodes>')
            self.content = []
            self.tnodeAttributes(attrs)
        #@+node:ekr.20060919110638.42: *6* sax.tnodeAttributes (SaxContentHandler)
        def tnodeAttributes(self, attrs):
            # The VNode must have a tx attribute to associate content
            # with the proper node.
            trace = False and not g.unitTesting
            verbose = False
            node = self.node
            self.nodeList = []
            val = None
            # Step one: find the tx attribute
            for bunch in self.attrsToList(attrs):
                name = bunch.name; val = bunch.val
                if name == 'tx':
                    # 2010/02/03: This code formerly did something
                    # different when unit testing just to support a unit test.
                    # Hahaha.  The unit test *caused* the bug!
                    self.nodeList = self.tnxToListDict.get(val, [])
                    if trace and verbose: g.trace('tx', self.nodeList)
                    break
            if not self.nodeList:
                self.error('Bad leo file: no node for <t tx=%s>' % (val))
                return
            # Step two: find all the other attributes:
            for bunch in self.attrsToList(attrs):
                name = bunch.name; val = bunch.val
                if name != 'tx':
                    # Huge bug fix: 2009/7/1: was node == self.node.
                    for node in self.nodeList:
                        if trace: g.trace('%s %s=%s...' % (node, name, val[: 20]))
                        node.tnodeAttributes[name] = val
            # if not self.nodeList:
                # self.error('Bad leo file: no tx attribute for VNode')
        #@+node:ekr.20060919110638.43: *5* sax.startVnode
        def startVnode(self, attrs):
            if not self.inElement('vnodes'):
                self.error('<v> outside <vnodes>')
            if self.rootNode:
                parent = self.node
            else:
                self.rootNode = parent = SaxNodeClass() # The dummy parent node.
                parent.headString = 'dummyNode'
            self.node = SaxNodeClass()
            parent.children.append(self.node)
            self.vnodeAttributes(attrs)
            self.nodeStack.append(parent)
            return parent
        #@+node:ekr.20060919110638.44: *6* sax.vnodeAttributes
        # The native attributes of <v> elements are a, t, vtag, tnodeList,
        # marks, expanded and descendentTnodeUnknownAttributes.

        def vnodeAttributes(self, attrs):
            node = self.node
            for bunch in self.attrsToList(attrs):
                name = bunch.name; val = bunch.val
                if name == 't':
                    aList = self.tnxToListDict.get(val, [])
                    aList.append(self.node)
                    self.tnxToListDict[val] = aList
                    node.tnx = str(val) # nodeIndices.toString returns a string.
                else:
                    node.attributes[name] = val
        #@-others
    #@+node:ekr.20060919110638.15: *3* class SaxNodeClass
    class SaxNodeClass:
        '''A class representing one <v> element.

        Use getters to access the attributes, properties and rules of this mode.'''
        #@+others
        #@+node:ekr.20060919110638.16: *4*  node.__init__
        def __init__(self):
            self.attributes = {}
            self.bodyString = ''
            self.headString = ''
            self.children = []
            self.tnodeAttributes = {}
            self.tnodeList = []
            self.tnx = None
        #@+node:ekr.20060919110638.17: *4*  node.__str__ & __repr__
        def __str__(self):
            return '<v:%s %s %s>' % (id(self), self.headString, len(self.bodyString))

        __repr__ = __str__
        #@+node:ekr.20060919110638.18: *4* node.dump
        def dump(self):
            g.pr('\nnode: %s tnx: %s len(body): %d %s' % (
                id(self), self.tnx, len(self.bodyString), self.headString))
            g.pr('children:', g.listToString(self.children))
            g.pr('attrs:', list(self.attributes.values()))
        #@-others
    #@-others
    #@-<< define sax classes >>

class FileCommands:
    """A class creating the FileCommands subcommander."""
    #@+others
    #@+node:ekr.20090218115025.4: ** fc.Birth
    #@+node:ekr.20150509194827.1: *3* fc.cmd (decorator)
    def cmd(name):
        '''Command decorator for the FileCommands class.'''
        # pylint: disable=no-self-argument
        return g.new_cmd_decorator(name, ['c', 'fileCommands',])
    #@+node:ekr.20031218072017.3019: *3* fc.ctor
    def __init__(self, c):
        '''Ctor for FileCommands class.'''
        # g.trace("__init__", "FileCommands.__init__")
        self.c = c
        self.frame = c.frame
        self.nativeTnodeAttributes = ('tx',)
        self.nativeVnodeAttributes = (
            'a',
            'descendentTnodeUnknownAttributes',
            'descendentVnodeUnknownAttributes', # New in Leo 4.5.
            'expanded', 'marks', 't', 'tnodeList',
            # 'vtag',
        )
        self.initIvars()
    #@+node:ekr.20090218115025.5: *3* fc.initIvars
    def initIvars(self):
        '''Init ivars of the FileCommands class.'''
        # General...
        c = self.c
        self.mFileName = ""
        self.fileDate = -1
        self.leo_file_encoding = c.config.new_leo_file_encoding
            # The bin param doesn't exist in Python 2.3;
            # the protocol param doesn't exist in earlier versions of Python.
            # version = '.'.join([str(sys.version_info[i]) for i in (0,1)])
        # For reading...
        self.checking = False # True: checking only: do *not* alter the outline.
        self.descendentExpandedList = []
        self.descendentMarksList = []
        self.forbiddenTnodes = []
        self.descendentTnodeUaDictList = []
        self.descendentVnodeUaDictList = []
        self.ratio = 0.5
        self.currentVnode = None
        self.rootVnode = None
        # For writing...
        self.read_only = False
        self.rootPosition = None
        self.outputFile = None
        self.openDirectory = None
        self.putCount = 0
        self.toString = False
        self.usingClipboard = False
        self.currentPosition = None
        # New in 3.12...
        self.copiedTree = None
        self.gnxDict = {}
            # keys are gnx strings as returned by canonicalTnodeIndex.
            # Values are vnodes.
            # 2011/12/10: This dict is never re-inited.
        self.vnodesDict = {}
            # keys are gnx strings; values are ignored
    #@+node:ekr.20031218072017.3020: ** fc.Reading
    #@+node:ekr.20060919104836: *3*  fc.Reading Top-level
    #@+node:ekr.20070919133659.1: *4* fc.checkLeoFile
    @cmd('check-leo-file')
    def checkLeoFile(self, event=None):
        '''The check-leo-file command.'''
        fc = self; c = fc.c; p = c.p
        # Put the body (minus the @nocolor) into the file buffer.
        s = p.b; tag = '@nocolor\n'
        if s.startswith(tag): s = s[len(tag):]
        # Do a trial read.
        self.checking = True
        self.initReadIvars()
        c.loading = True # disable c.changed
        try:
            try:
                theFile = g.app.loadManager.openLeoOrZipFile(c.mFileName)
                self.readSaxFile(
                    theFile, fileName='check-leo-file',
                    silent=False, inClipboard=False, reassignIndices=False)
                g.blue('check-leo-file passed')
            except Exception:
                junk, message, junk = sys.exc_info()
                # g.es_exception()
                g.error('check-leo-file failed:', str(message))
        finally:
            self.checking = False
            c.loading = False # reenable c.changed
    #@+node:ekr.20031218072017.1559: *4* fc.getLeoOutlineFromClipboard & helpers
    def getLeoOutlineFromClipboard(self, s, reassignIndices=True, tempOutline=False):
        '''Read a Leo outline from string s in clipboard format.'''
        trace = False and not g.unitTesting
        verbose = False
        c = self.c
        current = c.p
        if not current:
            g.trace('no c.p')
            return None
        if trace: g.trace('reassign', reassignIndices, 'temp', tempOutline)
        check = not reassignIndices
        self.initReadIvars()
        # Save the hidden root's children.
        children = c.hiddenRootNode.children
        # 2011/12/12: save and clear gnxDict.
        # This ensures that new indices will be used for all nodes.
        if reassignIndices or tempOutline:
            oldGnxDict = self.gnxDict
            self.gnxDict = {}
        else:
            # All pasted nodes should already have unique gnx's.
            ni = g.app.nodeIndices
            for v in c.all_unique_nodes():
                ni.check_gnx(c, v.fileIndex, v)
        self.usingClipboard = True
        try:
            # This encoding must match the encoding used in putLeoOutline.
            s = g.toEncodedString(s, self.leo_file_encoding, reportErrors=True)
            if trace and verbose: g.trace(s)
            # readSaxFile modifies the hidden root.
            v = self.readSaxFile(
                theFile=None, fileName='<clipboard>',
                silent=True, # don't tell about stylesheet elements.
                inClipboard=True, reassignIndices=reassignIndices, s=s)
            if not v:
                return g.es("the clipboard is not valid ", color="blue")
        finally:
            self.usingClipboard = False
        # Restore the hidden root's children
        c.hiddenRootNode.children = children
        # Unlink v from the hidden root.
        v.parents.remove(c.hiddenRootNode)
        p = leoNodes.Position(v)
        # Important: we must not adjust links when linking v
        # into the outline.  The read code has already done that.
        if current.hasChildren() and current.isExpanded():
            # What does checkPaste do??
            if check and not self.checkPaste(current, p):
                return None
            p._linkAsNthChild(current, 0, adjust=False)
        else:
            if check and not self.checkPaste(current.parent(), p):
                return None
            p._linkAfter(current, adjust=False)
        if tempOutline:
            self.gnxDict = oldGnxDict
        elif reassignIndices:
            self.gnxDict = oldGnxDict
            ni = g.app.nodeIndices
            for p2 in p.self_and_subtree():
                v = p2.v
                index = ni.getNewIndex(v)
                if g.trace_gnxDict: g.trace(c.shortFileName(), '**restoring**', index, v)
        if trace and verbose:
            g.trace('**** dumping outline...')
            c.dumpOutline()
        c.selectPosition(p)
        self.initReadIvars()
        return p

    getLeoOutline = getLeoOutlineFromClipboard # for compatibility
    #@+node:ekr.20080410115129.1: *5* checkPaste
    def checkPaste(self, parent, p):
        '''Return True if p may be pasted as a child of parent.'''
        if not parent: return True
        parents = [z.copy() for z in parent.self_and_parents()]
        for p in p.self_and_subtree():
            for z in parents:
                # g.trace(p.h,id(p.v),id(z.v))
                if p.v == z.v:
                    g.warning('Invalid paste: nodes may not descend from themselves')
                    return False
        return True
    #@+node:ekr.20031218072017.1553: *4* fc.getLeoFile & helpers
    def getLeoFile(self,
        theFile,
        fileName,
        readAtFileNodesFlag=True,
        silent=False,
        checkOpenFiles=True,
    ):
        '''
            Read a .leo file.
            The caller should follow this with a call to c.redraw().
        '''
        fc, c = self, self.c
        t1 = time.time()
        c.setChanged(False) # May be set when reading @file nodes.
        fc.warnOnReadOnlyFiles(fileName)
        fc.checking = False
        fc.mFileName = c.mFileName
        fc.initReadIvars()
        recoveryNode = None
        try:
            c.loading = True # disable c.changed
            if not silent and checkOpenFiles:
                # Don't check for open file when reverting.
                g.app.checkForOpenFile(c, fileName)
            ok = fc.getLeoFileHelper(theFile, fileName, silent)
                # Read the .leo file and create the outline.
            if ok:
                fc.resolveTnodeLists()
                    # Do this before reading external files.
                c.setFileTimeStamp(fileName)
                if readAtFileNodesFlag:
                    # Redraw before reading the @file nodes so the screen isn't blank.
                    # This is important for big files like LeoPy.leo.
                    c.redraw()
                    recoveryNode = fc.readExternalFiles(fileName)
        finally:
            if g.app.debug: g.trace('=====',
                recoveryNode and recoveryNode.h,
                c.p and c.p.h)
            p = recoveryNode or c.p or c.lastTopLevel()
                # lastTopLevel is a better fallback, imo.
            # New in Leo 5.3. Delay the second redraw until idle time.
            # This causes a slight flash, but corrects a hangnail.

            def handler(timer, c=c, p=c.p):
                c.initialFocusHelper()
                c.redraw(p)
                c.k.showStateAndMode()
                c.outerUpdate()
                timer.stop()
        
            timer = g.IdleTime(handler, delay=0, tag='getLeoFile')
            if timer:
                timer.start()
            else:
                # Defensive code:
                c.selectPosition(p)
                c.initialFocusHelper()
                c.k.showStateAndMode()
                c.outerUpdate()

            c.checkOutline()
                # Must be called *after* ni.end_holding.
            c.loading = False
                # reenable c.changed
            theFile.close()
                # Fix bug https://bugs.launchpad.net/leo-editor/+bug/1208942
                # Leo holding directory/file handles after file close?
        if c.changed:
            fc.propegateDirtyNodes()
        c.setChanged(c.changed) # Refresh the changed marker.
        fc.initReadIvars()
        t2 = time.time()
        g.es('read outline in %2.2f seconds' % (t2 - t1))
        return ok, c.frame.ratio
    #@+node:ekr.20090526081836.5841: *5* fc.getLeoFileHelper
    def getLeoFileHelper(self, theFile, fileName, silent):
        '''Read the .leo file and create the outline.'''
        c, fc = self.c, self
        try:
            ok = True
            v = fc.readSaxFile(
                theFile,
                fileName,
                silent,
                inClipboard=False,
                reassignIndices=False,
            )
            if v: # v is None for minimal .leo files.
                c.setRootVnode(v)
                fc.rootVnode = v
            else:
                v = leoNodes.VNode(context=c)
                v.setHeadString('created root node')
                p = leoNodes.Position(v)
                p._linkAsRoot(oldRoot=None)
                fc.rootVnode = v
                # c.setRootPosition()
                c.changed = False
        except BadLeoFile:
            junk, message, junk = sys.exc_info()
            if not silent:
                g.es_exception()
                c.alert(fc.mFileName + " is not a valid Leo file: " + str(message))
            ok = False
        return ok
    #@+node:ekr.20100205060712.8314: *5* fc.handleNodeConflicts & helper
    def handleNodeConflicts(self):
        '''Create a 'Recovered Nodes' node for each entry in c.nodeConflictList.'''
        c = self.c
        if not c.nodeConflictList:
            return
        if not c.make_node_conflicts_node:
            g.es_print('suppressed node conflicts', color='red')
            return
        # Find the last top-level node.
        sib = c.rootPosition()
        while sib.hasNext():
            sib.moveToNext()
        # Create the 'Recovered Nodes' node.
        root = sib.insertAfter()
        root.setHeadString('Recovered Nodes')
        root.expand()
        # For each conflict, create one child and two grandchildren.
        for bunch in c.nodeConflictList:
            tag = bunch.get('tag') or ''
            gnx = bunch.get('gnx') or ''
            fn = bunch.get('fileName') or ''
            b1, h1 = bunch.get('b_old'), bunch.get('h_old')
            b2, h2 = bunch.get('b_new'), bunch.get('h_new')
            child = root.insertAsLastChild()
            h = 'Recovered node "%s" from %s' % (h1, g.shortFileName(fn))
            child.setHeadString(h)
            # child.setBodyString('%s %s' % (tag,gnx))
            line1 = '%s %s\nDiff...\n' % (tag, gnx)
            d = difflib.Differ().compare(g.splitLines(b2), g.splitLines(b1))
            # d = difflib.unified_diff(g.splitLines(b2),g.splitLines(b1))
            diffLines = [z for z in d]
            lines = [line1]
            lines.extend(diffLines)
            # There is less need to show trailing newlines because
            # we don't report changes involving only trailing newlines.
            child.setBodyString(''.join(lines)) # .replace('\n','\\n\n'))
            n1 = child.insertAsNthChild(0)
            n2 = child.insertAsNthChild(1)
            n1.setHeadString('old:' + h1)
            n1.setBodyString(b1)
            n2.setHeadString('new:' + h2)
            n2.setBodyString(b2)
        return root
    #@+node:ekr.20100124110832.6212: *5* fc.propegateDirtyNodes
    def propegateDirtyNodes(self):
        fc = self; c = fc.c
        aList = [z.copy() for z in c.all_positions() if z.isDirty()]
        for p in aList:
            p.setAllAncestorAtFileNodesDirty()
    #@+node:ekr.20120212220616.10537: *5* fc.readExternalFiles
    def readExternalFiles(self, fileName):
        '''Read all external files.'''
        c, fc = self.c, self
        c.atFileCommands.readAll(c.rootVnode(), partialFlag=False)
        recoveryNode = fc.handleNodeConflicts()
        # Do this after reading external files.
        # The descendent nodes won't exist unless we have read
        # the @thin nodes!
        fc.restoreDescendentAttributes()
        fc.setPositionsFromVnodes()
        return recoveryNode
    #@+node:ekr.20031218072017.1554: *5* fc.warnOnReadOnlyFiles
    def warnOnReadOnlyFiles(self, fileName):
        # os.access may not exist on all platforms.
        try:
            self.read_only = not os.access(fileName, os.W_OK)
        except AttributeError:
            self.read_only = False
        except UnicodeError:
            self.read_only = False
        if self.read_only and not g.unitTesting:
            g.error("read only:", fileName)
    #@+node:ekr.20031218072017.3029: *4* fc.readAtFileNodes
    def readAtFileNodes(self):
        c = self.c; p = c.p
        c.endEditing()
        c.atFileCommands.readAll(p, partialFlag=True)
        c.redraw()
        # Force an update of the body pane.
        c.setBodyString(p, p.b)
        c.frame.body.onBodyChanged(undoType=None)
    #@+node:ekr.20031218072017.2297: *4* fc.openLeoFile
    def openLeoFile(self, theFile, fileName, readAtFileNodesFlag=True, silent=False):
        '''Open a Leo file.'''
        c, frame = self.c, self.c.frame
        # Set c.openDirectory
        theDir = g.os_path_dirname(fileName)
        if theDir:
            c.openDirectory = c.frame.openDirectory = theDir
        # Get the file.
        ok, ratio = self.getLeoFile(
            theFile, fileName,
            readAtFileNodesFlag=readAtFileNodesFlag,
            silent=silent,
        )
        if ok:
            frame.resizePanesToRatio(ratio, frame.secondary_ratio)
        return ok
    #@+node:ekr.20031218072017.3030: *4* fc.readOutlineOnly
    def readOutlineOnly(self, theFile, fileName):
        c = self.c
        #@+<< Set the default directory >>
        #@+node:ekr.20071211134300: *5* << Set the default directory >>
        #@+at
        # The most natural default directory is the directory containing the .leo file
        # that we are about to open. If the user has specified the "Default Directory"
        # preference that will over-ride what we are about to set.
        #@@c
        theDir = g.os_path_dirname(fileName)
        if len(theDir) > 0:
            c.openDirectory = c.frame.openDirectory = theDir
        #@-<< Set the default directory >>
        ok, ratio = self.getLeoFile(theFile, fileName, readAtFileNodesFlag=False)
        c.redraw()
        c.frame.deiconify()
        junk, junk, secondary_ratio = self.frame.initialRatios()
        c.frame.resizePanesToRatio(ratio, secondary_ratio)
        return ok
    #@+node:ekr.20060919133249: *3* fc.Reading Common
    # Methods common to both the sax and non-sax code.
    #@+node:ekr.20031218072017.2004: *4* fc.canonicalTnodeIndex
    def canonicalTnodeIndex(self, index):
        """Convert Tnnn to nnn, leaving gnx's unchanged."""
        # index might be Tnnn, nnn, or gnx.
        if index is None:
            g.trace('Can not happen: index is None')
            return None
        junk, theTime, junk = g.app.nodeIndices.scanGnx(index, 0)
        if theTime == None: # A pre-4.1 file index.
            if index[0] == "T":
                index = index[1:]
        return index
    #@+node:ekr.20040701065235.1: *4* fc.getDescendentAttributes
    def getDescendentAttributes(self, s, tag=""):
        '''s is a list of gnx's, separated by commas from a <v> or <t> element.
        Parses s into a list.

        This is used to record marked and expanded nodes.
        '''
        gnxs = s.split(',')
        result = [gnx for gnx in gnxs if len(gnx) > 0]
        # g.trace(tag,result)
        return result
    #@+node:EKR.20040627114602: *4* fc.getDescendentUnknownAttributes
    # Pre Leo 4.5 Only @thin vnodes had the descendentTnodeUnknownAttributes field.
    # New in Leo 4.5: @thin & @shadow vnodes have descendentVnodeUnknownAttributes field.

    def getDescendentUnknownAttributes(self, s, v=None):
        '''Unhexlify and unpickle t/v.descendentUnknownAttribute field.'''
        try:
            # Changed in version 3.2: Accept only bytestring or bytearray objects as input.
            s = g.toEncodedString(s) # 2011/02/22
            bin = binascii.unhexlify(s)
                # Throws a TypeError if val is not a hex string.
            val = pickle.loads(bin)
            return val
        except Exception:
            g.es_exception()
            g.trace('Can not unpickle', type(s), v and v.h, s[: 40])
            return None
    #@+node:ekr.20060919142200.1: *4* fc.initReadIvars
    def initReadIvars(self):
        self.descendentTnodeUaDictList = []
        self.descendentVnodeUaDictList = []
        self.descendentExpandedList = []
        self.descendentMarksList = []
            # 2011/12/10: never re-init this dict.
            # self.gnxDict = {}
            # g.trace('*** clearing gnxDict',g.callers())
        self.c.nodeConflictList = [] # 2010/01/05
        self.c.nodeConflictFileName = None # 2010/01/05
    #@+node:EKR.20040627120120: *4* fc.restoreDescendentAttributes
    def restoreDescendentAttributes(self):
        trace = False and not g.unitTesting
        verbose = True
        c = self.c
        for resultDict in self.descendentTnodeUaDictList:
            if trace and verbose: g.trace('t.dict', resultDict)
            for gnx in resultDict:
                tref = self.canonicalTnodeIndex(gnx)
                v = self.gnxDict.get(tref)
                if v:
                    v.unknownAttributes = resultDict[gnx]
                    v._p_changed = 1
                elif verbose:
                    g.error(
                        'restoreDescendantAttributes: '
                        'can not find VNode (duA): gnx = %s' % (gnx))
        # New in Leo 4.5: keys are archivedPositions, values are attributes.
        for root_v, resultDict in self.descendentVnodeUaDictList:
            if trace and verbose: g.trace('v.dict', resultDict)
            for key in resultDict:
                v = self.resolveArchivedPosition(key, root_v)
                if v:
                    v.unknownAttributes = resultDict[key]
                    v._p_changed = 1
                elif verbose:
                    g.error(
                        'restoreDescendantAttributes: '
                        'can not find VNode (duA): archivedPosition: %s, root_v: %s' % (
                            key, root_v))
        marks = {}; expanded = {}
        for gnx in self.descendentExpandedList:
            tref = self.canonicalTnodeIndex(gnx)
            v = self.gnxDict.get(gnx)
            if v:
                expanded[v] = v
                # if trace: g.trace('expanded',v)
            elif verbose:
                g.error(
                    'restoreDescendantAttributes: '
                    'can not find VNode (expanded): gnx = %s, tref: %s' % (gnx, tref))
        for gnx in self.descendentMarksList:
            tref = self.canonicalTnodeIndex(gnx)
            v = self.gnxDict.get(gnx)
            if v: marks[v] = v
            elif verbose:
                g.error(
                    'restoreDescendantAttributes: '
                    'can not find VNode (marks): gnx = %s tref: %s' % (gnx, tref))
        if marks or expanded:
            # g.trace('marks',len(marks),'expanded',len(expanded))
            for p in c.all_unique_positions():
                if marks.get(p.v):
                    p.v.initMarkedBit()
                        # This was the problem: was p.setMark.
                        # There was a big performance bug in the mark hook in the Node Navigator plugin.
                if expanded.get(p.v):
                    p.expand()
                    # if trace: g.trace('expand',p.h)
    #@+node:ekr.20060919104530: *3* fc.Reading Sax
    #@+node:ekr.20090525144314.6526: *4* fc.cleanSaxInputString
    def cleanSaxInputString(self, s):
        '''Clean control characters from s.
        s may be a bytes or a (unicode) string.'''
        # Note: form-feed ('\f') is 12 decimal.
        badchars = [chr(ch) for ch in range(32)]
        badchars.remove('\t')
        badchars.remove('\r')
        badchars.remove('\n')
        flatten = ''.join(badchars)
        pad = ' ' * len(flatten)
        # pylint: disable=no-member
        # Class 'str' has no 'maketrans' member
        if g.isPython3:
            flatten = bytes(flatten, 'utf-8')
            pad = bytes(pad, 'utf-8')
            transtable = bytes.maketrans(flatten, pad)
        else:
            transtable = string.maketrans(flatten, pad)
        return s.translate(transtable)
    # for i in range(32): print i,repr(chr(i))
    #@+node:ekr.20060919110638.5: *4* fc.createSaxChildren & helpers
    def createSaxChildren(self, sax_node, parent_v):
        '''Create vnodes for all children in sax_node.children.'''
        trace = False and not g.unitTesting
        children = []
        for sax_child in sax_node.children:
            tnx = sax_child.tnx
            v = self.gnxDict.get(tnx)
            if v: # A clone.
                if trace: g.trace('**clone', tnx, v.gnx, v)
                v = self.createSaxVnode(sax_child, parent_v, v=v)
            else:
                v = self.createSaxVnode(sax_child, parent_v)
                self.createSaxChildren(sax_child, v)
            children.append(v)
        parent_v.children = children
        for child in children:
            child.parents.append(parent_v)
            if trace: g.trace(
                '*** added parent', parent_v, 'to', child,
                'len(child.parents)', len(child.parents))
        return children
    #@+node:ekr.20060919110638.7: *5* fc.createSaxVnode & helpers
    def createSaxVnode(self, sax_node, parent_v, v=None):
        '''Create a vnode, or use an existing vnode.'''
        c, at = self.c, self.c.atFileCommands
        trace = True and not g.unitTesting
            # and self.mFileName and sax_node.headString == 'clone-test'
            # and not g.app.openingSettingsFile)
        trace_update, verbose = False, False
        h = sax_node.headString
        b = sax_node.bodyString
        # trace = trace and v and v.gnx == "vitalije.20150316130845.1"
        if v:
            # The body of the later node overrides the earlier.
            # Don't set t.h: h is always empty.
            if v.b == b:
                # if trace: g.trace('old: %r new: %r' % (v.bodyString(),b),h)
                if trace and verbose: g.trace(
                    '***no update\nold: %s\nnew: %s' % (v.b, b))
            else:
                if trace and trace_update: g.trace(
                    '***update\nold: %s\nnew: %s' % (v.b, b))
                v.setBodyString(b)
                at.bodySetInited(v)
        else:
            # Fix bug 158: Corrupt .leo files cause Leo to hang.
            # Part 1: must explicitly test against None: tnx could be 0.
            if sax_node.tnx is not None:
                # Important: this should retain compatibility with old .leo files.
                gnx = g.toUnicode(self.canonicalTnodeIndex(sax_node.tnx))
            else:
                # Part 2: Do *not* call ni.getNewIndex here: v is None!
                # Instead, let the VNode ctor below allocate the gnx.
                gnx = None
            # if trace and gnx: g.trace('%-25s new: %3s %s' % (gnx,len(b),h))
            v = leoNodes.VNode(context=c, gnx=gnx)
            v.setBodyString(b)
            at.bodySetInited(v)
            v.setHeadString(h)
        if g.trace_gnxDict: g.trace(c.shortFileName(), gnx, v)
        if trace and verbose: g.trace(
            'tnx', '%-22s' % (gnx), 'v', id(v),
            'len(body)', '%-4d' % (len(b)), h)
        self.handleVnodeSaxAttributes(sax_node, v)
        self.handleTnodeSaxAttributes(sax_node, v)
        return v
    #@+node:ekr.20060919110638.8: *6* fc.handleTnodeSaxAttributes
    def handleTnodeSaxAttributes(self, sax_node, v):
        trace = False and not g.unitTesting
        d = sax_node.tnodeAttributes
        if trace and d: g.trace(sax_node, list(d.keys()))
        aDict = {}
        for key in d:
            val = g.toUnicode(d.get(key)) # 2011/02/22
            val2 = self.getSaxUa(key, val)
            # g.trace(key,val,val2)
            aDict[key] = val2
        if aDict:
            if trace: g.trace('uA', v, list(aDict.keys()))
            v.unknownAttributes = aDict
    #@+node:ekr.20061004053644: *6* fc.handleVnodeSaxAttributes
    # The native attributes of <v> elements are a, t, vtag, tnodeList,
    # marks, expanded, and descendentTnodeUnknownAttributes.
    # New in Leo 4.5: added descendentVnodeUnknownAttributes to native attributes.

    def handleVnodeSaxAttributes(self, sax_node, v):
        trace = False and not g.unitTesting
        d = sax_node.attributes
        s = d.get('a')
        if s:
            if trace and 'E' in s: g.trace('expand', v)
            # g.trace('%s a=%s %s' % (id(sax_node),s,v.headString()))
            # 'C' (clone) and 'D' bits are not used.
            if 'M' in s: v.setMarked()
            if 'E' in s: v.expand()
            if 'O' in s: v.setOrphan()
            # if 'T' in s: self.topVnode = v
            if 'V' in s:
                # g.red('handleVnodeSaxAttributes: setting currentVnode',v)
                self.currentVnode = v
        s = d.get('tnodeList', '')
        tnodeList = s and s.split(',')
        if tnodeList:
            # This tnodeList will be resolved later.
            if trace: g.trace('found tnodeList', v.headString(), tnodeList)
            v.tempTnodeList = tnodeList
        s = d.get('descendentTnodeUnknownAttributes')
        if s:
            aDict = self.getDescendentUnknownAttributes(s, v=v)
            if aDict:
                # g.trace('descendentTnodeUaDictList',aDict)
                self.descendentTnodeUaDictList.append(aDict)
        s = d.get('descendentVnodeUnknownAttributes')
        if s:
            aDict = self.getDescendentUnknownAttributes(s, v=v)
            if aDict:
                # g.trace('descendentVnodeUaDictList',aDict)
                self.descendentVnodeUaDictList.append((v, aDict),)
        s = d.get('expanded')
        if s:
            aList = self.getDescendentAttributes(s, tag="expanded")
            # g.trace('expanded list',len(aList))
            self.descendentExpandedList.extend(aList)
        s = d.get('marks')
        if s:
            aList = self.getDescendentAttributes(s, tag="marks")
            # g.trace('marks list',len(aList))
            self.descendentMarksList.extend(aList)
        aDict = {}
        for key in d:
            if key in self.nativeVnodeAttributes:
                # This is not a bug.
                if False and trace: g.trace(
                    '****ignoring***', key, d.get(key))
            else:
                val = d.get(key)
                val2 = self.getSaxUa(key, val)
                aDict[key] = val2
                # g.trace(key,val,val2)
        if aDict:
            # if trace: g.trace('uA',v,aDict)
            v.unknownAttributes = aDict
    #@+node:ekr.20060919110638.2: *4* fc.dumpSaxTree
    def dumpSaxTree(self, root, dummy):
        if not root:
            g.pr('dumpSaxTree: empty tree')
            return
        if not dummy:
            root.dump()
        for child in root.children:
            self.dumpSaxTree(child, dummy=False)
    #@+node:tbrown.20140615093933.89639: *4* fc.bytes_to_unicode
    def bytes_to_unicode(self, ob):
        """recursively convert bytes objects in strings / lists / dicts to str
        objects, thanks to TNT
        http://stackoverflow.com/questions/22840092/unpickling-data-from-python-2-with-unicode-strings-in-python-3

        Needed for reading Python 2.7 pickles in Python 3.4 in getSaxUa()
        """
        # pylint: disable=unidiomatic-typecheck
        # This is simpler than using isinstance.
        t = type(ob)
        if t in (list, tuple):
            l = [str(i, 'utf-8') if type(i) is bytes else i for i in ob]
            l = [self.bytes_to_unicode(i) if type(i) in (list, tuple, dict) else i
                for i in l]
            ro = tuple(l) if t is tuple else l
        elif t is dict:
            byte_keys = [i for i in ob if type(i) is bytes]
            for bk in byte_keys:
                v = ob[bk]
                del(ob[bk])
                ob[str(bk, 'utf-8')] = v
            for k in ob:
                if type(ob[k]) is bytes:
                    ob[k] = str(ob[k], 'utf-8')
                elif type(ob[k]) in (list, tuple, dict):
                    ob[k] = self.bytes_to_unicode(ob[k])
            ro = ob
        elif t is bytes: # TNB added this clause
            ro = str(ob, 'utf-8')
        else:
            ro = ob
        return ro
    #@+node:ekr.20061003093021: *4* fc.getSaxUa
    def getSaxUa(self, attr, val, kind=None): # Kind is for unit testing.
        """Parse an unknown attribute in a <v> or <t> element.
        The unknown tag has been pickled and hexlify'd.
        """
        try:
            # val = str(val)
            val = g.toEncodedString(val) # 2011/02/22.
        except Exception:
            g.es_print('unexpected exception converting hexlified string to string')
            g.es_exception()
            return None
        # New in 4.3: leave string attributes starting with 'str_' alone.
        if attr.startswith('str_'):
            if g.isString(val) or g.isBytes(val):
                return g.toUnicode(val)
        # New in 4.3: convert attributes starting with 'b64_' using the base64 conversion.
        if 0: # Not ready yet.
            if attr.startswith('b64_'):
                try: pass
                except Exception: pass
        try:
            binString = binascii.unhexlify(val) # Throws a TypeError if val is not a hex string.
        except Exception:
            # Python 2.x throws TypeError
            # Python 3.x throws binascii.Error
            # Assume that Leo 4.1 wrote the attribute.
            if g.unitTesting:
                assert kind == 'raw', 'unit test failed: kind=' % repr(kind)
            else:
                g.trace('can not unhexlify %s=%s' % (attr, val))
            return val
        try:
            # No change needed to support protocols.
            val2 = pickle.loads(binString)
            # g.trace('v.3 val:',val2)
            return val2
        except(pickle.UnpicklingError, ImportError, AttributeError, ValueError, TypeError):
            try:
                # for python 2.7 in python 3.4
                # pylint: disable=unexpected-keyword-arg
                val2 = pickle.loads(binString, encoding='bytes')
                val2 = self.bytes_to_unicode(val2)
                return val2
            except(pickle.UnpicklingError, ImportError, AttributeError, ValueError, TypeError):
                g.trace('can not unpickle %s=%s' % (attr, val))
                return val
    #@+node:ekr.20060919110638.14: *4* fc.parse_leo_file
    def parse_leo_file(self, theFile, inputFileName, silent, inClipboard, s=None):
        c = self.c
        try:
            if g.isPython3:
                if theFile:
                    # Use the open binary file, opened by the caller.
                    s = theFile.read() # isinstance(s, bytes)
                    s = self.cleanSaxInputString(s)
                    theFile = BytesIO(s)
                else:
                    s = str(s, encoding='utf-8')
                    s = self.cleanSaxInputString(s)
                    theFile = StringIO(s)
            else:
                if theFile: s = theFile.read()
                s = self.cleanSaxInputString(s)
                theFile = cStringIO.StringIO(s)
            parser = xml.sax.make_parser()
            parser.setFeature(xml.sax.handler.feature_external_ges, 1)
                # Include external general entities, esp. xml-stylesheet lines.
            if 0: # Expat does not read external features.
                parser.setFeature(xml.sax.handler.feature_external_pes, 1)
                    # Include all external parameter entities
                    # Hopefully the parser can figure out the encoding from the <?xml> element.
            # It's very hard to do anything meaningful wih an exception.
            handler = SaxContentHandler(c, inputFileName, silent, inClipboard)
            parser.setContentHandler(handler)
            parser.parse(theFile) # expat does not support parseString
            # g.trace('parsing done')
            sax_node = handler.getRootNode()
        except Exception:
            g.error('error parsing', inputFileName)
            g.es_exception()
            sax_node = None
        return sax_node
    #@+node:ekr.20060919110638.3: *4* fc.readSaxFile
    def readSaxFile(self, theFile, fileName, silent, inClipboard, reassignIndices, s=None):
        '''Read the entire .leo file using the sax parser.'''
        dump = False and not g.unitTesting
        fc = self; c = fc.c
        # Pass one: create the intermediate nodes.
        saxRoot = fc.parse_leo_file(theFile, fileName,
            silent=silent, inClipboard=inClipboard, s=s)
        if dump: fc.dumpSaxTree(saxRoot, dummy=True)
        # Pass two: create the tree of vnodes from the intermediate nodes.
        if saxRoot:
            parent_v = c.hiddenRootNode
            children = fc.createSaxChildren(saxRoot, parent_v)
            assert c.hiddenRootNode.children == children
            v = children and children[0] or None
            return v
        else:
            return None
    #@+node:ekr.20060919110638.11: *4* fc.resolveTnodeLists
    def resolveTnodeLists(self):
        '''
        Called *before* reading external files.
        '''
        trace = False and not g.unitTesting
        c = self.c
        for p in c.all_unique_positions():
            if hasattr(p.v, 'tempTnodeList'):
                # g.trace(p.v.headString())
                result = []
                for tnx in p.v.tempTnodeList:
                    index = self.canonicalTnodeIndex(tnx)
                    # new gnxs:
                    index = g.toUnicode(index)
                    v = self.gnxDict.get(index)
                    if v:
                        if trace: g.trace(tnx, v)
                        result.append(v)
                    else:
                        g.trace('*** No VNode for %s' % tnx)
                if result:
                    p.v.tnodeList = result
                    # g.trace('*** tnodeList for',p.h,result)
                delattr(p.v, 'tempTnodeList')
    #@+node:ekr.20080805132422.3: *4* fc.resolveArchivedPosition
    def resolveArchivedPosition(self, archivedPosition, root_v):
        '''Return a VNode corresponding to the archived position relative to root node root_v.'''

        def oops(message):
            if not g.app.unitTesting:
                g.error('bad archived position: %s' % (message))

        try:
            aList = [int(z) for z in archivedPosition.split('.')]
            aList.reverse()
        except Exception:
            return oops('"%s"' % archivedPosition)
        if not aList:
            return oops('empty')
        last_v = root_v
        n = aList.pop()
        if n != 0:
            return oops('root index="%s"' % n)
        while aList:
            n = aList.pop()
            children = last_v.children
            if n < len(children):
                last_v = children[n]
            else:
                return oops('bad index="%s", len(children)="%s"' % (n, len(children)))
        return last_v
    #@+node:ekr.20060919110638.13: *4* fc.setPositionsFromVnodes & helper (sax read)
    def setPositionsFromVnodes(self):
        trace = False and not g.unitTesting
        c, root = self.c, self.c.rootPosition()
        current, str_pos = None, None
        use_db = g.enableDB and c.mFileName
        if use_db:
            str_pos = c.cacher.getCachedStringPosition()
            if trace: g.trace('cached str_pos', str_pos)
        if not str_pos:
            d = root.v.u
            if trace: g.trace(d)
            if d: str_pos = d.get('str_leo_pos')
            if trace: g.trace('p.v.u', str_pos)
        if str_pos:
            current = self.archivedPositionToPosition(str_pos)
            if trace: g.trace(current and current.h)
        elif trace:
            g.trace('no str_pos!')
        c.setCurrentPosition(current or c.rootPosition())
    #@+node:ekr.20061006104837.1: *5* fc.archivedPositionToPosition
    def archivedPositionToPosition(self, s):
        trace = True and not g.unitTesting
        c = self.c
        s = g.toUnicode(s) # 2011/02/25
        aList = s.split(',')
        try:
            aList = [int(z) for z in aList]
        except Exception:
            if trace: g.trace('not all ints:', aList)
            aList = None
        if not aList: return None
        p = c.rootPosition(); level = 0
        while level < len(aList):
            i = aList[level]
            while i > 0:
                if p.hasNext():
                    p.moveToNext()
                    i -= 1
                else:
                    # g.trace('oops: bad archived position. no sibling:',aList,p.h,c)
                    return None
            level += 1
            if level < len(aList):
                p.moveToFirstChild()
                # g.trace('level',level,'index',aList[level],p.h)
        return p
    #@+node:ekr.20031218072017.3032: ** fc.Writing
    #@+node:ekr.20070413045221.2: *3*  fc.Top-level
    #@+node:ekr.20031218072017.1720: *4* fc.save
    def save(self, fileName, silent=False):
        c = self.c; v = c.currentVnode()
        # New in 4.2.  Return ok flag so shutdown logic knows if all went well.
        ok = g.doHook("save1", c=c, p=v, v=v, fileName=fileName)
        if ok is None:
            c.endEditing() # Set the current headline text.
            self.setDefaultDirectoryForNewFiles(fileName)
            c.cacher.save(fileName, changeName=True)
            ok = c.checkFileTimeStamp(fileName)
            if ok:
                ok = self.write_Leo_file(fileName, False) # outlineOnlyFlag
            if ok:
                if not silent:
                    self.putSavedMessage(fileName)
                c.setChanged(False) # Clears all dirty bits.
                if c.config.save_clears_undo_buffer:
                    g.es("clearing undo")
                    c.undoer.clearUndoState()
            c.redraw_after_icons_changed()
        g.doHook("save2", c=c, p=v, v=v, fileName=fileName)
        return ok
    #@+node:ekr.20031218072017.3043: *4* fc.saveAs
    def saveAs(self, fileName):
        c = self.c; p = c.p
        if not g.doHook("save1", c=c, p=p, v=p, fileName=fileName):
            c.endEditing() # Set the current headline text.
            self.setDefaultDirectoryForNewFiles(fileName)
            c.cacher.save(fileName, changeName=True)
            # Disable path-changed messages in writeAllHelper.
            c.ignoreChangedPaths = True
            try:
                if self.write_Leo_file(fileName, outlineOnlyFlag=False):
                    c.setChanged(False) # Clears all dirty bits.
                    self.putSavedMessage(fileName)
            finally:
                c.ignoreChangedPaths = True
            c.redraw_after_icons_changed()
        g.doHook("save2", c=c, p=p, v=p, fileName=fileName)
    #@+node:ekr.20031218072017.3044: *4* fc.saveTo
    def saveTo(self, fileName):
        c = self.c; p = c.p
        if not g.doHook("save1", c=c, p=p, v=p, fileName=fileName):
            c.endEditing() # Set the current headline text.
            self.setDefaultDirectoryForNewFiles(fileName)
            c.cacher.save(fileName, changeName=False)
            # Disable path-changed messages in writeAllHelper.
            c.ignoreChangedPaths = True
            try:
                self.write_Leo_file(fileName, outlineOnlyFlag=False)
            finally:
                c.ignoreChangedPaths = False
            self.putSavedMessage(fileName)
            c.redraw_after_icons_changed()
        g.doHook("save2", c=c, p=p, v=p, fileName=fileName)
    #@+node:ekr.20070413061552: *4* fc.putSavedMessage
    def putSavedMessage(self, fileName):
        c = self.c
        zipMark = '[zipped] ' if c.isZipped else ''
        g.es("saved:", "%s%s" % (zipMark, g.shortFileName(fileName)))
    #@+node:ekr.20050404190914.2: *3* fc.deleteFileWithMessage
    def deleteFileWithMessage(self, fileName, unused_kind):
        try:
            os.remove(fileName)
        except Exception:
            if self.read_only:
                g.error("read only")
            if not g.unitTesting:
                g.error("exception deleting backup file:", fileName)
                g.es_exception(full=False)
            return False
    #@+node:ekr.20031218072017.1470: *3* fc.put & helpers
    def put(self, s):
        '''Put string s to self.outputFile. All output eventually comes here.'''
        # Improved code: self.outputFile (a cStringIO object) always exists.
        if s:
            # if g.unitTesting: g.trace(g.callers(1),repr(s))
            self.putCount += 1
            if not g.isPython3:
                s = g.toEncodedString(s, self.leo_file_encoding, reportErrors=True)
            self.outputFile.write(s)
    #@+node:ekr.20141020112451.18329: *4* put_dquote
    def put_dquote(self):
        self.put('"')
    #@+node:ekr.20141020112451.18330: *4* put_dquoted_bool
    def put_dquoted_bool(self, b):
        if b: self.put('"1"')
        else: self.put('"0"')
    #@+node:ekr.20141020112451.18331: *4* put_flag
    def put_flag(self, a, b):
        if a:
            self.put(" "); self.put(b); self.put('="1"')
    #@+node:ekr.20141020112451.18332: *4* put_in_dquotes
    def put_in_dquotes(self, a):
        self.put('"')
        if a: self.put(a) # will always be True if we use backquotes.
        else: self.put('0')
        self.put('"')
    #@+node:ekr.20141020112451.18333: *4* put_nl
    def put_nl(self):
        self.put("\n")
    #@+node:ekr.20141020112451.18334: *4* put_tab
    def put_tab(self):
        self.put("\t")
    #@+node:ekr.20141020112451.18335: *4* put_tabs
    def put_tabs(self, n):
        while n > 0:
            self.put("\t")
            n -= 1
    #@+node:ekr.20031218072017.1971: *3* fc.putClipboardHeader
    def putClipboardHeader(self):
        # Put the minimum header for sax.
        self.put('<leo_header file_format="2"/>\n')
    #@+node:ekr.20040324080819.1: *3* fc.putLeoFile & helpers
    def putLeoFile(self):
        self.updateFixedStatus()
        self.putProlog()
        self.putHeader()
        self.putGlobals()
        self.putPrefs()
        self.putFindSettings()
        #start = g.getTime()
        self.putVnodes()
        #start = g.printDiffTime("vnodes ",start)
        self.putTnodes()
        #start = g.printDiffTime("tnodes ",start)
        self.putPostlog()
    #@+node:ekr.20031218072017.3035: *4* fc.putFindSettings
    def putFindSettings(self):
        # New in 4.3:  These settings never get written to the .leo file.
        self.put("<find_panel_settings/>")
        self.put_nl()
    #@+node:ekr.20031218072017.3037: *4* fc.putGlobals
    # Changed for Leo 4.0.

    def putGlobals(self):
        trace = False and not g.unitTesting
        c = self.c
        use_db = g.enableDB and c.mFileName
        if use_db:
            if trace: g.trace(c.mFileName)
            c.cacher.setCachedGlobalsElement(c.mFileName)
        # Always put positions, to trigger sax methods.
        self.put("<globals")
        #@+<< put the body/outline ratios >>
        #@+node:ekr.20031218072017.3038: *5* << put the body/outline ratios >>
        self.put(" body_outline_ratio=")
        self.put_in_dquotes("0.5" if c.fixed or use_db else "%1.2f" % (
            c.frame.ratio))
        self.put(" body_secondary_ratio=")
        self.put_in_dquotes("0.5" if c.fixed or use_db else "%1.2f" % (
            c.frame.secondary_ratio))
        if trace: g.trace('fixed or use_db', c.fixed or use_db,
            '%1.2f %1.2f' % (c.frame.ratio, c.frame.secondary_ratio))
        #@-<< put the body/outline ratios >>
        self.put(">"); self.put_nl()
        #@+<< put the position of this frame >>
        #@+node:ekr.20031218072017.3039: *5* << put the position of this frame >>
        # New in Leo 4.5: support fixed .leo files.
        if c.fixed or use_db:
            width, height, left, top = 700, 500, 50, 50
                # Put fixed, immutable, reasonable defaults.
                # Leo 4.5 and later will ignore these when reading.
                # These should be reasonable defaults so that the
                # file will be opened properly by older versions
                # of Leo that do not support fixed .leo files.
        else:
            width, height, left, top = c.frame.get_window_info()
        # g.trace(width,height,left,top)
        self.put_tab()
        self.put("<global_window_position")
        self.put(" top="); self.put_in_dquotes(str(top))
        self.put(" left="); self.put_in_dquotes(str(left))
        self.put(" height="); self.put_in_dquotes(str(height))
        self.put(" width="); self.put_in_dquotes(str(width))
        self.put("/>"); self.put_nl()
        #@-<< put the position of this frame >>
        #@+<< put the position of the log window >>
        #@+node:ekr.20031218072017.3040: *5* << put the position of the log window >>
        top = left = height = width = 0 # no longer used
        self.put_tab()
        self.put("<global_log_window_position")
        self.put(" top="); self.put_in_dquotes(str(top))
        self.put(" left="); self.put_in_dquotes(str(left))
        self.put(" height="); self.put_in_dquotes(str(height))
        self.put(" width="); self.put_in_dquotes(str(width))
        self.put("/>"); self.put_nl()
        #@-<< put the position of the log window >>
        self.put("</globals>"); self.put_nl()
    #@+node:ekr.20031218072017.3041: *4* fc.putHeader
    def putHeader(self):
        tnodes = 0; clone_windows = 0 # Always zero in Leo2.
        if 1: # For compatibility with versions before Leo 4.5.
            self.put("<leo_header")
            self.put(" file_format="); self.put_in_dquotes("2")
            self.put(" tnodes="); self.put_in_dquotes(str(tnodes))
            self.put(" max_tnode_index="); self.put_in_dquotes(str(0))
            self.put(" clone_windows="); self.put_in_dquotes(str(clone_windows))
            self.put("/>"); self.put_nl()
        else:
            self.put('<leo_header file_format="2"/>\n')
    #@+node:ekr.20031218072017.3042: *4* fc.putPostlog
    def putPostlog(self):
        self.put("</leo_file>"); self.put_nl()
    #@+node:ekr.20031218072017.2066: *4* fc.putPrefs
    def putPrefs(self):
        # New in 4.3:  These settings never get written to the .leo file.
        self.put("<preferences/>")
        self.put_nl()
    #@+node:ekr.20031218072017.1246: *4* fc.putProlog
    def putProlog(self):
        '''Put the prolog of the xml file.'''
        c = self.c
        # tag = 'http://www.leo-editor.org/2011/leo'
        tag = 'http://leoeditor.com/namespaces/leo-python-editor/1.1'
        self.putXMLLine()
        # Put "created by Leo" line.
        self.put('<!-- Created by Leo: http://leoeditor.com/leo_toc.html -->')
        self.put_nl()
        self.putStyleSheetLine()
        # Put the namespace
        self.put('<leo_file xmlns:leo="%s" >' % tag)
        self.put_nl()
    #@+node:ekr.20031218072017.1248: *4* fc.putStyleSheetLine
    def putStyleSheetLine(self):
        '''
        Put the xml stylesheet line.

        Leo 5.3:
        - Use only the stylesheet setting, ignoreing c.frame.stylesheet.
        - Write no stylesheet element if there is no setting.
        
        The old way made it almost impossible to delete stylesheet element.
        '''
        trace = False and not g.unitTesting
        c = self.c
        sheet = (c.config.getString('stylesheet') or '').strip()
        # sheet2 = c.frame.stylesheet and c.frame.stylesheet.strip() or ''
        # sheet = sheet or sheet2
        if sheet:
            s = '<?xml-stylesheet %s ?>' % sheet
            if trace: g.trace(c.shortFileName(), s)
            self.put(s)
            self.put_nl()
    #@+node:ekr.20031218072017.1577: *4* fc.putTnode
    def putTnode(self, v):
        # Call put just once.
        gnx = v.fileIndex
        ua = hasattr(v, 'unknownAttributes') and self.putUnknownAttributes(v) or ''
        b = v.b
        body = xml.sax.saxutils.escape(b) if b else ''
        self.put('<t tx="%s"%s>%s</t>\n' % (gnx, ua, body))
    #@+node:ekr.20031218072017.1575: *4* fc.putTnodes
    def putTnodes(self):
        """Puts all tnodes as required for copy or save commands"""
        self.put("<tnodes>\n")
        self.putReferencedTnodes()
        self.put("</tnodes>\n")
    #@+node:ekr.20031218072017.1576: *5* fc.putReferencedTnodes
    def putReferencedTnodes(self):
        '''Put all referenced tnodes.'''
        c = self.c
        if self.usingClipboard: # write the current tree.
            theIter = c.p.self_and_subtree()
        else: # write everything
            theIter = c.all_unique_positions()
        # Populate tnodes
        tnodes = {}
        for p in theIter:
            # Make *sure* the file index has the proper form.
            # pylint: disable=unbalanced-tuple-unpacking
            index = p.v.fileIndex
            tnodes[index] = p.v
        # Put all tnodes in index order.
        for index in sorted(tnodes):
            # g.trace(index)
            v = tnodes.get(index)
            if v:
                # Write only those tnodes whose vnodes were written.
                # **Note**: @<file> trees are not written unless they contain clones.
                if v.isWriteBit():
                    self.putTnode(v)
            else:
                g.trace('can not happen: no VNode for', repr(index))
                # This prevents the file from being written.
                raise BadLeoFile('no VNode for %s' % repr(index))
    #@+node:ekr.20031218072017.1863: *4* fc.putVnode
    def putVnode(self, p, isIgnore=False):
        """Write a <v> element corresponding to a VNode."""
        fc = self; c = fc.c; v = p.v
        isAuto = p.isAtAutoNode() and p.atAutoNodeName().strip()
        isEdit = p.isAtEditNode() and p.atEditNodeName().strip() and not p.hasChildren()
            # 2010/09/02: @edit nodes must not have children.
            # If they do, the entire tree is written to the outline.
        isFile = p.isAtFileNode()
        isShadow = p.isAtShadowFileNode()
        isThin = p.isAtThinFileNode()
        isOrphan = p.isOrphan()
        if not isIgnore:
            isIgnore = p.isAtIgnoreNode()
        # 2010/10/22: force writes of orphan @edit, @auto and @shadow trees.
        if isIgnore: forceWrite = True # Always write full @ignore trees.
        elif isAuto: forceWrite = isOrphan # Force write of orphan @auto trees.
        elif isEdit: forceWrite = isOrphan # Force write of orphan @edit trees.
        elif isFile: forceWrite = isOrphan # Force write of orphan @file trees.
        elif isShadow: forceWrite = isOrphan # Force write of @shadow trees.
        elif isThin: forceWrite = isOrphan # Force write of  orphan @thin trees.
        else: forceWrite = True # Write all other @<file> trees.
        gnx = v.fileIndex
        if forceWrite or self.usingClipboard:
            v.setWriteBit() # 4.2: Indicate we wrote the body text.
        attrs = []
        #@+<< Append attribute bits to attrs >>
        #@+node:ekr.20031218072017.1865: *5* << Append attribute bits to attrs >> (fc.putVnode)
        # These string catenations are benign because they rarely happen.
        attr = ""
        # New in Leo 4.5: support fixed .leo files.
        if not c.fixed:
            if v.isExpanded() and v.hasChildren() and c.putBitsFlag:
                attr += "E"
            if v.isMarked(): attr += "M"
            # if v.isOrphan(): attr += "O"
                # New in Leo 5.2: never write the orphan bit.
                # It's useless in all cases.
            if attr:
                attrs.append(' a="%s"' % attr)
        # Put the archived *current* position in the *root* positions <v> element.
        if p == self.rootPosition:
            aList = [str(z) for z in self.currentPosition.archivedPosition()]
            d = v.u
            str_pos = ','.join(aList)
            if d.get('str_leo_pos'):
                del d['str_leo_pos']
            # Don't write the current position if we can cache it.
            if g.enableDB and c.mFileName:
                c.cacher.setCachedStringPosition(str_pos)
            elif c.fixed:
                pass
            else:
                d['str_leo_pos'] = str_pos
            v.u = d
        elif hasattr(v, "unknownAttributes"):
            d = v.unknownAttributes
            if d and not c.fixed and d.get('str_leo_pos'):
                # g.trace("clearing str_leo_pos",v)
                del d['str_leo_pos']
                v.unknownAttributes = d
        #@-<< Append attribute bits to attrs >>
        #@+<< Append unKnownAttributes to attrs >>
        #@+node:ekr.20040324082713: *5* << Append unKnownAttributes to attrs>> (fc.putVnode)
        # v.unknownAttributes are now put in <t> elements.
        if p.hasChildren() and not forceWrite and not self.usingClipboard:
            # We put the entire tree when using the clipboard, so no need for this.
            if not isAuto: # Bug fix: 2008/8/7.
                attrs.append(self.putDescendentVnodeUas(p)) # New in Leo 4.5.
                attrs.append(self.putDescendentAttributes(p))
        #@-<< Append unKnownAttributes to attrs >>
        attrs = ''.join(attrs)
        v_head = '<v t="%s"%s>' % (gnx, attrs)
        if gnx in fc.vnodesDict:
            fc.put(v_head + '</v>\n')
        else:
            fc.vnodesDict[gnx] = True
            v_head += '<vh>%s</vh>' % (xml.sax.saxutils.escape(p.v.headString() or ''))
            # New in 4.2: don't write child nodes of @file-thin trees
            # (except when writing to clipboard)
            if p.hasChildren() and (forceWrite or self.usingClipboard):
                fc.put('%s\n' % v_head)
                # This optimization eliminates all "recursive" copies.
                p.moveToFirstChild()
                while 1:
                    fc.putVnode(p, isIgnore)
                    if p.hasNext(): p.moveToNext()
                    else: break
                p.moveToParent() # Restore p in the caller.
                fc.put('</v>\n')
            else:
                fc.put('%s</v>\n' % v_head) # Call put only once.
    #@+node:ekr.20031218072017.1579: *4* fc.putVnodes
    def putVnodes(self):
        """Puts all <v> elements in the order in which they appear in the outline."""
        c = self.c
        c.clearAllVisited()
        self.put("<vnodes>\n")
        # Make only one copy for all calls.
        self.currentPosition = c.p
        self.rootPosition = c.rootPosition()
        # self.topPosition     = c.topPosition()
        self.vnodesDict = {}
        if self.usingClipboard:
            self.putVnode(self.currentPosition) # Write only current tree.
        else:
            for p in c.rootPosition().self_and_siblings():
                # New in Leo 4.4.2 b2 An optimization:
                self.putVnode(p, isIgnore=p.isAtIgnoreNode()) # Write the next top-level node.
        self.put("</vnodes>\n")
    #@+node:ekr.20031218072017.1247: *4* fc.putXMLLine
    def putXMLLine(self):
        '''Put the **properly encoded** <?xml> element.'''
        # Use self.leo_file_encoding encoding.
        self.put('%s"%s"%s\n' % (
            g.app.prolog_prefix_string,
            self.leo_file_encoding,
            g.app.prolog_postfix_string))
    #@+node:ekr.20031218072017.1573: *3* fc.putLeoOutline (to clipboard)
    def putLeoOutline(self):
        '''
        Return a string, *not unicode*, encoded with self.leo_file_encoding,
        suitable for pasting to the clipboard.
        '''
        self.outputFile = g.FileLikeObject()
        self.usingClipboard = True
        self.putProlog()
        self.putClipboardHeader()
        self.putVnodes()
        self.putTnodes()
        self.putPostlog()
        s = self.outputFile.getvalue()
        self.outputFile = None
        self.usingClipboard = False
        return s
    #@+node:ekr.20031218072017.3046: *3* fc.write_Leo_file & helpers
    def write_Leo_file(self, fileName, outlineOnlyFlag, toString=False, toOPML=False):
        '''Write the .leo file.'''
        c, fc = self.c, self
        structure_errors = c.checkOutline()
        if structure_errors:
            g.error('Major structural errors! outline not written')
            return False
        if not outlineOnlyFlag or toOPML:
            g.app.recentFilesManager.writeRecentFilesFile(c)
            fc.writeAllAtFileNodesHelper() # Ignore any errors.
        if fc.isReadOnly(fileName):
            return False
        try:
            fc.putCount = 0
            fc.toString = toString
            if toString:
                ok = fc.writeToStringHelper(fileName)
            else:
                ok = fc.writeToFileHelper(fileName, toOPML)
        finally:
            fc.outputFile = None
            fc.toString = False
        return ok

    write_LEO_file = write_Leo_file # For compatibility with old plugins.
    #@+node:ekr.20040324080359.1: *4* fc.isReadOnly
    def isReadOnly(self, fileName):
        # self.read_only is not valid for Save As and Save To commands.
        if g.os_path_exists(fileName):
            try:
                if not os.access(fileName, os.W_OK):
                    g.error("can not write: read only:", fileName)
                    return True
            except Exception:
                pass # os.access() may not exist on all platforms.
        return False
    #@+node:ekr.20100119145629.6114: *4* fc.writeAllAtFileNodesHelper
    def writeAllAtFileNodesHelper(self):
        '''Write all @<file> nodes and set orphan bits.'''
        c = self.c
        try:
            # 2010/01/19: Do *not* signal failure here.
            # This allows Leo to quit properly.
            c.atFileCommands.writeAll()
            return True
        except Exception:
            # Work around bug 1260415: https://bugs.launchpad.net/leo-editor/+bug/1260415
            g.es_error("exception writing external files")
            g.es_exception()
            g.es('Internal error writing one or more external files.', color='red')
            g.es('Please report this error to:', color='blue')
            g.es('https://groups.google.com/forum/#!forum/leo-editor', color='blue')
            g.es('All changes will be lost unless you', color='red')
            g.es('can save each changed file.', color='red')
            return False
    #@+node:ekr.20100119145629.6111: *4* fc.writeToFileHelper & helpers
    def writeToFileHelper(self, fileName, toOPML):
        c = self.c; toZip = c.isZipped
        ok, backupName = self.createBackupFile(fileName)
        if not ok: return False
        fileName, theActualFile = self.createActualFile(fileName, toOPML, toZip)
        if not theActualFile: return False
        self.mFileName = fileName
        self.outputFile = StringIO() # Always write to a string.
        try:
            if toOPML:
                if hasattr(c, 'opmlController'):
                    c.opmlController.putToOPML(owner=self)
                else:
                    # This is not likely ever to be called.
                    g.trace('leoOPML plugin not active.')
            else:
                self.putLeoFile()
            s = self.outputFile.getvalue()
            g.app.write_Leo_file_string = s # 2010/01/19: always set this.
            if toZip:
                self.writeZipFile(s)
            else:
                if g.isPython3:
                    s = bytes(s, self.leo_file_encoding, 'replace')
                theActualFile.write(s)
                theActualFile.close()
                c.setFileTimeStamp(fileName)
                # raise AttributeError # To test handleWriteLeoFileException.
                # Delete backup file.
                if backupName and g.os_path_exists(backupName):
                    self.deleteFileWithMessage(backupName, 'backup')
            return True
        except Exception:
            self.handleWriteLeoFileException(
                fileName, backupName, theActualFile)
            return False
    #@+node:ekr.20100119145629.6106: *5* fc.createActualFile
    def createActualFile(self, fileName, toOPML, toZip):
        if toZip:
            self.toString = True
            theActualFile = None
        else:
            try:
                # 2010/01/21: always write in binary mode.
                theActualFile = open(fileName, 'wb')
            except IOError:
                g.es('can not open %s' % fileName)
                g.es_exception()
                theActualFile = None
        return fileName, theActualFile
    #@+node:ekr.20031218072017.3047: *5* fc.createBackupFile
    def createBackupFile(self, fileName):
        '''
            Create a closed backup file and copy the file to it,
            but only if the original file exists.
        '''
        if g.os_path_exists(fileName):
            fd, backupName = tempfile.mkstemp(text=False)
            f = open(fileName, 'rb') # rb is essential.
            s = f.read()
            f.close()
            try:
                try:
                    os.write(fd, s)
                finally:
                    os.close(fd)
                ok = True
            except Exception:
                g.error('exception creating backup file')
                g.es_exception()
                ok, backupName = False, None
            if not ok and self.read_only:
                g.error("read only")
        else:
            ok, backupName = True, None
        return ok, backupName
    #@+node:ekr.20100119145629.6108: *5* fc.handleWriteLeoFileException
    def handleWriteLeoFileException(self, fileName, backupName, theActualFile):
        c = self.c
        g.es("exception writing:", fileName)
        g.es_exception(full=True)
        if theActualFile:
            theActualFile.close()
        # Delete fileName.
        if fileName and g.os_path_exists(fileName):
            self.deleteFileWithMessage(fileName, '')
        # Rename backupName to fileName.
        if backupName and g.os_path_exists(backupName):
            g.es("restoring", fileName, "from", backupName)
            # No need to create directories when restoring.
            g.utils_rename(c, backupName, fileName)
        else:
            g.error('backup file does not exist!', repr(backupName))
    #@+node:ekr.20100119145629.6110: *4* fc.writeToStringHelper
    def writeToStringHelper(self, fileName):
        try:
            self.mFileName = fileName
            self.outputFile = StringIO()
            self.putLeoFile()
            s = self.outputFile.getvalue()
            g.app.write_Leo_file_string = s
            return True
        except Exception:
            g.es("exception writing:", fileName)
            g.es_exception(full=True)
            g.app.write_Leo_file_string = ''
            return False
    #@+node:ekr.20070412095520: *4* fc.writeZipFile
    def writeZipFile(self, s):
        # The name of the file in the archive.
        contentsName = g.toEncodedString(
            g.shortFileName(self.mFileName),
            self.leo_file_encoding, reportErrors=True)
        # The name of the archive itself.
        fileName = g.toEncodedString(
            self.mFileName,
            self.leo_file_encoding, reportErrors=True)
        # Write the archive.
        theFile = zipfile.ZipFile(fileName, 'w', zipfile.ZIP_DEFLATED)
        theFile.writestr(contentsName, s)
        theFile.close()
    #@+node:ekr.20031218072017.2012: *3* fc.writeAtFileNodes
    @cmd('write-at-file-nodes')
    def writeAtFileNodes(self, event=None):
        '''Write all @file nodes in the selected outline.'''
        c = self.c
        c.init_error_dialogs()
        c.atFileCommands.writeAll(writeAtFileNodesFlag=True)
        c.raise_error_dialogs(kind='write')
    #@+node:ekr.20080801071227.5: *3* fc.writeAtShadowNodes
    def writeAtShadowNodes(self, event=None):
        '''Write all @file nodes in the selected outline.'''
        c = self.c
        c.init_error_dialogs()
        c.atFileCommands.writeAll(writeAtFileNodesFlag=True)
        c.raise_error_dialogs(kind='write')
    #@+node:ekr.20031218072017.1666: *3* fc.writeDirtyAtFileNodes
    @cmd('write-dirty-at-file-nodes')
    def writeDirtyAtFileNodes(self, event=None):
        '''Write all changed @file Nodes.'''
        c = self.c
        c.init_error_dialogs()
        c.atFileCommands.writeAll(writeDirtyAtFileNodesFlag=True)
        c.raise_error_dialogs(kind='write')
    #@+node:ekr.20080801071227.6: *3* fc.writeDirtyAtShadowNodes
    def writeDirtyAtShadowNodes(self, event=None):
        '''Write all changed @shadow Nodes.'''
        self.c.atFileCommands.writeDirtyAtShadowNodes()
    #@+node:ekr.20031218072017.2013: *3* fc.writeMissingAtFileNodes
    @cmd('write-missing-at-file-nodes')
    def writeMissingAtFileNodes(self, event=None):
        '''Write all @file nodes for which the corresponding external file does not exist.'''
        c = self.c
        if c.p:
            c.atFileCommands.writeMissing(c.p)
    #@+node:ekr.20031218072017.3050: *3* fc.writeOutlineOnly
    @cmd('write-outline-only')
    def writeOutlineOnly(self, event=None):
        '''Write the entire outline without writing any derived files.'''
        c = self.c
        c.endEditing()
        self.write_Leo_file(self.mFileName, outlineOnlyFlag=True)
        g.blue('done')
    #@+node:ekr.20080805114146.2: ** fc.Utils
    #@+node:ekr.20031218072017.1570: *3* fc.assignFileIndices & compactFileIndices
    def assignFileIndices(self):
        """Assign a file index to all tnodes"""
        pass # No longer needed: we assign indices as needed.
    # Indices are now immutable, so there is no longer any difference between these two routines.

    compactFileIndices = assignFileIndices
    #@+node:ekr.20080805085257.1: *3* fc.createUaList
    def createUaList(self, aList):
        '''Given aList of pairs (p,torv), return a list of pairs (torv,d)
        where d contains all picklable items of torv.unknownAttributes.'''
        result = []
        for p, torv in aList:
            if isinstance(torv.unknownAttributes, dict):
                # Create a new dict containing only entries that can be pickled.
                d = dict(torv.unknownAttributes) # Copy the dict.
                for key in d:
                    # Just see if val can be pickled.  Suppress any error.
                    ok = self.pickle(torv=torv, val=d.get(key), tag=None)
                    if not ok:
                        del d[key]
                        g.warning("ignoring bad unknownAttributes key", key, "in", p.h)
                if d:
                    result.append((torv, d),)
            else:
                g.warning("ignoring non-dictionary uA for", p)
        return result
    #@+node:ekr.20080805085257.2: *3* fc.pickle
    def pickle(self, torv, val, tag):
        '''Pickle val and return the hexlified result.'''
        trace = False and g.unitTesting
        try:
            s = pickle.dumps(val, protocol=1)
            s2 = binascii.hexlify(s)
            s3 = g.ue(s2, 'utf-8')
            if trace: g.trace('\n',
                type(val), val, '\n', type(s), repr(s), '\n',
                type(s2), s2, '\n', type(s3), s3)
            field = ' %s="%s"' % (tag, s3)
            return field
        except pickle.PicklingError:
            if tag: # The caller will print the error if tag is None.
                g.warning("ignoring non-pickleable value", val, "in", torv)
            return ''
        except Exception:
            g.error("fc.pickle: unexpected exception in", torv)
            g.es_exception()
            return ''
    #@+node:ekr.20040701065235.2: *3* fc.putDescendentAttributes
    def putDescendentAttributes(self, p):
        nodeIndices = g.app.nodeIndices
        # Create lists of all tnodes whose vnodes are marked or expanded.
        marks = []; expanded = []
        for p in p.subtree():
            v = p.v
            if p.isMarked() and p.v not in marks:
                marks.append(v)
            if p.hasChildren() and p.isExpanded() and v not in expanded:
                expanded.append(v)
        result = []
        for theList, tag in ((marks, "marks"), (expanded, "expanded")):
            if theList:
                sList = []
                for v in theList:
                    sList.append("%s," % v.fileIndex)
                s = ''.join(sList)
                # g.trace(tag,[str(p.h) for p in theList])
                result.append('\n%s="%s"' % (tag, s))
        return ''.join(result)
    #@+node:ekr.20080805071954.2: *3* fc.putDescendentVnodeUas
    def putDescendentVnodeUas(self, p):
        '''Return the a uA field for descendent VNode attributes,
        suitable for reconstituting uA's for anonymous vnodes.'''
        trace = False
        if trace: g.trace(p.h)
        # Create aList of tuples (p,v) having a valid unknownAttributes dict.
        # Create dictionary: keys are vnodes, values are corresonding archived positions.
        pDict = {}; aList = []
        for p2 in p.self_and_subtree():
            if hasattr(p2.v, "unknownAttributes"):
                aList.append((p2.copy(), p2.v),)
                pDict[p2.v] = p2.archivedPosition(root_p=p)
        # Create aList of pairs (v,d) where d contains only pickleable entries.
        if aList: aList = self.createUaList(aList)
        if not aList: return ''
        # Create d, an enclosing dict to hold all the inner dicts.
        d = {}
        for v, d2 in aList:
            aList2 = [str(z) for z in pDict.get(v)]
            # g.trace(aList2)
            key = '.'.join(aList2)
            d[key] = d2
        if trace: g.trace(p.h, g.dictToString(d))
        # Pickle and hexlify d
        return d and self.pickle(
            torv=p.v, val=d, tag='descendentVnodeUnknownAttributes') or ''
    #@+node:ekr.20050418161620.2: *3* fc.putUaHelper
    def putUaHelper(self, torv, key, val):
        '''Put attribute whose name is key and value is val to the output stream.'''
        # New in 4.3: leave string attributes starting with 'str_' alone.
        if key.startswith('str_'):
            if g.isString(val) or g.isBytes(val):
                val = g.toUnicode(val)
                attr = ' %s="%s"' % (key, xml.sax.saxutils.escape(val))
                return attr
            else:
                g.trace(type(val), repr(val))
                g.warning("ignoring non-string attribute", key, "in", torv)
                return ''
        else:
            return self.pickle(torv=torv, val=val, tag=key)
    #@+node:EKR.20040526202501: *3* fc.putUnknownAttributes
    def putUnknownAttributes(self, torv):
        """Put pickleable values for all keys in torv.unknownAttributes dictionary."""
        attrDict = torv.unknownAttributes
        if isinstance(attrDict, dict):
            val = ''.join(
                [self.putUaHelper(torv, key, val)
                    for key, val in attrDict.items()])
            return val
        else:
            g.warning("ignoring non-dictionary unknownAttributes for", torv)
            return ''
    #@+node:ekr.20031218072017.3045: *3* fc.setDefaultDirectoryForNewFiles
    def setDefaultDirectoryForNewFiles(self, fileName):
        """Set c.openDirectory for new files for the benefit of leoAtFile.scanAllDirectives."""
        c = self.c
        if not c.openDirectory:
            theDir = g.os_path_dirname(fileName)
            if theDir and g.os_path_isabs(theDir) and g.os_path_exists(theDir):
                c.openDirectory = c.frame.openDirectory = theDir
    #@+node:ekr.20080412172151.2: *3* fc.updateFixedStatus
    def updateFixedStatus(self):
        c = self.c
        p = c.config.findSettingsPosition('@bool fixedWindow')
        if p:
            import leo.core.leoConfig as leoConfig
            parser = leoConfig.SettingsTreeParser(c)
            kind, name, val = parser.parseHeadline(p.h)
            if val and val.lower() in ('true', '1'):
                val = True
            else:
                val = False
            c.fixed = val
        # g.trace('c.fixed',c.fixed)
    #@-others
#@@language python
#@@tabwidth -4
#@@pagewidth 70
#@-leo
