#!/usr/local/bin/python3.12 -s
#
# Copyright (c) 2010-2018 Shaun McCance <shaunm@gnome.org>
#
# ITS Tool program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by the
# Free Software Foundation, either version 3 of the License, or (at your
# option) any later version.
#
# ITS Tool is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
# for more details.
#
# You should have received a copy of the GNU General Public License along
# with ITS Tool; if not, write to the Free Software Foundation, 59 Temple
# Place, Suite 330, Boston, MA  0211-1307  USA.
#
from __future__ import print_function
from __future__ import unicode_literals

VERSION="2.0.7"
DATADIR="/usr/local/share"

import gettext
import hashlib
from copy import deepcopy
from lxml import etree
import optparse
import os
import os.path
import re
import sys
import time
import io

PY3 = sys.version_info[0] == 3
if PY3:
    string_types = str,
    def ustr(s, encoding=None):
        if isinstance(s, str):
            return s
        elif encoding:
            return str(s, encoding)
        else:
            return str(s)
    ustr_type = str
else:
    string_types = basestring,
    ustr = ustr_type = unicode

NS_ITS = 'http://www.w3.org/2005/11/its'
NS_ITST = 'http://itstool.org/extensions/'
NS_BLANK = 'http://itstool.org/extensions/blank/'
NS_XLINK = 'http://www.w3.org/1999/xlink'
NS_XML = 'http://www.w3.org/XML/1998/namespace'

class NoneTranslations:
    def gettext(self, message):
        return None

    def lgettext(self, message):
        return None

    def ngettext(self, msgid1, msgid2, n):
        return None

    def lngettext(self, msgid1, msgid2, n):
        return None

    def ugettext(self, message):
        return None

    def ungettext(self, msgid1, msgid2, n):
        return None


class MessageList (object):
    def __init__ (self):
        self._messages = []
        self._by_node = {}
        self._has_credits = False

    def add_message (self, message, node):
        self._messages.append (message)
        if node is not None:
            self._by_node[node] = message

    def add_credits(self):
        if self._has_credits:
            return
        msg = Message()
        msg.set_context('_')
        msg.add_text('translator-credits')
        msg.add_comment(Comment('Put one translator per line, in the form NAME <EMAIL>, YEAR1, YEAR2'))
        self._messages.append(msg)
        self._has_credits = True

    def get_message_by_node (self, node):
        return self._by_node.get(node, None)

    def get_nodes_with_messages (self):
        return list(self._by_node.keys())

    def output (self, out):
        msgs = []
        msgdict = {}
        for msg in self._messages:
            key = (msg.get_context(), msg.get_string())
            if key in msgdict:
                for source in msg.get_sources():
                    msgdict[key].add_source(source)
                for marker in msg.get_markers():
                    msgdict[key].add_marker(marker)
                for comment in msg.get_comments():
                    msgdict[key].add_comment(comment)
                for idvalue in msg.get_id_values():
                    msgdict[key].add_id_value(idvalue)
                if msg.get_preserve_space():
                    msgdict[key].set_preserve_space()
                if msg.get_locale_filter() is not None:
                    locale = msgdict[key].get_locale_filter()
                    if locale is not None:
                        msgdict[key].set_locale_filter('%s, %s' % (locale, msg.get_locale_filter()))
                    else:
                        msgdict[key].set_locale_filter(msg.get_locale_filter())

            else:
                msgs.append(msg)
                msgdict[key] = msg
        out.write('msgid ""\n')
        out.write('msgstr ""\n')
        out.write('"Project-Id-Version: PACKAGE VERSION\\n"\n')
        out.write('"POT-Creation-Date: %s\\n"\n' % time.strftime("%Y-%m-%d %H:%M%z"))
        out.write('"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\\n"\n')
        out.write('"Last-Translator: FULL NAME <EMAIL@ADDRESS>\\n"\n')
        out.write('"Language-Team: LANGUAGE <LL@li.org>\\n"\n')
        out.write('"MIME-Version: 1.0\\n"\n')
        out.write('"Content-Type: text/plain; charset=UTF-8\\n"\n')
        out.write('"Content-Transfer-Encoding: 8bit\\n"\n')
        out.write('\n')
        for msg in msgs:
            out.write(msg.format())
            out.write('\n')


class Comment (object):
    def __init__ (self, text):
        self._text = ustr(text)
        assert(text is not None)
        self._markers = []

    def add_marker (self, marker):
        self._markers.append(marker)

    def get_markers (self):
        return self._markers

    def get_text (self):
        return self._text

    def format (self):
        ret = ''
        markers = {}
        for marker in self._markers:
            if marker not in markers:
                ret += '#. (itstool) comment: ' + marker + '\n'
                markers[marker] = marker
        if '\n' in self._text:
            doadd = False
            for line in self._text.split('\n'):
                if line != '':
                    doadd = True
                if not doadd:
                    continue
                ret += '#. %s\n' % line
        else:
            text = self._text
            while len(text) > 72:
                j = text.rfind(' ', 0, 72)
                if j == -1:
                    j = text.find(' ')
                if j == -1:
                    break
                ret += '#. %s\n' % text[:j]
                text = text[j+1:]
            ret += '#. %s\n' % text
        return ret


class Placeholder (object):
    def __init__ (self, node):
        self.node = node
        self.name = ustr(xml_localname(node), 'utf-8')


class Message (object):
    def __init__ (self):
        self._message = []
        self._empty = True
        self._ctxt = None
        self._placeholders = []
        self._sources = []
        self._markers = []
        self._id_values = []
        self._locale_filter = None
        self._comments = []
        self._preserve = False

    def __repr__(self):
        if self._empty:
            return "Empty message"
        return self.get_string()

    def escape (self, text):
        return text.replace('\\','\\\\').replace('"', "\\\"").replace("\n","\\n").replace("\t","\\t")

    def add_text (self, text):
        if len(self._message) == 0 or not(isinstance(self._message[-1], string_types)):
            self._message.append('')
        if not isinstance(text, ustr_type):
            text = ustr(text, 'utf-8')
        self._message[-1] += text.replace('&', '&amp;').replace('<', '&lt;').replace('>', '&gt;')
        if re.sub(r'\s+', ' ', text).strip() != '':
            self._empty = False

    def add_entity_ref (self, name):
        self._message.append('&' + name + ';')
        self._empty = False

    def add_placeholder (self, node):
        holder = Placeholder(node)
        self._placeholders.append(holder)
        self._message.append(holder)

    def get_placeholder (self, name):
        placeholder = 1
        for holder in self._placeholders:
            holdername = '%s-%i' % (holder.name, placeholder)
            if holdername == ustr(name, 'utf-8'):
                return holder
            placeholder += 1

    def add_start_tag (self, node):
        if len(self._message) == 0 or not(isinstance(self._message[-1], string_types)):
            self._message.append('')
        self._message[-1] += ('<%s' % ustr(xml_qname(node), 'utf-8'))
        for name, atval in node.items():
            qname = etree.QName(name)
            if qname.namespace is not None:
                # lxml doesn't expose the prefix of attributes, so we use
                # an XPath expression to get the attribute's prefixed name.
                # This is horribly inefficient.
                expr = 'name(@*[local-name()="%s" and namespace-uri()="%s"])' % (
                    qname.localname, qname.namespace)
                name = node.xpath(expr)
            if not isinstance(atval, ustr_type):
                atval = ustr(atval, 'utf-8')
            atval = atval.replace('&', '&amp;').replace('<', '&lt;').replace('>', '&gt;').replace('"', '&quot;')
            self._message += " %s=\"%s\"" % (name, atval)
        if len(node) > 0 or node.text:
            self._message[-1] += '>'
        else:
            self._message[-1] += '/>'

    def add_end_tag (self, node):
        if len(node) > 0 or node.text:
            if len(self._message) == 0 or not(isinstance(self._message[-1], string_types)):
                self._message.append('')
            self._message[-1] += ('</%s>' % ustr(xml_qname(node), 'utf-8'))

    def is_empty (self):
        return self._empty

    def get_context (self):
        return self._ctxt

    def set_context (self, ctxt):
        self._ctxt = ctxt

    def add_source (self, source):
        if not isinstance(source, ustr_type):
            source = ustr(source, 'utf-8')
        self._sources.append(source)

    def get_sources (self):
        return self._sources

    def add_marker (self, marker):
        if not isinstance(marker, ustr_type):
            marker = ustr(marker, 'utf-8')
        self._markers.append(marker)

    def get_markers (self):
        return self._markers

    def add_id_value(self, id_value):
        self._id_values.append(id_value)

    def get_id_values(self):
        return self._id_values

    def add_comment (self, comment):
        if comment is not None:
            self._comments.append(comment)

    def get_comments (self):
        return self._comments

    def get_string (self):
        message = ''
        placeholder = 1
        for msg in self._message:
            if isinstance(msg, string_types):
                message += msg
            elif isinstance(msg, Placeholder):
                message += '<_:%s-%i/>' % (msg.name, placeholder)
                placeholder += 1
        if not self._preserve:
            message = re.sub(r'\s+', ' ', message).strip()
        return message

    def get_preserve_space (self):
        return self._preserve

    def set_preserve_space (self, preserve=True):
        self._preserve = preserve

    def get_locale_filter(self):
        return self._locale_filter

    def set_locale_filter(self, locale):
        self._locale_filter = locale

    def format (self):
        ret = ''
        markers = {}
        for marker in self._markers:
            if marker not in markers:
                ret += '#. (itstool) path: ' + marker + '\n'
                markers[marker] = marker
        for idvalue in self._id_values:
            ret += '#. (itstool) id: ' + idvalue + '\n'
        if self._locale_filter is not None:
            ret += '#. (itstool) ' + self._locale_filter[1] + ' locale: ' + self._locale_filter[0] + '\n'
        comments = []
        commentsdict = {}
        for comment in self._comments:
            key = comment.get_text()
            if key in commentsdict:
                for marker in comment.get_markers():
                    commentsdict[key].add_marker(marker)
            else:
                comments.append(comment)
                commentsdict[key] = comment
        for i in range(len(comments)):
            if i != 0:
                ret += '#.\n'
            ret += comments[i].format()
        for source in self._sources:
            ret += '#: %s\n' % source
        if self._preserve:
            ret += '#, no-wrap\n'
        if self._ctxt is not None:
            ret += 'msgctxt "%s"\n' % self._ctxt
        message = self.get_string()
        if self._preserve:
            ret += 'msgid ""\n'
            lines = message.split('\n')
            for line, no in zip(lines, list(range(len(lines)))):
                if no == len(lines) - 1:
                    ret += '"%s"\n' % self.escape(line)
                else:
                    ret += '"%s\\n"\n' % self.escape(line)
        else:
            ret += 'msgid "%s"\n' % self.escape(message)
        ret += 'msgstr ""\n'
        return ret


def xml_localname (node):
    return etree.QName(node.tag).localname

def xml_qname (node):
    qname = etree.QName(node.tag).localname
    if node.prefix is not None:
        qname = node.prefix + ':' + qname
    return qname

def xml_content (node):
    if isinstance(node, string_types):
        return node
    if isinstance(node, XMLAttr):
        return node.parent.get(node.tag)
    return etree.tostring(node, method='text', encoding='unicode')

def xml_delete_node (node):
    parent = node.getparent()
    prev = node.getprevious()
    tail = node.tail
    if parent is not None:
        parent.remove(node)
    if prev is not None:
        if prev.tail is None or re.fullmatch(r'\s+', prev.tail):
            prev.tail = tail
        else:
            prev.tail += tail
    elif parent is not None:
        if parent.text is None or re.fullmatch(r'\s+', parent.text):
            parent.text = tail
        else:
            parent.text += tail

def xml_get_node_path(node):
    # The built-in nodePath() method only does numeric indexes
    # when necessary for disambiguation. For various reasons,
    # we prefer always using indexes.
    name = xml_qname(node)
    if isinstance(node, XMLAttr):
        name = '@' + name
    name = '/' + name
    if node.getparent() is not None:
        count = 1
        prev = node.getprevious()
        while prev is not None:
            if prev.tag == node.tag:
                count += 1
            prev = prev.getprevious()
        name = '%s[%i]' % (name, count)
        name = xml_get_node_path(node.getparent()) + name
    return name


# lxml doesn't support attribute nodes, so we have to emulate them.
class XMLAttr (object):
    def __init__(self, element, tag):
        self.parent = element
        self.tag = tag
        self.attrib = {}
        self.sourceline = element.sourceline

    def __repr__(self):
        return '%s@%s' % (repr(self.parent), self.tag)

    def __eq__(self, other):
        return other and self.parent == other.parent and self.tag == other.tag

    def __ne__(self, other):
        return not self.__eq__(other)

    def __hash__(self):
        return hash(repr(self))

    def getparent(self):
        return self.parent

    def get(self, default=None):
        return default


class LocNote (object):
    def __init__(self, locnote=None, locnoteref=None, locnotetype=None, space=False):
        self.locnote = locnote
        self.locnoteref = locnoteref
        self.locnotetype = locnotetype
        if self.locnotetype != 'alert':
            self.locnotetype = 'description'
        self._preserve_space=space

    def __repr__(self):
        if self.locnote is not None:
            if self._preserve_space:
                return self.locnote
            else:
                return re.sub(r'\s+', ' ', self.locnote).strip()
        elif self.locnoteref is not None:
            return '(itstool) link: ' + re.sub(r'\s+', ' ', self.locnoteref).strip()
        return ''


class Document (object):
    def __init__ (self, filename, messages, load_dtd=False, keep_entities=False):
        self._load_dtd = load_dtd
        self._keep_entities = keep_entities
        parser = etree.XMLParser(load_dtd = load_dtd or keep_entities,
                                 resolve_entities = not(keep_entities))
        doc = etree.parse(filename, parser)
        doc.xinclude()
        self._filename = filename
        self._doc = doc
        self._localrules = []
        for child in doc.iter():
            if child.tag == '{' + NS_ITS + '}rules':
                href = child.get('{' + NS_XLINK + '}href')
                if href is not None:
                    fileref = os.path.join(os.path.dirname(filename), href)
                    if not os.path.exists(fileref):
                        if opts.itspath is not None:
                            for pathdir in opts.itspath:
                                fileref = os.path.join(pathdir, href)
                                if os.path.exists(fileref):
                                    break
                    if not os.path.exists(fileref):
                        sys.stderr.write('Error: Could not locate ITS file %s\n' % href)
                        sys.exit(1)
                    root = etree.parse(fileref).getroot()
                    version = None
                    version = root.get('version')
                    if version is None:
                        sys.stderr.write('Warning: ITS file %s missing version attribute\n' %
                                         os.path.basename(href))
                    elif version not in ('1.0', '2.0'):
                        sys.stderr.write('Warning: Skipping ITS file %s with unknown version %s\n' %
                                         (os.path.basename(href), root.get('version')))
                    else:
                        self._localrules.append(root)
                version = child.get('version')
                if version is None:
                    root = child.getroottree()
                    version = root.get('{' + NS_ITS + '}version')
                if version is None:
                    sys.stderr.write('Warning: Local ITS rules missing version attribute\n')
                elif version not in ('1.0', '2.0'):
                    sys.stderr.write('Warning: Skipping local ITS rules with unknown version %s\n' %
                                     version)
                else:
                    self._localrules.append(child)
        self._msgs = messages
        self._its_translate_nodes = {}
        self._its_within_text_nodes = {}
        self._its_locale_filters = {}
        self._its_id_values = {}
        self._its_loc_notes = {}
        self._its_preserve_space_nodes = {}
        self._itst_drop_nodes = {}
        self._itst_contexts = {}
        self._its_lang = {}
        self._itst_lang_attr = {}
        self._itst_credits = None
        self._its_externals = {}

        self._clear_cache()

    def _clear_cache(self):
        self._its_translate_nodes_cache = {}
        self._its_locale_filters_cache = {}
        self._its_loc_notes_cache = {}

    def get_its_params(self, rules):
        params = {}
        for child in rules.iterchildren():
            if child.tag == '{' + NS_ITS + '}param':
                params[child.get('name')] = xml_content(child)
        return params

    def register_its_params(self, var, params, userparams={}):
        for name in params:
            if name in userparams:
                var[name] = userparams[name]
            else:
                var[name] = params[name]

    def apply_its_rule(self, rule, xpath):
        self._clear_cache()
        if rule.tag == '{' + NS_ITS + '}translateRule':
            sel = rule.get('selector')
            if sel is not None:
                for node in self._try_xpath_eval(xpath, sel):
                    self._its_translate_nodes[node] = rule.get('translate')
        elif rule.tag == '{' + NS_ITS + '}withinTextRule':
            sel = rule.get('selector')
            if sel is not None:
                for node in self._try_xpath_eval(xpath, sel):
                    self._its_within_text_nodes[node] = rule.get('withinText')
        elif rule.tag == '{' + NS_ITST + '}preserveSpaceRule':
            sel = rule.get('selector')
            if sel is not None:
                for node in self._try_xpath_eval(xpath, sel):
                    val = rule.get('preserveSpace')
                    if val == 'yes':
                        self._its_preserve_space_nodes[node] = 'preserve'
        elif rule.tag == '{' + NS_ITS + '}preserveSpaceRule':
            sel = rule.get('selector')
            if sel is not None:
                for node in self._try_xpath_eval(xpath, sel):
                    self._its_preserve_space_nodes[node] = rule.get('space')
        elif rule.tag == '{' + NS_ITS + '}localeFilterRule':
            sel = rule.get('selector')
            if sel is not None:
                lst = rule.get('localeFilterList', '*')
                typ = rule.get('localeFilterType', 'include')
                for node in self._try_xpath_eval(xpath, sel):
                    self._its_locale_filters[node] = (lst, typ)
        elif rule.tag == '{' + NS_ITST + '}dropRule':
            sel = rule.get('selector')
            if sel is not None:
                for node in self._try_xpath_eval(xpath, sel):
                    self._itst_drop_nodes[node] = rule.get('drop')
        elif rule.tag == '{' + NS_ITS + '}idValueRule':
            sel = rule.get('selector')
            idv = rule.get('idValue')
            if sel is not None and idv is not None:
                for node in self._try_xpath_eval(xpath, sel):
                    idvalue = self._try_xpath_eval(xpath, idv, node=node)
                    if isinstance(idvalue, string_types):
                        self._its_id_values[node] = idvalue
                    else:
                        for val in idvalue:
                            self._its_id_values[node] = xml_content(val)
                            break
            pass
        elif rule.tag == '{' + NS_ITST + '}contextRule':
            sel = rule.get('selector')
            if sel is not None:
                for node in self._try_xpath_eval(xpath, sel):
                    ctxt = rule.get('context')
                    cp = rule.get('contextPointer')
                    if ctxt is not None:
                        self._itst_contexts[node] = ctxt
                    elif cp is not None:
                        ctxt = self._try_xpath_eval(xpath, cp, node=node)
                        if isinstance(ctxt, string_types):
                            self._itst_contexts[node] = ctxt
                        else:
                            for ctxt in ctxt:
                                self._itst_contexts[node] = xml_content(ctxt)
                                break
        elif rule.tag == '{' + NS_ITS + '}locNoteRule':
            locnote = None
            notetype = rule.get('locNoteType')
            for child in rule.iterchildren('{' + NS_ITS + '}locNote'):
                locnote = LocNote(locnote=xml_content(child), locnotetype=notetype)
                break
            if locnote is None:
                if 'locNoteRef' in rule.attrib:
                    locnote = LocNote(locnoteref=rule.get('locNoteRef'), locnotetype=notetype)
            sel = rule.get('selector')
            if sel is not None:
                for node in self._try_xpath_eval(xpath, sel):
                    if locnote is not None:
                        self._its_loc_notes.setdefault(node, []).append(locnote)
                    else:
                        if 'locNotePointer' in rule.attrib:
                            sel = rule.get('locNotePointer')
                            ref = False
                        elif 'locNoteRefPointer' in rule.attrib:
                            sel = rule.get('locNoteRefPointer')
                            ref = True
                        else:
                            continue
                        note = self._try_xpath_eval(xpath, sel, node=node)
                        if isinstance(note, string_types):
                            if ref:
                                nodenote = LocNote(locnoteref=note, locnotetype=notetype)
                            else:
                                nodenote = LocNote(locnote=note, locnotetype=notetype)
                            self._its_loc_notes.setdefault(node, []).append(nodenote)
                        else:
                            for note in note:
                                text = xml_content(note)
                                if ref:
                                    nodenote = LocNote(locnoteref=text, locnotetype=notetype)
                                else:
                                    nodenote = LocNote(locnote=text, locnotetype=notetype,
                                                       space=self.get_preserve_space(note))
                                self._its_loc_notes.setdefault(node, []).append(nodenote)
                                break
        elif rule.tag == '{' + NS_ITS + '}langRule':
            sel = rule.get('selector')
            lp = rule.get('langPointer')
            if sel is not None and lp is not None:
                for node in self._try_xpath_eval(xpath, sel):
                    res = self._try_xpath_eval(xpath, lp, node=node)
                    if len(res) > 0:
                        self._its_lang[node] = xml_content(res[0])
                    # We need to construct language attributes, not just read
                    # language information. Technically, langPointer could be
                    # any XPath expression. But if it looks like an attribute
                    # accessor, just use the attribute name.
                    # TODO: This should probably be skipped if langPointer
                    # equals '@xml:lang' which is the default.
                    if lp[0] == '@':
                        name = lp[1:]
                        if ':' in name:
                            prefix, lname = name.split(':', 2)
                            nsuri = node.nsmap.get(prefix)
                            if nsuri is None:
                                name = lname
                            else:
                                name = '{' + nsuri + '}' + lname
                        self._itst_lang_attr[node] = name
        elif rule.tag == '{' + NS_ITST + '}credits':
            sel = rule.get('appendTo')
            if sel is not None:
                for node in self._try_xpath_eval(xpath, sel):
                    self._itst_credits = (node, rule)
                    break
        elif (rule.tag == '{' + NS_ITS  + '}externalResourceRefRule' or
              rule.tag == '{' + NS_ITST + '}externalRefRule'):
            sel = rule.get('selector')
            if rule.tag == '{' + NS_ITS + '}externalResourceRefRule':
                ptr = rule.get('externalResourceRefPointer')
            else:
                ptr = rule.get('refPointer')
            if sel is not None and ptr is not None:
                for node in self._try_xpath_eval(xpath, sel):
                    res = self._try_xpath_eval(xpath, ptr, node=node)
                    if len(res) > 0:
                        self._its_externals[node] = xml_content(res[0])

    def apply_its_rules(self, builtins, userparams={}):
        self._clear_cache()
        if builtins:
            dirs = []
            ddir = os.getenv('XDG_DATA_HOME', '')
            if ddir == '':
                ddir = os.path.join(os.path.expanduser('~'), '.local', 'share')
            dirs.append(ddir)
            ddir = os.getenv('XDG_DATA_DIRS', '')
            if ddir == '':
                if DATADIR not in ('/usr/local/share', '/usr/share'):
                    ddir += DATADIR + ':'
                ddir += '/usr/local/share:/usr/share'
            dirs.extend(ddir.split(':'))
            ddone = {}
            for ddir in dirs:
                itsdir = os.path.join(ddir, 'itstool', 'its')
                if not os.path.exists(itsdir):
                    continue
                for dfile in os.listdir(itsdir):
                    if dfile.endswith('.its'):
                        if not ddone.get(dfile, False):
                            self.apply_its_file(os.path.join(itsdir, dfile), userparams=userparams)
                            ddone[dfile] = True
        self.apply_local_its_rules(userparams=userparams)

    def apply_its_file(self, filename, userparams={}):
        self._clear_cache()
        parser = etree.XMLParser(resolve_entities = False)
        root = etree.parse(filename, parser).getroot()
        if root.tag != '{' + NS_ITS + '}rules':
            return
        version = root.get('version')
        if version is None:
            sys.stderr.write('Warning: ITS file %s missing version attribute\n' %
                             os.path.basename(filename))
        elif version not in ('1.0', '2.0'):
            sys.stderr.write('Warning: Skipping ITS file %s with unknown version %s\n' %
                             (os.path.basename(filename), root.get('version')))
            return
        matched = True
        for match in root.iterchildren():
            if match.tag == '{' + NS_ITST + '}match':
                matched = False
                sel = match.get('selector')
                if sel is not None:
                    ns = { k: v for k, v in match.nsmap.items() if k is not None }
                    xpath = (ns, {})
                    if len(self._try_xpath_eval(xpath, sel)) > 0:
                        matched = True
                        break
        if matched == False:
            return
        ns = { k: v for k, v in match.nsmap.items() if k is not None }
        var = {}
        params = self.get_its_params(root)
        self.register_its_params(var, params, userparams=userparams)
        xpath = (ns, var)
        for rule in root.iterchildren():
            self.apply_its_rule(rule, xpath)

    def apply_local_its_rules(self, userparams={}):
        self._clear_cache()
        for rules in self._localrules:
            var = {}
            params = self.get_its_params(rules)
            self.register_its_params(var, params, userparams=userparams)
            for rule in rules.iterchildren():
                ns = { k: v for k, v in rule.nsmap.items() if k is not None }
                rule_xpath = (ns, var)
                self.apply_its_rule(rule, rule_xpath)

    def _append_credits(self, parent, node, trdata):
        if node.tag == '{' + NS_ITST + '}for-each':
            select = node.get('select')
            if select == 'years':
                for year in trdata[2].split(','):
                    for child in node.iterchildren():
                        self._append_credits(parent, child, trdata + (year.strip(),))
        elif node.tag == '{' + NS_ITST + '}value-of':
            select = node.get('select')
            val = None
            if select == 'name':
                val = trdata[0]
            elif select == 'email':
                val = trdata[1]
            elif select == 'years':
                val = trdata[2]
            elif select == 'year' and len(trdata) == 4:
                val = trdata[3]
            if val is not None:
                if not PY3:
                    val = val.encode('utf-8')
                if len(parent):
                    if parent[-1].tail:
                        parent[-1].tail += val
                    else:
                        parent[-1].tail = val
                else:
                    if parent.text:
                        parent.text += val
                    else:
                        parent.text = val
        else:
            newnode = parent.makeelement(node.tag, node.attrib)
            parent.append(newnode)
            for child in node.iterchildren():
                self._append_credits(newnode, child, trdata)

    def merge_credits(self, translations, language, node):
        if self._itst_credits is None:
            return
        # Dear Python, please implement pgettext.
        # http://bugs.python.org/issue2504
        # Sincerely, Shaun
        trans = translations.ugettext('_\x04translator-credits')
        if trans is None or trans == 'translator-credits':
            return
        regex = re.compile(r'(.*) \<(.*)\>, (.*)')
        for credit in trans.split('\n'):
            match = regex.match(credit)
            if not match:
                continue
            trdata = match.groups()
            for node in self._itst_credits[1].iterchildren():
                self._append_credits(self._itst_credits[0], node, trdata)

    def join_translations(self, translations, node=None, strict=False):
        is_root = False
        if node is None:
            is_root = True
            self.generate_messages(comments=False)
            node = self._doc.getroot()
        if node is None:
            return
        if self.get_itst_drop(node) == 'yes':
            xml_delete_node(node)
            return
        msg = self._msgs.get_message_by_node(node)
        if msg is None:
            #self.translate_attrs(node, node)
            for child in node.iterchildren():
                self.join_translations(translations, node=child, strict=strict)
        else:
            prevtext = None
            prev = node.getprevious()
            if prev is None:
                parent = node.getparent()
                if parent is not None:
                    prevtext = parent.text
            else:
                prevtext = prev.tail
            if prevtext is not None:
                if not re.fullmatch(r'\s+', prevtext):
                    prevtext = None
            i = 0
            for lang in sorted(list(translations.keys()), reverse=True):
                locale = self.get_its_locale_filter(node)
                lmatch = match_locale_list(locale[0], lang)
                if (locale[1] == 'include' and not lmatch) or (locale[1] == 'exclude' and lmatch):
                    continue
                newnode = self.get_translated(node, translations[lang], strict=strict, lang=lang)
                if newnode != node:
                    newnode.set('{' + NS_XML + '}lang', lang)
                    node.addnext(newnode)
                    if i == 0:
                        # Move tail to first new node
                        newnode.tail = node.tail
                        if prevtext is not None:
                            node.tail = prevtext
                    else:
                        if prevtext is not None:
                            newnode.tail = prevtext
                    i += 1

    def merge_translations(self, translations, language, node=None, strict=False):
        is_root = False
        if node is None:
            is_root = True
            self.generate_messages(comments=False)
            node = self._doc.getroot()
        if node is None:
            return
        drop = False
        locale = self.get_its_locale_filter(node)
        if locale[1] == 'include':
            if locale[0] != '*':
                if not match_locale_list(locale[0], language):
                    drop = True
        elif locale[1] == 'exclude':
            if match_locale_list(locale[0], language):
                drop = True
        if self.get_itst_drop(node) == 'yes' or drop:
            xml_delete_node(node)
            return
        if is_root:
            self.merge_credits(translations, language, node)
        msg = self._msgs.get_message_by_node(node)
        if msg is None:
            self.translate_attrs(node, node)
            for child in node.iterchildren():
                self.merge_translations(translations, language, node=child, strict=strict)
        else:
            newnode = self.get_translated(node, translations, strict=strict, lang=language)
            if newnode != node:
                self.translate_attrs(node, newnode)
                newnode.tail = node.tail
                parent = node.getparent()
                if parent is not None:
                    parent.replace(node, newnode)
        if is_root:
            # Apply language attributes to untranslated nodes. We don't do
            # this before processing, because then these attributes would
            # be copied into the new nodes. We apply the attribute without
            # checking whether it was translated, because any that were will
            # just be floating around, unattached to a document.
            for lcnode in self._msgs.get_nodes_with_messages():
                attr = self._itst_lang_attr.get(lcnode)
                if attr is None:
                    continue
                origlang = None
                lcpar = lcnode
                while lcpar is not None:
                    origlang = self._its_lang.get(lcpar)
                    if origlang is not None:
                        break
                    lcpar = lcpar.getparent()
                if origlang is not None:
                    lcnode.set(attr, origlang)
            # And then set the language attribute on the root node.
            if language is not None:
                attr = self._itst_lang_attr.get(node)
                if attr is not None:
                    node.set(attr, language)

    def translate_attrs(self, oldnode, newnode):
        for attrname, srccontent in oldnode.items():
            attr = XMLAttr(oldnode, attrname)
            if self._its_translate_nodes.get(attr, 'no') != 'yes':
                continue
            if not PY3:
                srccontent = srccontent.decode('utf-8')
            newcontent = translations.ugettext(srccontent)
            if newcontent:
                if not PY3:
                    newcontent = newcontent.encode('utf-8')
                newnode.set(attrname, newcontent)

    def get_translated (self, node, translations, strict=False, lang=None):
        msg = self._msgs.get_message_by_node(node)
        if msg is None:
            return node
        msgstr = msg.get_string()
        # Dear Python, please implement pgettext.
        # http://bugs.python.org/issue2504
        # Sincerely, Shaun
        if msg.get_context() is not None:
            msgstr = msg.get_context() + '\x04' + msgstr
        trans = translations.ugettext(msgstr)
        if trans is None:
            return node
        blurb = ''
        doc = node.getroottree()
        if doc.docinfo.internalDTD:
            # This is an ugly hack to serialize the DTD. We copy the
            # document, replace the document element, serialize the
            # document and remove the last line which contains the
            # document element, leaving only the DTD.
            copy = deepcopy(doc)
            root = copy.getroot()
            newroot = root.makeelement(root.tag)
            copy._setroot(newroot)
            blurb = re.sub('.*$', '', etree.tostring(copy, encoding='unicode'))
        localname = ustr(xml_localname(node), 'utf-8')
        blurb += '<' + localname
        blurb += ' xmlns:_="%s"' % NS_BLANK
        for nsname, nsuri in node.nsmap.items():
            if nsname is None:
                blurb += ' xmlns="%s"' % nsuri
            else:
                blurb += ' xmlns:%s="%s"' % (nsname, nsuri)
        blurb += '>%s</%s>' % (trans, localname)
        parser = etree.XMLParser(load_dtd = self._load_dtd or self._keep_entities,
                                 resolve_entities = not(self._keep_entities))
        try:
            trnode = etree.fromstring(blurb, parser)
        except:
            if strict:
                raise
            else:
                sys.stderr.write('Warning: Could not merge %stranslation for msgid:\n%s\n' % (
                    (lang + ' ') if lang is not None else '',
                    msgstr.encode('utf-8')))
                return node
        try:
            for child in trnode.iterdescendants():
                if isinstance(child, (etree._Entity, etree._Comment, etree._ProcessingInstruction)):
                    continue
                qname = etree.QName(child.tag)
                if qname.namespace == NS_BLANK:
                    ph = msg.get_placeholder(qname.localname)
                    if ph is None:
                        sys.stderr.write('Warning: Could not find placeholder %s\n' % (
                            qname.localname))
                        continue
                    ph_node = ph.node
                    if len(ph_node):
                        self.merge_translations(translations, None, ph_node, strict=strict)
                        newnode = deepcopy(ph_node)
                        newnode.tail = child.tail
                        child.getparent().replace(child, newnode)
                    else:
                        repl = self.get_translated(ph_node, translations, strict=strict, lang=lang)
                        repl.tail = child.tail
                        child.getparent().replace(child, repl)
        except:
            raise
            if strict:
                raise
            else:
                sys.stderr.write('Warning: Could not merge %stranslation for msgid:\n%s\n' % (
                    (lang + ' ') if lang is not None else '',
                    msgstr.encode('utf-8')))
                return node
        retnode = self._doc.getroot().makeelement(node.tag, node.attrib, node.nsmap)
        retnode.text = trnode.text
        for child in trnode.iterchildren():
            retnode.append(child)

        return retnode

    def generate_messages(self, comments=True):
        if self._itst_credits is not None:
            self._msgs.add_credits()
        if self._doc is not None:
            self.generate_message(self._doc.getroot(), None, comments=comments)

    def generate_message(self, node, msg, comments=True, path=None):
        if isinstance(node, etree._Entity):
            msg.add_entity_ref(node.name)
            return
        # Only allow elements
        if isinstance(node, XMLAttr) or not isinstance(node.tag, str):
            return
        if node.get('{' + NS_ITST + '}drop', 'no') == 'yes':
            return
        if self._itst_drop_nodes.get(node, 'no') == 'yes':
            return
        locfil = self.get_its_locale_filter(node)
        if locfil == ('', 'include') or locfil == ('*', 'exclude'):
            return
        if path is None:
            path = ''
        translate = self.get_its_translate(node)
        withinText = False
        if translate == 'no':
            if msg is not None:
                msg.add_placeholder(node)
            is_unit = False
            msg = None
        else:
            is_unit = msg is None or self.is_translation_unit(node)
            if is_unit:
                if msg is not None:
                    msg.add_placeholder(node)
                msg = Message()
                ctxt = node.get('{' + NS_ITST + '}context')
                if ctxt is None:
                    ctxt = self._itst_contexts.get(node)
                if ctxt is not None:
                    msg.set_context(ctxt)
                idvalue = self.get_its_id_value(node)
                if idvalue is not None:
                    basename = os.path.basename(self._filename)
                    msg.add_id_value(basename + '#' + idvalue)
                if self.get_preserve_space(node):
                    msg.set_preserve_space()
                if self.get_its_locale_filter(node) != ('*', 'include'):
                    msg.set_locale_filter(self.get_its_locale_filter(node))
                msg.add_source('%s:%i' % (self._doc.docinfo.URL, node.sourceline))
                parent = node.getparent()
                if parent is None:
                    ptag = '#root'
                else:
                    ptag = xml_localname(parent)
                msg.add_marker('%s/%s' % (ustr(ptag, 'utf-8'), ustr(xml_localname(node), 'utf-8')))
            else:
                withinText = True
                msg.add_start_tag(node)

        if not withinText:
            # Add msg for translatable node attributes
            for attrname, attrval in node.items():
                attr = XMLAttr(node, attrname)
                if self._its_translate_nodes.get(attr, 'no') == 'yes':
                    attr_msg = Message()
                    if self.get_preserve_space(attr):
                        attr_msg.set_preserve_space()
                    attr_msg.add_source('%s:%i' % (self._doc.docinfo.URL, node.sourceline))
                    attr_msg.add_marker('%s/%s@%s' % (
                        xml_localname(node.getparent()),
                        xml_localname(node),
                        etree.QName(attrname).localname))
                    attr_msg.add_text(attrval)
                    if comments:
                        for locnote in self.get_its_loc_notes(attr):
                            comment = Comment(locnote)
                            comment.add_marker ('%s/%s@%s' % (
                                xml_localname(node.getparent()),
                                xml_localname(node),
                                etree.QName(attrname).localname))
                            attr_msg.add_comment(comment)
                    self._msgs.add_message(attr_msg, attr)

        if comments and msg is not None:
            cnode = node
            while cnode is not None:
                hasnote = False
                for locnote in self.get_its_loc_notes(cnode, inherit=(not withinText)):
                    comment = Comment(locnote)
                    if withinText:
                        comment.add_marker('.%s/%s' % (path, xml_localname(cnode)))
                    msg.add_comment(comment)
                    hasnote = True
                if hasnote or not is_unit:
                    break
                cnode = cnode.getparent()

        self.generate_external_resource_message(node)
        for attrname in node.keys():
            attr = XMLAttr(node, attrname)
            self.generate_external_resource_message(attr)
            idvalue = self.get_its_id_value(attr)
            if idvalue is not None:
                basename = os.path.basename(self._filename)
                msg.add_id_value(basename + '#' + idvalue)

        if withinText:
            path = path + '/' + node.tag
        if node.text is not None and msg is not None:
            msg.add_text(node.text)
        for child in node.iterchildren():
            self.generate_message(child, msg, comments=comments, path=path)
            if child.tail is not None and msg is not None:
                msg.add_text(child.tail)

        if translate:
            if is_unit and not msg.is_empty():
                self._msgs.add_message(msg, node)
            elif msg is not None:
                msg.add_end_tag(node)

    def generate_external_resource_message(self, node):
        if node not in self._its_externals:
            return
        resref = self._its_externals[node]
        if isinstance(node, XMLAttr):
            elem = node.getparent()
            translate = self.get_its_translate(elem)
            marker = '%s/%s/@%s' % (
                xml_localname(elem.getparent()),
                xml_localname(elem),
                xml_localname(node))
        else:
            translate = self.get_its_translate(node)
            marker = '%s/%s' % (xml_localname(node.getparent()),
                                xml_localname(node))
        if translate == 'no':
            return
        msg = Message()
        try:
            fullfile = os.path.join(os.path.dirname(self._filename), resref)
            filefp = open(fullfile, 'rb')
            filemd5 = hashlib.md5(filefp.read()).hexdigest()
            filefp.close()
        except Exception:
            filemd5 = '__failed__'
        txt = "external ref='%s' md5='%s'" % (resref, filemd5)
        msg.set_context('_')
        msg.add_text(txt)
        msg.add_source('%s:%i' % (self._doc.docinfo.URL, node.sourceline))
        msg.add_marker(marker)
        msg.add_comment(Comment('This is a reference to an external file such as an image or'
                                ' video. When the file changes, the md5 hash will change to'
                                ' let you know you need to update your localized copy. The'
                                ' msgstr is not used at all. Set it to whatever you like'
                                ' once you have updated your copy of the file.'))
        self._msgs.add_message(msg, None)

    def is_translation_unit (self, node):
        return self.get_its_within_text(node) != 'yes'

    def get_preserve_space (self, node):
        while node is not None:
            if node.get('{' + NS_XML + '}space') == 'preserve':
                return True
            if node in self._its_preserve_space_nodes:
                return (self._its_preserve_space_nodes[node] == 'preserve')
            node = node.getparent()
        return False

    def get_its_translate(self, node):
        if node in self._its_translate_nodes_cache:
            return self._its_translate_nodes_cache[node]
        val = None
        if '{' + NS_ITS + '}translate' in node.attrib:
            val = node.get('{' + NS_ITS + '}translate')
        elif node.tag == '{' + NS_ITS + '}span' and 'translate' in node.attrib:
            val = node.get('translate')
        elif node in self._its_translate_nodes:
            val = self._its_translate_nodes[node]
        if val is not None:
            self._its_translate_nodes_cache[node] = val
            return val
        if isinstance(node, XMLAttr):
            return 'no'
        if node.getparent() is not None:
            parval = self.get_its_translate(node.getparent())
            self._its_translate_nodes_cache[node] = parval
            return parval
        return 'yes'

    def get_its_within_text(self, node):
        if '{' + NS_ITS + '}withinText' in node.attrib:
            val = node.get('{' + NS_ITS + '}withinText')
        elif node.tag == '{' + NS_ITS + '}span' and 'withinText' in node.attrib:
            val = node.get('withinText')
        else:
            return self._its_within_text_nodes.get(node, 'no')
        if val in ('yes', 'nested'):
            return val
        return 'no'

    def get_its_locale_filter(self, node):
        if node in self._its_locale_filters_cache:
            return self._its_locale_filters_cache[node]
        if ('{' + NS_ITS + '}localeFilterList' in node.attrib or
            '{' + NS_ITS + '}localeFilterType' in node.attrib):
            lst = node.get('{' + NS_ITS + '}localeFilterList', '*')
            typ = node.get('{' + NS_ITS + '}localeFilterType', 'include')
            return (lst, typ)
        if (node.tag == '{' + NS_ITS + '}span' and
            ('localeFilterList' in node.attrib or 'localeFilterType' in node.attrib)):
            lst = node.get('localeFilterList', '*')
            typ = node.get('localeFilterType', 'include')
            return (lst, typ)
        if node in self._its_locale_filters:
            return self._its_locale_filters[node]
        if node.getparent() is not None:
            parval = self.get_its_locale_filter(node.getparent())
            self._its_locale_filters_cache[node] = parval
            return parval
        return ('*', 'include')

    def get_itst_drop(self, node):
        if node.get('{' + NS_ITST + '}drop') == 'yes':
            return 'yes'
        if self._itst_drop_nodes.get(node, 'no') == 'yes':
            return 'yes'
        return 'no'

    def get_its_id_value(self, node):
        if '{' + NS_XML + '}id' in node.attrib:
            return node.get('{' + NS_XML + '}id')
        return self._its_id_values.get(node, None)

    def get_its_loc_notes(self, node, inherit=True):
        if node in self._its_loc_notes_cache:
            return self._its_loc_notes_cache[node]
        ret = []
        if ( '{' + NS_ITS + '}locNote'     in node.attrib or
             '{' + NS_ITS + '}locNoteRef'  in node.attrib or
             '{' + NS_ITS + '}locNoteType' in node.attrib ):
            notetype = node.get('{' + NS_ITS + '}locNoteType')
            if '{' + NS_ITS + '}locNote' in node.attrib:
                ret.append(LocNote(locnote=node.get('{' + NS_ITS + '}locNote'), locnotetype=notetype))
            elif '{' + NS_ITS + '}locNoteRef' in node.attrib:
                ret.append(LocNote(locnoteref=node.get('{' + NS_ITS + '}locNoteRef'), locnotetype=notetype))
        elif node.tag == '{' + NS_ITS + '}span':
            if ( 'locNote'     in node.attrib or
                 'locNoteRef'  in node.attrib or
                 'locNoteType' in node.attrib ):
                notetype = node.get('locNoteType')
                if 'locNote' in node.attrib:
                    ret.append(LocNote(locnote=node.get('locNote'), locnotetype=notetype))
                elif 'locNoteRef' in node.attrib:
                    ret.append(LocNote(locnoteref=node.get('locNoteRef'), locnotetype=notetype))
        for locnote in reversed(self._its_loc_notes.get(node, [])):
            ret.append(locnote)
        if (len(ret) == 0 and inherit and
            not isinstance(node, XMLAttr) and
            node.getparent() is not None):
            parval = self.get_its_loc_notes(node.getparent())
            self._its_loc_notes_cache[node] = parval
            return parval
        self._its_loc_notes_cache[node] = ret
        return ret

    def output_test_data(self, category, out, node=None):
        if node is None:
            node = self._doc.getroot()
        compval = ''
        if category == 'translate':
            compval = 'translate="%s"' % self.get_its_translate(node)
        elif category == 'withinText':
            if not isinstance(node, XMLAttr):
                compval = 'withinText="%s"' % self.get_its_within_text(node)
        elif category == 'localeFilter':
            compval = 'localeFilterList="%s"\tlocaleFilterType="%s"' % self.get_its_locale_filter(node)
        elif category == 'locNote':
            val = self.get_its_loc_notes(node)
            if len(val) > 0:
                if val[0].locnote is not None:
                    compval = 'locNote="%s"\tlocNoteType="%s"' % (ustr(val[0]), val[0].locnotetype)
                elif val[0].locnoteref is not None:
                    compval = 'locNoteRef="%s"\tlocNoteType="%s"' % (val[0].locnoteref, val[0].locnotetype)
        elif category == 'externalResourceRef':
            val = self._its_externals.get(node, '')
            if val != '':
                compval = 'externalResourceRef="%s"' % val
        elif category == 'idValue':
            val = self.get_its_id_value(node)
            if val is not None:
                compval = 'idValue="%s"' % val
        elif category == 'preserveSpace':
            if self.get_preserve_space(node):
                compval = 'space="preserve"'
            else:
                compval = 'space="default"'
        else:
            sys.stderr.write('Error: Unrecognized category %s\n' % category)
            sys.exit(1)
        if compval != '':
            out.write('%s\t%s\r\n' % (xml_get_node_path(node), compval))
        else:
            out.write('%s\r\n' % (xml_get_node_path(node)))
        for attrname in sorted(node.keys(), key=ustr):
            attr = XMLAttr(node, attrname)
            self.output_test_data(category, out, attr)
        for child in node.iterchildren():
            self.output_test_data(category, out, child)

    def _try_xpath_eval (self, xpath, expr, node=None):
        if node is None:
            node = self._doc
        elif isinstance(node, XMLAttr):
            # lxml doesn't support attributes as XPath context nodes.
            if expr == '.':
                return [ node ]
            sys.stderr.write('Warning: Unsupported XPath on attribute: %s\n' % expr)
            return []
        try:
            result = node.xpath(expr, namespaces=xpath[0], **xpath[1])
            if not isinstance(result, str):
                for i in range(len(result)):
                    val = result[i]
                    # Use lxml's "smart string" feature to determine
                    # the attribute node.
                    if (isinstance(val, etree._ElementUnicodeResult) and
                        val.is_attribute):
                        result[i] = XMLAttr(val.getparent(), val.attrname)
            return result
        except:
            sys.stderr.write('Warning: Invalid XPath: %s\n' % expr)
            return []

def match_locale_list(extranges, locale):
    if extranges.strip() == '':
        return False
    for extrange in [extrange.strip() for extrange in extranges.split(',')]:
        if match_locale(extrange, locale):
            return True
    return False

def match_locale(extrange, locale):
    # Extended filtering for extended language ranges as
    # defined by RFC4647, part of BCP47.
    # http://tools.ietf.org/html/rfc4647#section-3.3.2
    rangelist = [x.lower() for x in extrange.split('-')]
    localelist = [x.lower() for x in locale.split('-')]
    if rangelist[0] not in ('*', localelist[0]):
        return False
    rangei = localei = 0
    while rangei < len(rangelist):
        if rangelist[rangei] == '*':
            rangei += 1
            continue
        if localei >= len(localelist):
            return False
        if rangelist[rangei] in ('*', localelist[localei]):
            rangei += 1
            localei += 1
            continue
        if len(localelist[localei]) == 1:
            return False
        localei += 1
    return True

_locale_pattern = re.compile(r'([a-zA-Z0-9-]+)(_[A-Za-z0-9]+)?(@[A-Za-z0-9]+)?(\.[A-Za-z0-9]+)?')
def convert_locale (locale):
    # Automatically convert POSIX-style locales to BCP47
    match = _locale_pattern.match(locale)
    if match is None:
        return locale
    ret = match.group(1).lower()
    variant = match.group(3)
    if variant == '@cyrillic':
        ret += '-Cyrl'
        variant = None
    if variant == '@devanagari':
        ret += '-Deva'
        variant = None
    elif variant == '@latin':
        ret += '-Latn'
        variant = None
    elif variant == '@shaw':
        ret += '-Shaw'
        variant = None
    if match.group(2) is not None:
        ret += '-' + match.group(2)[1:].upper()
    if variant is not None and variant != '@euro':
        ret += '-' + variant[1:].lower()
    return ret


#def main():
if __name__ == '__main__':
    options = optparse.OptionParser()
    options.set_usage('\n  itstool [OPTIONS] [XMLFILES]\n' +
                      '  itstool -m <MOFILE> [OPTIONS] [XMLFILES]\n' +
                      '  itstool -j <XMLFILE> [OPTIONS] [MOFILES]')
    options.add_option('-i', '--its',
                       action='append',
                       dest='itsfile',
                       metavar='ITS',
                       help='Load the ITS rules in the file ITS (can specify multiple times)')
    options.add_option('-l', '--lang',
                       dest='lang',
                       default=None,
                       metavar='LANGUAGE',
                       help='Explicitly set the language code for output file')
    options.add_option('-j', '--join',
                       dest='join',
                       metavar='FILE',
                       help='Join multiple MO files with the XML file FILE and output XML file')
    options.add_option('-m', '--merge',
                       dest='merge',
                       metavar='FILE',
                       help='Merge from a PO or MO file FILE and output XML files')
    options.add_option('-n', '--no-builtins',
                       action='store_true',
                       dest='nobuiltins',
                       default=False,
                       help='Do not apply the built-in ITS rules')
    options.add_option('-o', '--output',
                       dest='output',
                       default=None,
                       metavar='OUT',
                       help='Output PO files to file OUT or XML files in directory OUT')
    options.add_option('--path',
                       action='append',
                       dest='itspath',
                       default=None,
                       metavar='PATHS',
                       help='Extra path where ITS files may be found (can specify multiple times)')
    options.add_option('-s', '--strict',
                       action='store_true',
                       dest='strict',
                       default=False,
                       help='Exit with error when PO files contain broken XML')
    options.add_option('-d', '--load-dtd',
                       action='store_true',
                       dest='load_dtd',
                       default=False,
                       help='Load external DTDs used by input XML')
    options.add_option('-k', '--keep-entities',
                       action='store_true',
                       dest='keep_entities',
                       default=False,
                       help='Keep entity reference unexpanded')
    options.add_option('-p', '--param',
                       action='append',
                       dest='userparams',
                       default=[],
                       nargs=2,
                       metavar='NAME VALUE',
                       help='Define the ITS parameter NAME to the value VALUE (can specify multiple times)')
    options.add_option('-t', '--test',
                       dest='test',
                       default=None,
                       metavar='CATEGORY',
                       help='Generate conformance test output for CATEGORY')
    options.add_option('-v', '--version',
                       action='store_true',
                       dest='version',
                       default=False,
                       help='Print itstool version and exit')
    (opts, args) = options.parse_args(sys.argv)

    if opts.version:
        print('itstool %s' % VERSION)
        sys.exit(0)

    userparams = {}
    for name, value in opts.userparams:
        userparams[name] = value

    if opts.merge is None and opts.join is None:
        messages = MessageList()
        for filename in args[1:]:
            doc = Document(filename, messages, load_dtd=opts.load_dtd, keep_entities=opts.keep_entities)
            doc.apply_its_rules(not(opts.nobuiltins), userparams=userparams)
            if opts.itsfile is not None:
                for itsfile in opts.itsfile:
                    doc.apply_its_file(itsfile, userparams=userparams)
            if opts.test is None:
                doc.generate_messages()
        if opts.output is None or opts.output == '-':
            out = sys.stdout
        else:
            try:
                out = io.open(opts.output, 'wt', encoding='utf-8')
            except:
                sys.stderr.write('Error: Cannot write to file %s\n' % opts.output)
                sys.exit(1)
        if opts.test is not None:
            doc.output_test_data(opts.test, out)
        else:
            messages.output(out)
            out.flush()
    elif opts.merge is not None:
        try:
            translations = gettext.GNUTranslations(open(opts.merge, 'rb'))
        except:
            sys.stderr.write('Error: cannot open mo file %s\n' % opts.merge)
            sys.exit(1)
        if PY3:
            translations.ugettext = translations.gettext
        translations.add_fallback(NoneTranslations())
        if opts.lang is None:
            opts.lang = convert_locale(os.path.splitext(os.path.basename(opts.merge))[0])
        if opts.output is None:
            out = './'
        elif os.path.isdir(opts.output):
            out = opts.output
        elif len(args) == 2:
            if opts.output == '-':
                out = sys.stdout
            else:
                out = open(opts.output, 'wb')
        else:
            sys.stderr.write('Error: Non-directory output for multiple files\n')
            sys.exit(1)
        for filename in args[1:]:
            messages = MessageList()
            doc = Document(filename, messages, load_dtd=opts.load_dtd, keep_entities=opts.keep_entities)
            doc.apply_its_rules(not(opts.nobuiltins), userparams=userparams)
            if opts.itsfile is not None:
                for itsfile in opts.itsfile:
                    doc.apply_its_file(itsfile, userparams=userparams)
            try:
                doc.merge_translations(translations, opts.lang, strict=opts.strict)
            except Exception as e:
                raise
                sys.stderr.write('Error: Could not merge translations:\n%s\n' % ustr(e))
                sys.exit(1)
            # lxml generates XML declarations with single quotes.
            serialized = (
                b'<?xml version="1.0" encoding="utf-8"?>\n' +
                etree.tostring(doc._doc, encoding='utf-8') +
                b'\n')
            fout = out
            fout_is_str = isinstance(fout, string_types)
            if fout_is_str:
                fout = open(os.path.join(fout, os.path.basename(filename)), 'wb')
            fout.write(serialized)
            fout.flush()
            if fout_is_str:
                fout.close()
    elif opts.join is not None:
        translations = {}
        for filename in args[1:]:
            try:
                thistr = gettext.GNUTranslations(open(filename, 'rb'))
            except:
                sys.stderr.write('Error: cannot open mo file %s\n' % filename)
                sys.exit(1)
            thistr.add_fallback(NoneTranslations())
            if PY3:
                thistr.ugettext = thistr.gettext
            lang = convert_locale(os.path.splitext(os.path.basename(filename))[0])
            translations[lang] = thistr
        if opts.output is None:
            out = sys.stdout
        elif os.path.isdir(opts.output):
            out = open(os.path.join(opts.output, os.path.basename(filename)), 'wb')
        else:
            out = open(opts.output, 'wb')
        messages = MessageList()
        doc = Document(opts.join, messages)
        doc.apply_its_rules(not(opts.nobuiltins), userparams=userparams)
        if opts.itsfile is not None:
            for itsfile in opts.itsfile:
                doc.apply_its_file(itsfile, userparams=userparams)
        doc.join_translations(translations, strict=opts.strict)
        # lxml generates XML declarations with single quotes.
        serialized = (
            b'<?xml version="1.0" encoding="utf-8"?>\n' +
            etree.tostring(doc._doc, encoding='utf-8') +
            b'\n')
        out.write(serialized)
        out.flush()

#if __name__ == '__main__':
#    if os.getenv('ITSTOOL_PROFILE') is not None:
#        import cProfile
#        cProfile.run('main()')
#    else:
#        main()
