"""
@file
@brief Helpers to convert docstring to various format.
"""
import os
import sys
from collections import deque
import warnings
import pickle
import platform
from html import escape as htmlescape
from io import StringIO
from docutils.parsers.rst import roles
from docutils.languages import en as docutils_en
from docutils import nodes
from docutils.utils import Reporter
from sphinx.application import Sphinx
from sphinx.environment import BuildEnvironment
from sphinx.errors import ExtensionError
from sphinx.ext.extlinks import setup_link_roles
from sphinx.transforms import SphinxTransformer
from sphinx.writers.html import HTMLWriter
from sphinx.util.build_phase import BuildPhase
from sphinx.util.logging import prefixed_warnings
from sphinx.project import Project
from sphinx.errors import ApplicationError
from sphinx.util.logging import getLogger
from ..sphinxext.sphinx_doctree_builder import (
    DocTreeBuilder, DocTreeWriter, DocTreeTranslator)
from ..sphinxext.sphinx_md_builder import MdBuilder, MdWriter, MdTranslator
from ..sphinxext.sphinx_latex_builder import (
    EnhancedLaTeXBuilder, EnhancedLaTeXWriter, EnhancedLaTeXTranslator)
from ..sphinxext.sphinx_rst_builder import RstBuilder, RstWriter, RstTranslator
from ._single_file_html_builder import CustomSingleFileHTMLBuilder

def _get_LaTeXTranslator():
    try:
        from sphinx.writers.latex import LaTeXTranslator
    except ImportError:  # pragma: no cover
        # Since sphinx 1.7.3 (circular reference).
        import sphinx.builders.latex.transforms
        from sphinx.writers.latex import LaTeXTranslator
    return LaTeXTranslator

try:
    from sphinx.util.docutils import is_html5_writer_available
except ImportError:
    def is_html5_writer_available():
        return True

if is_html5_writer_available():
    from sphinx.writers.html5 import HTML5Translator as HTMLTranslator
else:
    from sphinx.writers.html import HTMLTranslator  # pragma: no cover

def update_docutils_languages(values=None):
    """
    Updates ``docutils/languages/en.py`` with missing labels.
    It Does it for languages *en*.

    @param      values      consider values in this dictionaries first
    """
    if values is None:
        values = dict()
    lab = docutils_en.labels
    if 'versionmodified' not in lab:
        lab['versionmodified'] = values.get(
            'versionmodified', 'modified version')
    if 'desc' not in lab:
        lab['desc'] = values.get('desc', 'description')

class _AdditionalVisitDepart:
    """
    Additional visitors and departors.
    """

    def __init__(self, output_format):
        self.output_format = output_format

    def is_html(self):
        """
        Tells if the translator is :epkg:`html` format.
        """
        return self.base_class is HTMLTranslator

    def is_rst(self):
        """
        Tells if the translator is :epkg:`rst` format.
        """
        return self.base_class is RstTranslator

    def is_latex(self):
        """
        Tells if the translator is :epkg:`latex` format.
        """
        return self.base_class is _get_LaTeXTranslator()

    def is_md(self):
        """
        Tells if the translator is :epkg:`markdown` format.
        """
        return self.base_class is _get_LaTeXTranslator()

    def is_doctree(self):
        """
        Tells if the translator is doctree format.
        """
        return self.base_class is _get_LaTeXTranslator()

    def add_secnumber(self, node):
        """
        Overwrites this method to catch errors due when
        it is a single document being processed.
        """
        if node.get('secnumber'):
            self.base_class.add_secnumber(self, node)
        elif len(node.parent['ids']) > 0:
            self.base_class.add_secnumber(self, node)
        else:
            n = len(self.builder.secnumbers)
            node.parent['ids'].append("custom_label_%d" % n)
            self.base_class.add_secnumber(self, node)

    def eval_expr(self, expr):
        rst = self.output_format == 'rst'
        latex = self.output_format in ('latex', 'elatex')
        texinfo = [('index', 'A_AdditionalVisitDepart', 'B_AdditionalVisitDepart',   # pylint: disable=W0612
                    'C_AdditionalVisitDepart', 'D_AdditionalVisitDepart',
                    'E_AdditionalVisitDepart', 'Miscellaneous')]
        html = self.output_format == 'html'
        md = self.output_format == 'md'
        doctree = self.output_format in ('doctree', 'doctree.txt')
        if not (rst or html or latex or md or doctree):
            raise ValueError(  # pragma: no cover
                f"Unknown output format '{self.output_format}'.")
        try:
            ev = eval(expr)
        except Exception:  # pragma: no cover
            raise ValueError(
                f"Unable to interpret expression '{expr}'")
        return ev

    def visit_only(self, node):
        ev = self.eval_expr(node.attributes['expr'])
        if ev:
            pass
        else:
            raise nodes.SkipNode

    def depart_only(self, node):
        ev = self.eval_expr(node.attributes['expr'])
        if ev:
            pass
        else:
            # The program should not necessarily be here.
            pass

    def visit_viewcode_anchor(self, node):
        # Removed in sphinx 3.5
        pass

    def depart_viewcode_anchor(self, node):
        # Removed in sphinx 3.5
        pass

    def unknown_visit(self, node):  # pragma: no cover
        raise NotImplementedError(
            "[_AdditionalVisitDepart] Unknown node: '{0}' in '{1}'".format(
                node.__class__.__name__, self.__class__.__name__))

class HTMLTranslatorWithCustomDirectives(_AdditionalVisitDepart, HTMLTranslator):
    """
    See @see cl HTMLWriterWithCustomDirectives.
    """

    def __init__(self, document, builder, *args, **kwds):
        HTMLTranslator.__init__(self, document, builder, *args, **kwds)
        _AdditionalVisitDepart.__init__(self, 'html')
        nodes_list = getattr(builder, '_function_node', None)
        if nodes_list is not None:
            for name, f1, f2 in nodes_list:
                setattr(self.__class__, "visit_" + name, f1)
                setattr(self.__class__, "depart_" + name, f2)
        self.base_class = HTMLTranslator

    def visit_field(self, node):
        if not hasattr(self, '_fieldlist_row_index'):
            # needed when a docstring starts with :param:
            self._fieldlist_row_index = 0
        return HTMLTranslator.visit_field(self, node)

    def visit_pending_xref(self, node):
        self.visit_Text(node)
        raise nodes.SkipNode

    def unknown_visit(self, node):  # pragma: no cover
        raise NotImplementedError("[HTMLTranslatorWithCustomDirectives] Unknown node: '{0}' in '{1}'".format(
            node.__class__.__name__, self.__class__.__name__))

class RSTTranslatorWithCustomDirectives(_AdditionalVisitDepart, RstTranslator):
    """
    See @see cl HTMLWriterWithCustomDirectives.
    """

    def __init__(self, document, builder, *args, **kwds):
        """
        constructor
        """
        RstTranslator.__init__(self, document, builder, *args, **kwds)
        _AdditionalVisitDepart.__init__(self, 'rst')
        for name, f1, f2 in builder._function_node:
            setattr(self.__class__, "visit_" + name, f1)
            setattr(self.__class__, "depart_" + name, f2)
        self.base_class = RstTranslator

class MDTranslatorWithCustomDirectives(_AdditionalVisitDepart, MdTranslator):
    """
    See @see cl HTMLWriterWithCustomDirectives.
    """

    def __init__(self, document, builder, *args, **kwds):
        """
        constructor
        """
        MdTranslator.__init__(self, document, builder, *args, **kwds)
        _AdditionalVisitDepart.__init__(self, 'md')
        for name, f1, f2 in builder._function_node:
            setattr(self.__class__, "visit_" + name, f1)
            setattr(self.__class__, "depart_" + name, f2)
        self.base_class = MdTranslator

class DocTreeTranslatorWithCustomDirectives(DocTreeTranslator):
    """
    See @see cl HTMLWriterWithCustomDirectives.
    """

    def __init__(self, document, builder, *args, **kwds):
        """
        constructor
        """
        DocTreeTranslator.__init__(self, document, builder, *args, **kwds)
        self.base_class = DocTreeTranslator

class LatexTranslatorWithCustomDirectives(_AdditionalVisitDepart, EnhancedLaTeXTranslator):
    """
    See @see cl LatexWriterWithCustomDirectives.
    """

    def __init__(self, document, builder, *args, **kwds):
        """
        constructor
        """
        if not hasattr(builder, "config"):
            builder, document = document, builder
        if not hasattr(builder, "config"):
            raise TypeError(  # pragma: no cover
                f"Builder has no config: {type(builder)} - {type(document)}")
        EnhancedLaTeXTranslator.__init__(
            self, document, builder, *args, **kwds)
        _AdditionalVisitDepart.__init__(self, 'md')
        for name, f1, f2 in builder._function_node:
            setattr(self.__class__, "visit_" + name, f1)
            setattr(self.__class__, "depart_" + name, f2)
        self.base_class = EnhancedLaTeXTranslator

class _WriterWithCustomDirectives:
    """
    Common class to @see cl HTMLWriterWithCustomDirectives and @see cl RSTWriterWithCustomDirectives.
    """

    def _init(self, base_class, translator_class, app=None):
        """
        @param      base_class  base class
        @param      app         Sphinx application
        """
        if app is None:
            self.app = _CustomSphinx(srcdir=None, confdir=None, outdir=None, doctreedir=None,
                                     buildername='memoryhtml')
        else:
            self.app = app
        builder = self.app.builder
        builder.fignumbers = {}
        base_class.__init__(self, builder)
        self.translator_class = translator_class
        self.builder.secnumbers = {}
        self.builder._function_node = []
        self.builder.current_docname = None
        self.base_class = base_class

    def connect_directive_node(self, name, f_visit, f_depart):
        """
        Adds custom node to the translator.

        @param      name        name of the directive
        @param      f_visit     visit function
        @param      f_depart    depart function
        """
        if self.builder.format != "doctree":
            self.builder._function_node.append((name, f_visit, f_depart))

    def add_configuration_options(self, new_options):
        """
        Add new options.

        @param      new_options     new options
        """
        for k, v in new_options.items():
            self.builder.config.values[k] = v

    def write(self, document, destination):
        """
        Processes a document into its final form.
        Translates `document` (a Docutils document tree) into the Writer's
        native format, and write it out to its `destination` (a
        `docutils.io.Output` subclass object).

        Normally not overridden or extended in subclasses.
        """
        self.base_class.write(self, document, destination)

class HTMLWriterWithCustomDirectives(_WriterWithCustomDirectives, HTMLWriter):
    """
    This :epkg:`docutils` writer extends the HTML writer with
    custom directives implemented in this module,
    @see cl RunPythonDirective, @see cl BlogPostDirective.

    See `Write your own ReStructuredText-Writer <http://www.arnebrodowski.de/blog/write-your-own-restructuredtext-writer.html>`_.

    This class needs to tell :epkg:`docutils` to call the added function
    when directives *runpython* or *blogpost* are met.
    """

    def __init__(self, builder=None, app=None):  # pylint: disable=W0231
        """
        @param      builder builder
        @param      app     Sphinx application
        """
        _WriterWithCustomDirectives._init(
            self, HTMLWriter, HTMLTranslatorWithCustomDirectives, app)

    def translate(self):
        self.visitor = visitor = self.translator_class(
            self.document, self.builder)
        self.document.walkabout(visitor)
        self.output = visitor.astext()
        for attr in ('head_prefix', 'stylesheet', 'head', 'body_prefix',
                     'body_pre_docinfo', 'docinfo', 'body', 'fragment',
                     'body_suffix', 'meta', 'title', 'subtitle', 'header',
                     'footer', 'html_prolog', 'html_head', 'html_title',
                     'html_subtitle', 'html_body', ):
            setattr(self, attr, getattr(visitor, attr, None))
        self.clean_meta = ''.join(visitor.meta[2:])

class RSTWriterWithCustomDirectives(_WriterWithCustomDirectives, RstWriter):
    """
    This :epkg:`docutils` writer extends the :epkg:`RST` writer with
    custom directives implemented in this module.
    """

    def __init__(self, builder=None, app=None):  # pylint: disable=W0231
        """
        @param      builder builder
        @param      app     Sphinx application
        """
        _WriterWithCustomDirectives._init(
            self, RstWriter, RSTTranslatorWithCustomDirectives, app)

    def translate(self):
        visitor = self.translator_class(self.document, self.builder)
        self.document.walkabout(visitor)
        self.output = visitor.body

class MDWriterWithCustomDirectives(_WriterWithCustomDirectives, MdWriter):
    """
    This :epkg:`docutils` writer extends the :epkg:`MD` writer with
    custom directives implemented in this module.
    """

    def __init__(self, builder=None, app=None):  # pylint: disable=W0231
        """
        @param      builder builder
        @param      app     Sphinx application
        """
        _WriterWithCustomDirectives._init(
            self, MdWriter, MDTranslatorWithCustomDirectives, app)

    def translate(self):
        visitor = self.translator_class(self.document, self.builder)
        self.document.walkabout(visitor)
        self.output = visitor.body

class DocTreeWriterWithCustomDirectives(_WriterWithCustomDirectives, DocTreeWriter):
    """
    This :epkg:`docutils` writer creates a doctree writer with
    custom directives implemented in this module.
    """

    def __init__(self, builder=None, app=None):  # pylint: disable=W0231
        """
        @param      builder builder
        @param      app     Sphinx application
        """
        _WriterWithCustomDirectives._init(
            self, DocTreeWriter, DocTreeTranslatorWithCustomDirectives, app)

    def translate(self):
        visitor = self.translator_class(self.document, self.builder)
        self.document.walkabout(visitor)
        self.output = visitor.body

class LatexWriterWithCustomDirectives(_WriterWithCustomDirectives, EnhancedLaTeXWriter):
    """
    This :epkg:`docutils` writer extends the :epkg:`Latex` writer with
    custom directives implemented in this module.
    """

    def __init__(self, builder=None, app=None):  # pylint: disable=W0231
        """
        @param      builder builder
        @param      app     Sphinx application
        """
        _WriterWithCustomDirectives._init(
            self, EnhancedLaTeXWriter, LatexTranslatorWithCustomDirectives, app)
        if not hasattr(self.builder, "config"):
            raise TypeError(  # pragma: no cover
                f"Builder has no config: {type(self.builder)}")

    def translate(self):
        if not hasattr(self.builder, "config"):
            raise TypeError(  # pragma: no cover
                f"Builder has no config: {type(self.builder)}")
        # The instruction
        # visitor = self.builder.create_translator(self.document, self.builder)
        # automatically adds methods visit_ and depart_ for translator
        # based on the list of registered extensions. Might be worth using it.
        theme = self.builder.themes.get('manual')
        if theme is None:
            raise RuntimeError(  # pragma: no cover
                "theme cannot be None.")
        visitor = self.translator_class(
            self.document, self.builder, theme=theme)
        self.document.walkabout(visitor)
        self.output = visitor.body

class _MemoryBuilder:
    """
    Builds :epkg:`HTML` output in memory.
    The API is defined by the page
    :epkg:`builderapi`.
    """

    def _init(self, base_class, app, env=None):
        """
        Constructs the builder.
        Most of the parameter are static members of the class and cannot
        be overwritten (yet).

        :param base_class: base builder class
        :param app: :epkg:`Sphinx application`
        :param env: Environment
        """
        if "IMPOSSIBLE:TOFIND" in app.srcdir:
            import sphinx.util.osutil
            from .conf_path_tools import custom_ensuredir
            sphinx.util.osutil.ensuredir = custom_ensuredir
            sphinx.builders.ensuredir = custom_ensuredir

        try:
            base_class.__init__(self, app=app, env=env)
        except TypeError:
            # older version of sphinx
            base_class.__init__(self, app=app)
        self.built_pages = {}
        self.base_class = base_class

    def iter_pages(self):
        """
        Enumerate created pages.

        @return     iterator on tuple(name, content)
        """
        for k, v in self.built_pages.items():
            yield k, v.getvalue()

    def create_translator(self, *args):
        """
        Returns an instance of translator.
        This method returns an instance of ``default_translator_class`` by default.
        Users can replace the translator class with ``app.set_translator()`` API.
        """
        translator_class = self.translator_class
        return translator_class(*args)

    def _write_serial(self, docnames):
        """
        Overwrites *_write_serial* to avoid writing on disk.
        """
        from sphinx.util.logging import pending_warnings
        try:
            from sphinx.util.display import status_iterator
        except ImportError:
            from sphinx.util import status_iterator
        with pending_warnings():
            for docname in status_iterator(docnames, 'writing output... ', "darkgreen",
                                           len(docnames), self.app.verbosity):
                doctree = self.env.get_and_resolve_doctree(docname, self)
                self.write_doc_serialized(docname, doctree)
                self.write_doc(docname, doctree)

    def _write_parallel(self, docnames, nproc):
        """
        Not supported.
        """
        raise NotImplementedError(
            "Use parallel=0 when creating the sphinx application.")

    def assemble_doctree(self, *args, **kwargs):
        """
        Overwrites *assemble_doctree* to control the doctree.
        """
        from sphinx.util.nodes import inline_all_toctrees
        from sphinx.util.console import darkgreen
        master = self.config.master_doc
        if hasattr(self, "doctree_"):
            tree = self.doctree_
        else:
            raise AttributeError(  # pragma: no cover
                "Attribute 'doctree_' is not present. Call method finalize().")
        tree = inline_all_toctrees(
            self, set(), master, tree, darkgreen, [master])
        tree['docname'] = master
        self.env.resolve_references(tree, master, self)
        self.fix_refuris(tree)
        return tree

    def fix_refuris(self, tree):
        """
        Overwrites *fix_refuris* to control the reference names.
        """
        fname = "__" + self.config.master_doc + "__"
        for refnode in tree.traverse(nodes.reference):
            if 'refuri' not in refnode:
                continue
            refuri = refnode['refuri']
            hashindex = refuri.find('#')
            if hashindex < 0:
                continue
            hashindex = refuri.find('#', hashindex + 1)
            if hashindex >= 0:
                refnode['refuri'] = fname + refuri[hashindex:]

    def get_target_uri(self, docname, typ=None):
        """
        Overwrites *get_target_uri* to control the page name.
        """
        if docname in self.env.all_docs:
            # all references are on the same page...
            return self.config.master_doc + '#document-' + docname
        elif docname in ("genindex", "search"):
            return self.config.master_doc + '-#' + docname
        else:
            docs = ", ".join(  # pragma: no cover
                sorted(f"'{_}'" for _ in self.env.all_docs))
            raise ValueError(  # pragma: no cover
                f"docname='{docname}' should be in 'self.env.all_docs' which contains:\n{docs}")

    def get_outfilename(self, pagename):
        """
        Overwrites *get_target_uri* to control file names.
        """
        return f"{self.outdir}/{pagename}.m.html".replace("\\", "/")

    def handle_page(self, pagename, addctx, templatename='page.html',
                    outfilename=None, event_arg=None):
        """
        Overrides *handle_page* to write into stream instead of files.
        """
        from sphinx.util.osutil import relative_uri
        ctx = self.globalcontext.copy()
        if hasattr(self, "warning"):
            ctx['warn'] = self.warning
        elif hasattr(self, "warn"):
            ctx['warn'] = self.warn
        # current_page_name is backwards compatibility
        ctx['pagename'] = ctx['current_page_name'] = pagename
        ctx['encoding'] = self.config.html_output_encoding
        default_baseuri = self.get_target_uri(pagename)
        # in the singlehtml builder, default_baseuri still contains an #anchor
        # part, which relative_uri doesn't really like...
        default_baseuri = default_baseuri.rsplit('#', 1)[0]

        def pathto(otheruri, resource=False, baseuri=default_baseuri):
            if resource and '://' in otheruri:
                # allow non-local resources given by scheme
                return otheruri
            elif not resource:
                otheruri = self.get_target_uri(otheruri)
            uri = relative_uri(baseuri, otheruri) or '#'
            if uri == '#' and not self.allow_sharp_as_current_path:
                uri = baseuri
            return uri
        ctx['pathto'] = pathto

        def css_tag(css):
            attrs = []
            for key in sorted(css.attributes):
                value = css.attributes[key]
                if value is not None:
                    attrs.append('%s="%s"' % (key, htmlescape(    # pylint: disable=W1505
                        value, True)))  # pylint: disable=W1505
            attrs.append(f'href="{pathto(css.filename, resource=True)}"')
            return f"<link {' '.join(attrs)} />"
        ctx['css_tag'] = css_tag

        def hasdoc(name):
            if name in self.env.all_docs:
                return True
            elif name == 'search' and self.search:
                return True
            elif name == 'genindex' and self.get_builder_config('use_index', 'html'):
                return True
            return False
        ctx['hasdoc'] = hasdoc

        ctx['toctree'] = lambda **kw: self._get_local_toctree(pagename, **kw)
        self.add_sidebars(pagename, ctx)
        ctx.update(addctx)

        self.update_page_context(pagename, templatename, ctx, event_arg)
        newtmpl = self.app.emit_firstresult('html-page-context', pagename,
                                            templatename, ctx, event_arg)
        if newtmpl:
            templatename = newtmpl

        try:
            output = self.templates.render(templatename, ctx)
        except UnicodeError:  # pragma: no cover
            logger = getLogger("MockSphinxApp")
            logger.warning("[_CustomSphinx] A unicode error occurred when rendering the page %s. "
                           "Please make sure all config values that contain "
                           "non-ASCII content are Unicode strings.", pagename)
            return

        if not outfilename:
            outfilename = self.get_outfilename(pagename)
        # outfilename's path is in general different from self.outdir
        # ensuredir(path.dirname(outfilename))
        if outfilename not in self.built_pages:
            self.built_pages[outfilename] = StringIO()
        self.built_pages[outfilename].write(output)

class MemoryHTMLBuilder(_MemoryBuilder, CustomSingleFileHTMLBuilder):
    """
    Builds :epkg:`HTML` output in memory.
    The API is defined by the page
    :epkg:`builderapi`.
    """
    name = 'memoryhtml'
    format = 'html'
    out_suffix = None  # ".memory.html"
    supported_image_types = ['application/pdf', 'image/png', 'image/jpeg']
    default_translator_class = HTMLTranslatorWithCustomDirectives
    translator_class = HTMLTranslatorWithCustomDirectives
    _writer_class = HTMLWriterWithCustomDirectives
    supported_remote_images = True
    supported_data_uri_images = True
    html_scaled_image_link = True

    def __init__(self, app, env=None):  # pylint: disable=W0231
        """
        Construct the builder.
        Most of the parameter are static members of the class and cannot
        be overwritten (yet).

        :param app: :epkg:`Sphinx application`
        """
        _MemoryBuilder._init(self, CustomSingleFileHTMLBuilder, app, env=env)

class MemoryRSTBuilder(_MemoryBuilder, RstBuilder):

    """
    Builds :epkg:`RST` output in memory.
    The API is defined by the page
    :epkg:`builderapi`.
    The writer simplifies the :epkg:`RST` syntax by replacing
    custom roles into true :epkg:`RST` syntax.
    """

    name = 'memoryrst'
    format = 'rst'
    out_suffix = None  # ".memory.rst"
    supported_image_types = ['application/pdf', 'image/png', 'image/jpeg']
    default_translator_class = RSTTranslatorWithCustomDirectives
    translator_class = RSTTranslatorWithCustomDirectives
    _writer_class = RSTWriterWithCustomDirectives
    supported_remote_images = True
    supported_data_uri_images = True
    html_scaled_image_link = True

    def __init__(self, app, env=None):  # pylint: disable=W0231
        """
        Construct the builder.
        Most of the parameter are static members of the class and cannot
        be overwritten (yet).

        :param app: :epkg:`Sphinx application`
        """
        _MemoryBuilder._init(self, RstBuilder, app, env=env)

    def handle_page(self, pagename, addctx, templatename=None,
                    outfilename=None, event_arg=None):
        """
        Override *handle_page* to write into stream instead of files.
        """
        if templatename is not None:
            raise NotImplementedError(
                "templatename must be None.")  # pragma: no cover
        if not outfilename:
            outfilename = self.get_outfilename(pagename)
        if outfilename not in self.built_pages:
            self.built_pages[outfilename] = StringIO()
        self.built_pages[outfilename].write(self.writer.output)

class MemoryMDBuilder(_MemoryBuilder, MdBuilder):
    """
    Builds :epkg:`MD` output in memory.
    The API is defined by the page
    :epkg:`builderapi`.
    """
    name = 'memorymd'
    format = 'md'
    out_suffix = None  # ".memory.rst"
    supported_image_types = ['application/pdf', 'image/png', 'image/jpeg']
    default_translator_class = MDTranslatorWithCustomDirectives
    translator_class = MDTranslatorWithCustomDirectives
    _writer_class = MDWriterWithCustomDirectives
    supported_remote_images = True
    supported_data_uri_images = True
    html_scaled_image_link = True

    def __init__(self, app, env=None):  # pylint: disable=W0231
        """
        Construct the builder.
        Most of the parameter are static members of the class and cannot
        be overwritten (yet).

        :param app: :epkg:`Sphinx application`
        """
        _MemoryBuilder._init(self, MdBuilder, app, env=env)

    def handle_page(self, pagename, addctx, templatename=None,
                    outfilename=None, event_arg=None):
        """
        Override *handle_page* to write into stream instead of files.
        """
        if templatename is not None:
            raise NotImplementedError(
                "templatename must be None.")  # pragma: no cover
        if not outfilename:
            outfilename = self.get_outfilename(pagename)
        if outfilename not in self.built_pages:
            self.built_pages[outfilename] = StringIO()
        self.built_pages[outfilename].write(self.writer.output)

class MemoryDocTreeBuilder(_MemoryBuilder, DocTreeBuilder):
    """
    Builds doctree output in memory.
    The API is defined by the page
    :epkg:`builderapi`.
    """
    name = 'memorydoctree'
    format = 'doctree'
    out_suffix = None  # ".memory.rst"
    default_translator_class = DocTreeTranslatorWithCustomDirectives
    translator_class = DocTreeTranslatorWithCustomDirectives
    _writer_class = DocTreeWriterWithCustomDirectives
    supported_remote_images = True
    supported_data_uri_images = True
    html_scaled_image_link = True

    def __init__(self, app, env=None):  # pylint: disable=W0231
        """
        Constructs the builder.
        Most of the parameter are static members of the class and cannot
        be overwritten (yet).

        :param app: :epkg:`Sphinx application`
        """
        _MemoryBuilder._init(self, DocTreeBuilder, app, env=env)

    def handle_page(self, pagename, addctx, templatename=None,
                    outfilename=None, event_arg=None):
        """
        Override *handle_page* to write into stream instead of files.
        """
        if templatename is not None:
            raise NotImplementedError(
                "templatename must be None.")  # pragma: no cover
        if not outfilename:
            outfilename = self.get_outfilename(pagename)
        if outfilename not in self.built_pages:
            self.built_pages[outfilename] = StringIO()
        self.built_pages[outfilename].write(self.writer.output)

class MemoryLatexBuilder(_MemoryBuilder, EnhancedLaTeXBuilder):
    """
    Builds :epkg:`Latex` output in memory.
    The API is defined by the page
    :epkg:`builderapi`.
    """
    name = 'memorylatex'
    format = 'tex'
    out_suffix = None  # ".memory.tex"
    supported_image_types = ['image/png', 'image/jpeg', 'image/gif']
    default_translator_class = LatexTranslatorWithCustomDirectives
    translator_class = LatexTranslatorWithCustomDirectives
    _writer_class = LatexWriterWithCustomDirectives
    supported_remote_images = True
    supported_data_uri_images = True
    html_scaled_image_link = True

    def __init__(self, app, env=None):  # pylint: disable=W0231
        """
        Constructs the builder.
        Most of the parameter are static members of the class and cannot
        be overwritten (yet).

        :param app: :epkg:`Sphinx application`
        """
        _MemoryBuilder._init(self, EnhancedLaTeXBuilder, app, env=env)

    def write_stylesheet(self):
        from sphinx.highlighting import PygmentsBridge
        highlighter = PygmentsBridge('latex', self.config.pygments_style)
        rows = []
        rows.append('\\NeedsTeXFormat{LaTeX2e}[1995/12/01]\n')
        rows.append('\\ProvidesPackage{sphinxhighlight}')
        rows.append(
            '[2016/05/29 stylesheet for highlighting with pygments]\n\n')
        rows.append(highlighter.get_stylesheet())
        self.built_pages['sphinxhighlight.sty'] = StringIO()
        self.built_pages['sphinxhighlight.sty'].write("".join(rows))

    class EnhancedStringIO(StringIO):
        def write(self, content):
            if isinstance(content, str):
                StringIO.write(self, content)
            else:
                for line in content:
                    StringIO.write(self, line)

    def _get_filename(self, targetname, encoding='utf-8', overwrite_if_changed=True):
        if not isinstance(targetname, str):
            raise TypeError(  # pragma: no cover
                f"targetname must be a string: {targetname}")
        destination = MemoryLatexBuilder.EnhancedStringIO()
        self.built_pages[targetname] = destination
        return destination

class _CustomBuildEnvironment(BuildEnvironment):
    """
    Overrides some functionalities of
    `BuildEnvironment <https://www.sphinx-doc.org/en/master/extdev/envapi.html>`_.
    """

    def __init__(self, app):
        """
        """
        BuildEnvironment.__init__(self, app)
        self.doctree_ = {}

    def get_doctree(self, docname):
        """Read the doctree for a file from the pickle and return it."""
        if hasattr(self, "doctree_") and docname in self.doctree_:
            from sphinx.util.docutils import WarningStream
            doctree = self.doctree_[docname]
            doctree.settings.env = self
            doctree.reporter = Reporter(self.doc2path(
                docname), 2, 5, stream=WarningStream())
            return doctree

        if hasattr(self, "doctree_"):
            available = list(sorted(self.doctree_))
            if len(available) > 10:
                available = available[10:]
            raise KeyError(
                "Unable to find entry '{}' (has doctree: {})\nFirst documents:\n{}"
                "".format(
                    docname, hasattr(self, "doctree_"),
                    "\n".join(available)))

        raise KeyError(  # pragma: no cover
            "Doctree empty or not found for '{}' (has doctree: {})"
            "".format(
                docname, hasattr(self, "doctree_")))
        # return BuildEnvironment.get_doctree(self, docname)

    def apply_post_transforms(self, doctree, docname):
        """Apply all post-transforms."""
        # set env.docname during applying post-transforms
        self.temp_data['docname'] = docname

        transformer = SphinxTransformer(doctree)
        transformer.set_environment(self)
        transformer.add_transforms(self.app.post_transforms)
        transformer.apply_transforms()
        self.temp_data.clear()

class _CustomSphinx(Sphinx):
    """
    Custom :epkg:`Sphinx` application to avoid using disk.
    """

    def __init__(self, srcdir, confdir, outdir, doctreedir, buildername="memoryhtml",  # pylint: disable=W0231
                 confoverrides=None, status=None, warning=None,
                 freshenv=False, warningiserror=False,
                 tags=None, verbosity=0, parallel=0, keep_going=False,
                 new_extensions=None):
        '''
        Same constructor as :epkg:`Sphinx application`.
        Additional parameters:

        @param      new_extensions      extensions to add to the application

        Some insights about domains:

        ::

            {'cpp': sphinx.domains.cpp.CPPDomain,
             'hpp': sphinx.domains.cpp.CPPDomain,
             'h': sphinx.domains.cpp.CPPDomain,
             'js': sphinx.domains.javascript.JavaScriptDomain,
             'std': sphinx.domains.std.StandardDomain,
             'py': sphinx.domains.python.PythonDomain,
             'rst': sphinx.domains.rst.ReSTDomain,
             'c': sphinx.domains.c.CDomain}

        And builders:

        ::

            {'epub': ('epub', 'EpubBuilder'),
            'singlehtml': ('html', 'SingleFileHTMLBuilder'),
            'qthelp': ('qthelp', 'QtHelpBuilder'),
            'epub3': ('epub3', 'Epub3Builder'),
            'man': ('manpage', 'ManualPageBuilder'),
            'dummy': ('dummy', 'DummyBuilder'),
            'json': ('html', 'JSONHTMLBuilder'),
            'html': ('html', 'StandaloneHTMLBuilder'),
            'xml': ('xml', 'XMLBuilder'),
            'texinfo': ('texinfo', 'TexinfoBuilder'),
            'devhelp': ('devhelp', 'DevhelpBuilder'),
            'web': ('html', 'PickleHTMLBuilder'),
            'pickle': ('html', 'PickleHTMLBuilder'),
            'htmlhelp': ('htmlhelp', 'HTMLHelpBuilder'),
            'applehelp': ('applehelp', 'AppleHelpBuilder'),
            'linkcheck': ('linkcheck', 'CheckExternalLinksBuilder'),
            'dirhtml': ('html', 'DirectoryHTMLBuilder'),
            'latex': ('latex', 'LaTeXBuilder'),
            'elatex': ('latex', 'EnchancedLaTeXBuilder'),
            'text': ('text', 'TextBuilder'),
            'changes': ('changes', 'ChangesBuilder'),
            'websupport': ('websupport', 'WebSupportBuilder'),
            'gettext': ('gettext', 'MessageCatalogBuilder'),
            'pseudoxml': ('xml', 'PseudoXMLBuilder')}
            'rst': ('rst', 'RstBuilder')}
            'md': ('md', 'MdBuilder'),
            'doctree': ('doctree', 'DocTreeBuilder')}
        '''
        # own purpose (to monitor)
        self._logger = getLogger("_CustomSphinx")
        self._added_objects = []
        self._added_collectors = []

        # from sphinx.domains.cpp import CPPDomain
        # from sphinx.domains.javascript import JavaScriptDomain
        # from sphinx.domains.python import PythonDomain
        # from sphinx.domains.std import StandardDomain
        # from sphinx.domains.rst import ReSTDomain
        # from sphinx.domains.c import CDomain

        from sphinx.registry import SphinxComponentRegistry
        self.phase = BuildPhase.INITIALIZATION
        self.verbosity = verbosity
        self.extensions = {}
        self.builder = None
        self.env = None
        self.project = None
        self.registry = SphinxComponentRegistry()
        self.post_transforms = []
        self.pdb = False

        if doctreedir is None:
            doctreedir = "IMPOSSIBLE:TOFIND"
        if srcdir is None:
            srcdir = "IMPOSSIBLE:TOFIND"
        update_docutils_languages()

        self.srcdir = os.path.abspath(srcdir)
        self.confdir = os.path.abspath(
            confdir) if confdir is not None else None
        self.outdir = os.path.abspath(outdir) if confdir is not None else None
        self.doctreedir = os.path.abspath(doctreedir)
        self.parallel = parallel

        if self.srcdir == self.outdir:
            raise ApplicationError('Source directory and destination '  # pragma: no cover
                                   'directory cannot be identical')

        if status is None:
            self._status = StringIO()
            self.quiet = True
        else:
            self._status = status
            self.quiet = False

        from sphinx.events import EventManager
        # logging.setup(self, self._status, self._warning)
        self.events = EventManager(self)

        # keep last few messages for traceback
        # This will be filled by sphinx.util.logging.LastMessagesWriter
        self.messagelog = deque(maxlen=10)

        # say hello to the world
        from sphinx import __display_version__
        self.info(f'Running Sphinx v{__display_version__}')  # pragma: no cover

        # notice for parallel build on macOS and py38+
        if sys.version_info > (3, 8) and platform.system() == 'Darwin' and parallel > 1:
            self._logger.info(  # pragma: no cover
                "For security reason, parallel mode is disabled on macOS and "
                "python3.8 and above.  For more details, please read "
                "https://github.com/sphinx-doc/sphinx/issues/6803")

        # status code for command-line application
        self.statuscode = 0

        # delayed import to speed up time
        from sphinx.application import builtin_extensions
        from sphinx.config import CONFIG_FILENAME, Config, Tags

        # read config
        self.tags = Tags(tags)
        with warnings.catch_warnings():
            warnings.simplefilter(
                "ignore", (DeprecationWarning, PendingDeprecationWarning))
            if self.confdir is None:
                self.config = Config({}, confoverrides or {})
            else:  # pragma: no cover
                try:
                    self.config = Config.read(
                        self.confdir, confoverrides or {}, self.tags)
                except AttributeError:
                    try:
                        self.config = Config(  # pylint: disable=E1121
                            confdir, confoverrides or {}, self.tags)
                    except TypeError:
                        try:
                            self.config = Config(confdir, CONFIG_FILENAME,  # pylint: disable=E1121
                                                 confoverrides or {}, self.tags)
                        except TypeError:
                            # Sphinx==3.0.0
                            self.config = Config({}, confoverrides or {})
        self.sphinx__display_version__ = __display_version__

        # create the environment
        self.config.pre_init_values()

        # set up translation infrastructure
        self._init_i18n()

        # check the Sphinx version if requested
        if (self.config.needs_sphinx and self.config.needs_sphinx >
                __display_version__):  # pragma: no cover
            from sphinx.locale import _
            from sphinx.application import VersionRequirementError
            raise VersionRequirementError(
                _('This project needs at least Sphinx v%s and therefore cannot '
                  'be built with this version.') % self.config.needs_sphinx)

        # set confdir to srcdir if -C given (!= no confdir); a few pieces
        # of code expect a confdir to be set
        if self.confdir is None:
            self.confdir = self.srcdir

        # load all built-in extension modules
        for extension in builtin_extensions:
            try:
                with warnings.catch_warnings():
                    warnings.filterwarnings(
                        "ignore", category=DeprecationWarning)
                    self.setup_extension(extension)
            except Exception as e:  # pragma: no cover
                if 'sphinx.builders.applehelp' not in str(e):  # pragma: no cover
                    mes = "Unable to run setup_extension '{0}'\nWHOLE LIST\n{1}".format(
                        extension, "\n".join(builtin_extensions))
                    raise ExtensionError(mes) from e

        # load all user-given extension modules
        for extension in self.config.extensions:
            self.setup_extension(extension)

        # /1 addition to the original code
        # additional extensions
        if new_extensions:
            for extension in new_extensions:
                if isinstance(extension, str):
                    self.setup_extension(extension)
                else:  # pragma: no cover
                    # We assume it is a module.
                    dirname = os.path.dirname(extension.__file__)
                    sys.path.insert(0, dirname)
                    self.setup_extension(extension.__name__)
                    del sys.path[0]

        # add default HTML builders
        self.add_builder(MemoryHTMLBuilder)
        self.add_builder(MemoryRSTBuilder)
        self.add_builder(MemoryMDBuilder)
        self.add_builder(MemoryLatexBuilder)
        self.add_builder(MemoryDocTreeBuilder)

        if isinstance(buildername, tuple):
            if len(buildername) != 2:
                raise ValueError(  # pragma: no cover
                    "The builder can be custom but it must be specifed "
                    "as a 2-uple=(builder_name, builder_class).")
            self.add_builder(buildername[1])
            buildername = buildername[0]

        # /1 end of addition

        # preload builder module (before init config values)
        self.preload_builder(buildername)

        # the config file itself can be an extension
        if self.config.setup:
            prefix = f"while setting up extension {'conf.py'}:"
            if prefixed_warnings is not None:
                with prefixed_warnings(prefix):
                    if callable(self.config.setup):
                        self.config.setup(self)
                    else:  # pragma: no cover
                        from sphinx.locale import _
                        from sphinx.application import ConfigError
                        raise ConfigError(
                            _("'setup' as currently defined in conf.py isn't a Python callable. "
                              "Please modify its definition to make it a callable function. This is "
                              "needed for conf.py to behave as a Sphinx extension.")
                        )
            elif callable(self.config.setup):
                self.config.setup(self)

        # now that we know all config values, collect them from conf.py
        noallowed = []
        rem = []
        for k in confoverrides:
            if k in {'initial_header_level', 'doctitle_xform', 'input_encoding',
                     'outdir', 'warnings_log', 'extensions'}:
                continue
            if k == 'override_image_directive':
                self.config.images_config["override_image_directive"] = True
                rem.append(k)
                continue
            if k not in self.config.values:
                noallowed.append(k)
        for k in rem:
            del confoverrides[k]
        if len(noallowed) > 0:
            raise ValueError(  # pragma: no cover
                "The following configuration values are declared in any extension.\n--???--\n"
                "{0}\n--DECLARED--\n{1}".format(
                    "\n".join(sorted(noallowed)),
                    "\n".join(sorted(self.config.values))))

        # now that we know all config values, collect them from conf.py
        self.config.init_values()
        self.events.emit('config-inited', self.config)

        # /2 addition to the original code
        # check extension versions if requested
        # self.config.needs_extensions = self.config.extensions
        if not hasattr(self.config, 'items'):

            def _citems():
                for k, v in self.config.values.items():
                    yield k, v

            self.config.items = _citems

        # /2 end of addition

        # create the project
        self.project = Project(self.srcdir, self.config.source_suffix)
        # set up the build environment
        self._init_env(freshenv)
        assert self.env is not None
        # create the builder, initializes _MemoryBuilder
        self.builder = self.create_builder(buildername)
        # build environment post-initialisation, after creating the builder
        if hasattr(self, "_post_init_env"):
            self._post_init_env()
        # set up the builder
        self._init_builder()

        if not isinstance(self.env, _CustomBuildEnvironment):
            raise TypeError(  # pragma: no cover
                f"self.env is not _CustomBuildEnvironment: {type(self.env)!r} "
                f"buildername='{buildername}'")

        # addition
        self._extended_init_()

        # verification
        self._check_init_()

    def _init_builder(self) -> None:
        if not hasattr(self.builder, "env") or self.builder.env is None:
            self.builder.set_environment(self.env)
        self.builder.init()
        self.events.emit('builder-inited')

    def _check_init_(self):
        pass

    def _init_env(self, freshenv):
        ENV_PICKLE_FILENAME = 'environment.pickle'
        filename = os.path.join(self.doctreedir, ENV_PICKLE_FILENAME)
        if freshenv or not os.path.exists(filename):
            self.env = _CustomBuildEnvironment(self)
            self._fresh_env_used = True
            self.env.setup(self)
            if (self.srcdir is not None and self.srcdir != "IMPOSSIBLE:TOFIND" and
                    self.builder is not None):
                self.env.find_files(self.config, self.builder)
            return self.env

        if "IMPOSSIBLE:TOFIND" not in self.doctreedir:  # pragma: no cover
            from sphinx.application import ENV_PICKLE_FILENAME
            filename = os.path.join(self.doctreedir, ENV_PICKLE_FILENAME)
            try:
                self.info('loading pickled environment... ')
                with open(filename, 'rb') as f:
                    self.env = pickle.load(f)
                    self.env.setup(self)
                self.info('done')
                return self.env
            except Exception as err:
                self.info('failed: %r', err)
                return self._init_env(freshenv=True)

        if self.env is None:  # pragma: no cover
            self.env = _CustomBuildEnvironment(self)
            if hasattr(self.env, 'setup'):
                self.env.setup(self)
            return self.env

        if not hasattr(self.env, 'project') or self.env.project is None:
            raise AttributeError(  # pragma: no cover
                "self.env.project is not initialized.")

    def create_builder(self, name):
        """
        Creates a builder, raises an exception if name is None.
        """
        if name is None:
            raise ValueError(  # pragma: no cover
                "Builder name cannot be None")
        try:
            return self.registry.create_builder(self, name, env=self.env)
        except TypeError:
            # old version of sphinx
            return self.registry.create_builder(self, name)

    def _extended_init_(self):
        """
        Additional initialization steps.
        """
        if not hasattr(self, "domains"):
            self.domains = {}
        if not hasattr(self, "_events"):
            self._events = {}

        # Otherwise, role issue is missing.
        setup_link_roles(self)

    def _lookup_doctree(self, doctree, node_type):
        for node in doctree.traverse(node_type):
            yield node

    def _add_missing_ids(self, doctree):
        for i, node in enumerate(self._lookup_doctree(doctree, None)):
            stype = str(type(node))
            if ('section' not in stype and 'title' not in stype and
                    'reference' not in stype):
                continue
            try:
                node['ids'][0]
            except IndexError:
                node['ids'] = ['missing%d' % i]
            except TypeError:  # pragma: no cover
                pass

    def finalize(self, doctree, external_docnames=None):
        """
        Finalizes the documentation after it was parsed.

        @param      doctree             doctree (or pub.document), available after publication
        @param      external_docnames   other docnames the doctree references
        """
        imgs = list(self._lookup_doctree(doctree, nodes.image))
        for img in imgs:
            img['save_uri'] = img['uri']

        if not isinstance(self.env, _CustomBuildEnvironment):
            raise TypeError(  # pragma: no cover
                f"self.env is not _CustomBuildEnvironment: '{type(self.env)}'")
        if not isinstance(self.builder.env, _CustomBuildEnvironment):
            raise TypeError(  # pragma: no cover
                "self.builder.env is not _CustomBuildEnvironment: '{0}'".format(
                    type(self.builder.env)))
        self.doctree_ = doctree
        self.builder.doctree_ = doctree
        self.env.doctree_[self.config.master_doc] = doctree
        self.env.all_docs = {self.config.master_doc: self.config.master_doc}

        if external_docnames:
            for doc in external_docnames:
                self.env.all_docs[doc] = doc

        # This steps goes through many function including one
        # modifying paths in image node.
        # Look for node['candidates'] = candidates in Sphinx code.
        # If a path startswith('/'), it is removed.
        from sphinx.environment.collectors.asset import logger as logger_asset
        logger_asset.setLevel(40)  # only errors
        self._add_missing_ids(doctree)
        self.events.emit('doctree-read', doctree)
        logger_asset.setLevel(30)  # back to warnings

        for img in imgs:
            img['uri'] = img['save_uri']

        self.events.emit('doctree-resolved', doctree,
                         self.config.master_doc)
        self.builder.write(None, None, 'all')

    def debug(self, message, *args, **kwargs):
        self._logger.debug(message, *args, **kwargs)

    def info(self, message, *args):
        self._logger.info(message, *args)

    def warning(self, message='', name=None, type=None, subtype=None):
        if "is already registered" not in message:  # pragma: no cover
            self._logger.warning(
                "[_CustomSphinx] %s -- %s", message, name,
                type=type, subtype=subtype)

    def add_builder(self, builder, override=False):
        self._added_objects.append(('builder', builder))
        if builder.name not in self.registry.builders:
            self.debug('[_CustomSphinx]  adding builder: %r', builder)
            self.registry.add_builder(builder, override=override)
        else:
            self.debug('[_CustomSphinx]  already added builder: %r', builder)

    def setup_extension(self, extname):
        self._added_objects.append(('extension', extname))

        logger = getLogger('sphinx.application')
        disa = logger.logger.disabled
        logger.logger.disabled = True

        # delayed import to speed up time
        try:
            with warnings.catch_warnings():
                warnings.filterwarnings(
                    "ignore", category=DeprecationWarning)
                self.registry.load_extension(self, extname)
        except Exception as e:  # pragma: no cover
            raise ExtensionError(
                f"Unable to setup extension '{extname}'") from e
        finally:
            logger.logger = disa

    def add_directive(self, name, obj, content=None, arguments=None,  # pylint: disable=W0221,W0237
                      override=True, **options):
        self._added_objects.append(('directive', name))
        if name == 'plot' and obj.__name__ == 'PlotDirective':

            old_run = obj.run

            def run(self):
                """Run the plot directive."""
                logger = getLogger("MockSphinxApp")
                logger.info('[MockSphinxApp] PlotDirective: %r', self.content)
                try:
                    res = old_run(self)
                    logger.info('[MockSphinxApp] PlotDirective ok')
                    return res
                except OSError as e:  # pragma: no cover
                    logger = getLogger("MockSphinxApp")
                    logger.info('[MockSphinxApp] PlotDirective failed: %s', e)
                return []

            obj.run = run

        Sphinx.add_directive(self, name, obj, override=override, **options)

    def add_domain(self, domain, override=True):
        self._added_objects.append(('domain', domain))
        Sphinx.add_domain(self, domain, override=override)
        # For some reason, the directives are missing from the main catalog
        # in docutils.
        for k, v in domain.directives.items():
            self.add_directive(f"{domain.name}:{k}", v)
            if domain.name in ('py', 'std', 'rst'):
                # We add the directive without the domain name as a prefix.
                self.add_directive(k, v)
        for k, v in domain.roles.items():
            self.add_role(f"{domain.name}:{k}", v)
            if domain.name in ('py', 'std', 'rst'):
                # We add the role without the domain name as a prefix.
                self.add_role(k, v)

    def add_role(self, name, role, override=True):
        self._added_objects.append(('role', name))
        self.debug('[_CustomSphinx]  adding role: %r', (name, role))
        roles.register_local_role(name, role)

    def add_generic_role(self, name, nodeclass, override=True):
        self._added_objects.append(('generic_role', name))
        self.debug("[_CustomSphinx] adding generic role: '%r'",
                   (name, nodeclass))
        role = roles.GenericRole(name, nodeclass)
        roles.register_local_role(name, role)

    def add_node(self, node, override=True, **kwds):
        self._added_objects.append(('node', node))
        self.debug('[_CustomSphinx]  adding node: %r', (node, kwds))
        nodes._add_node_class_names([node.__name__])
        for key, val in kwds.items():
            try:
                visit, depart = val
            except ValueError:  # pragma: no cover
                raise ExtensionError(("Value for key '%r' must be a "
                                      "(visit, depart) function tuple") % key)
            translator = self.registry.translators.get(key)
            translators = []
            if translator is not None:
                translators.append(translator)
            elif key == 'html':
                from sphinx.writers.html import HTMLTranslator
                translators.append(HTMLTranslator)
                if is_html5_writer_available():
                    from sphinx.writers.html5 import HTML5Translator
                    translators.append(HTML5Translator)
            elif key == 'latex':
                translators.append(_get_LaTeXTranslator())
            elif key == 'elatex':
                translators.append(EnhancedLaTeXBuilder)
            elif key == 'text':
                from sphinx.writers.text import TextTranslator
                translators.append(TextTranslator)
            elif key == 'man':
                from sphinx.writers.manpage import ManualPageTranslator
                translators.append(ManualPageTranslator)
            elif key == 'texinfo':
                from sphinx.writers.texinfo import TexinfoTranslator
                translators.append(TexinfoTranslator)

            for translator in translators:
                setattr(translator, 'visit_' + node.__name__, visit)
                if depart:
                    setattr(translator, 'depart_' + node.__name__, depart)

    def add_event(self, name):
        self._added_objects.append(('event', name))
        Sphinx.add_event(self, name)

    def add_config_value(self, name, default, rebuild, types_=(), types=()):  # pylint: disable=W0221,W0237
        types_ = types or types_
        self._added_objects.append(('config_value', name))
        Sphinx.add_config_value(self, name, default, rebuild, types_)

    def add_directive_to_domain(self, domain, name, obj, has_content=None,  # pylint: disable=W0221,W0237
                                argument_spec=None, override=False, **option_spec):
        self._added_objects.append(('directive_to_domain', domain, name))
        Sphinx.add_directive_to_domain(self, domain, name, obj,
                                       override=override, **option_spec)

    def add_role_to_domain(self, domain, name, role, override=False):
        self._added_objects.append(('roles_to_domain', domain, name))
        Sphinx.add_role_to_domain(self, domain, name, role, override=override)

    def add_transform(self, transform):
        self._added_objects.append(('transform', transform))
        Sphinx.add_transform(self, transform)

    def add_post_transform(self, transform):
        self._added_objects.append(('post_transform', transform))
        Sphinx.add_post_transform(self, transform)

    def add_js_file(self, filename, priority=500, **kwargs):  # pylint: disable=W0221
        # loading_method=None: added in Sphinx 4.4
        self._added_objects.append(('js', filename))
        Sphinx.add_js_file(self, filename, priority=priority, **kwargs)

    def add_css_file(self, filename, priority=500, **kwargs):
        self._added_objects.append(('css', filename))
        Sphinx.add_css_file(self, filename, priority=priority, **kwargs)

    def add_latex_package(self, packagename, options=None, after_hyperref=False):
        self._added_objects.append(('latex', packagename))
        Sphinx.add_latex_package(
            self, packagename=packagename, options=options,
            after_hyperref=after_hyperref)

    def add_object_type(self, directivename, rolename, indextemplate='',
                        parse_node=None, ref_nodeclass=None, objname='',
                        doc_field_types=None, override=False):
        if doc_field_types is None:
            doc_field_types = []
        self._added_objects.append(('object', directivename, rolename))
        Sphinx.add_object_type(self, directivename, rolename, indextemplate=indextemplate,
                               parse_node=parse_node, ref_nodeclass=ref_nodeclass,
                               objname=objname, doc_field_types=doc_field_types,
                               override=override)

    def add_env_collector(self, collector):
        """
        See :epkg:`class Sphinx`.
        """
        self.debug(
            '[_CustomSphinx] adding environment collector: %r', collector)
        coll = collector()
        coll.enable(self)
        self._added_collectors.append(coll)

    def disconnect_env_collector(self, clname, exc=True):
        """
        Disables a collector given its class name.

        @param      cl      name
        @param      exc     raises an exception if not found
        @return             found collector
        """
        found = None
        foundi = None
        for i, co in enumerate(self._added_collectors):
            if clname == co.__class__.__name__:
                found = co
                foundi = i
                break
        if found is not None and not exc:
            return None
        if found is None:
            raise ValueError(  # pragma: no cover
                "Unable to find a collector '{0}' in \n{1}".format(
                    clname, "\n".join(
                        map(lambda x: x.__class__.__name__,
                            self._added_collectors))))
        for v in found.listener_ids.values():
            self.disconnect(v)
        del self._added_collectors[foundi]
        return found

{'sphinx.util.osutil': 'sphinx.util.osutil', 'custom_ensuredir': 'conf_path_tools.custom_ensuredir', 'pending_warnings': 'sphinx.util.logging.pending_warnings', 'status_iterator': 'sphinx.util.status_iterator', 'inline_all_toctrees': 'sphinx.util.nodes.inline_all_toctrees', 'darkgreen': 'sphinx.util.console.darkgreen', 'relative_uri': 'sphinx.util.osutil.relative_uri'}._init


from pyspark import SparkContext,SparkConf
from pyspark.sql import SQLContext
from pyspark.sql import SparkSession
from datetime import datetime, date, timedelta

from imports.plotter import Plotter

import time
import sys

def main(argv):
    plotter    = Plotter()

    y = []
    x = []

    uri_conf = 'mongodb://localhost:27017/meteo_station.outside'
    spark = SparkSession \
                .builder \
                .appName("Meteo_Station") \
                .config("spark.mongodb.input.uri", uri_conf) \
                .config("spark.executor.memory", "4g") \
                .config("spark.executor.number", "4") \
                .config("spark.executor.cores", "4") \
                .config("spark.network.timeout", "800s") \
                .config('spark.yarn.executor.memoryOverhead', "4096m") \
                .config("spark.driver.memoryOverhead",) \
                .getOrCreate()

    df = spark.read.format("com.mongodb.spark.sql.DefaultSource").load()
    df.printSchema()
    df.registerTempTable('temp')
    pressure = spark.sql('SELECT * FROM temp WHERE sensor = \'BMP Pressure \' ')
    coll = pressure.collect()

    for c in coll:
        y.append(float(c['data']))

        date = c['date']
        xtime = c['time']
        dtime = date + '_' + xtime

        try:
            xdtime = datetime.strptime(dtime, '%Y_%m_%d_%H:%M:%S')
        except:
            xdtime = datetime.strptime(dtime, '%Y_%m_%d_%H:%M:%S.%f')

        x.append(xdtime)

    print('-->Data count\t: ' + str(len(x)))

    plotter.show_plotter(x, y, plotLabel = 'Pressure (BMP180)', yLabel = 'mm Hg')

if (__name__ == '__main__'):
    main(sys.argv[1:])


import json
import urllib.request as urlr

from geopandas.geodataframe import GeoDataFrame
from src.helpers.database import Database
import pandas as pd
import geopandas as gpd
from shapely.geometry import shapely
from src.helpers.environment_variables import DATABASE_URL, SUPERUSER_DATABASE_URL

def download_epa_sld_data_as_geojson(filepath:str):
    """
    - This function downloads a geojson from the EPA's Smart Location Database
    - It uses a loop to iterate over all features in the table, which gets around
    the issue of the EPA rest service not allowing all of the features to be queried at once
    """

    # set variables for map service
    # map service url
    mapservice = (
        r"https://geodata.epa.gov/arcgis/rest/services/OA/SmartLocationDatabase/MapServer/15/"
    )
    # map service query
    query = r'query?where=GEOID10+like+%2734005%25%27+or+GEOID10+like+%2734007%25%27+or+GEOID10+like+%2734015%25%27+or+GEOID10+like+%2734021%25%27'

    # geojson structure
    dict = '{"type": "FeatureCollection"}'
    geojson = json.loads(dict)
    geojson["features"] = []

    # find total records allowed to be queried from the mapservice
    maxurl = mapservice + r"?f=json"
    response = urlr.urlopen(maxurl)
    maxjson = json.load(response)
    maxcount = int(maxjson["maxRecordCount"])

    # query all objectids, count total objectids
    oidurl = mapservice + query + r"&returnIdsOnly=true&f=json"
    print(oidurl)

    response = urlr.urlopen(oidurl)
    print(response)
    oidjson = json.load(response)
    print(oidjson)
    idfield = oidjson["objectIdFieldName"]
    print(idfield)
    idlist = oidjson["objectIds"]
    idlist.sort()
    numrec = len(idlist)

    # iterate through map service objectids by max record count
    for i in range(0, numrec, maxcount):
        torec = i + (maxcount - 1)
        if torec > numrec:
            torec = numrec - 1
        fromid = idlist[i]
        toid = idlist[torec]
        where = r"{}+%3E%3D+{}+and+{}+%3C%3D+{}".format(idfield, fromid, idfield, toid)
        # possible to adjust api parameters here to request only specified field (outfields) or even change projection (outsr)
        urlstring = (
            mapservice
            + "query?where={}&outfields=*&outsr=4326&returnGeometry=true&f=geojson".format(where)
        )
        response = urlr.urlopen(urlstring)
        testjson = json.load(response)
        for feature in testjson["features"]:
            # append each feature to geojson dict
            geojson["features"].append(feature)

    # write final geojson to location on hard drive
    file_path = r"/Users/fakram/Downloads/SLD.geojson"

    with open(file_path, "w") as outfile:
        json.dump(geojson, outfile)

    return file_path

if __name__ == "__main__":
    db = Database()

    print("-" * 80)
    print("Importing data from SLD/Map Service")
    print("-" * 80)

    filepath = download_epa_sld_data_as_geojson("./SLD.geojson")
    gdf = gpd.read_file(filepath)
    explodedgdf = gdf.explode()
    db.import_geodataframe(explodedgdf, new_tablename="sld", schema="extract")


import game
import pygame
from kuposztok.Info.InfoActors import *
import kuposztok

class InfoStage(game.scene2d.MyStage):

    def soundvaltread(self):
        with open('../kuposztok/Save/options.txt', 'r') as beskinfile1:
            self.soundvaltbe = int(beskinfile1.readline())
            self.musica = int(beskinfile1.readline())
            self.allstagebe = int(beskinfile1.readline())
            beskinfile1.close()

    def __init__(self):
        super().__init__()
        self.height = pygame.display.get_surface().get_height()
        self.width = pygame.display.get_surface().get_width()
        self.soundvaltread()
        self.musicaselect  = self.musica
        self.allstageben = self.allstagebe
        self.bg = BgActor()
        self.add_actor(self.bg)
        self.soundvalt = self.soundvaltbe
        pygame.mixer.init()
        self.allstage = False
        if self.allstageben == 0 or self.allstageben == 1:
            self.allstage = True
        if self.allstageben == 2:
            self.allstage = False
        if self.allstage == False:
            pygame.mixer.music.load("../kuposztok/music/infomusica.wav")
        else:
            if self.musicaselect == 0 or self.musicaselect == 1:
                pygame.mixer.music.load("../kuposztok/music/gamemusica1.wav")
            if self.musicaselect == 2:
                pygame.mixer.music.load("../kuposztok/music/gamemusica2.wav")
            if self.musicaselect == 3:
                pygame.mixer.music.load("../kuposztok/music/gamemusica3.wav")
            if self.musicaselect == 4:
                pygame.mixer.music.load("../kuposztok/music/gamemusica4.wav")
            if self.musicaselect == 5:
                pygame.mixer.music.load("../kuposztok/music/gamemusica5.wav")
        pygame.mixer.music.play(-1)
        if self.soundvalt == 0 or self.soundvalt == 1:
            pygame.mixer.music.set_volume(0.5)
        if self.soundvalt == 2:
            pygame.mixer.music.set_volume(0.20)
        if self.soundvalt == 3:
            pygame.mixer.music.set_volume(0.07)
        if self.soundvalt == 4:
            pygame.mixer.music.stop()
        self.text1 = game.scene2d.MyLabel("A játék lényege, hogy a karakterünkel minél több")
        self.text1.set_color(0, 0, 0)
        self.text1.x = 100
        self.text1.y = 0 + self.text1.get_height()
        self.add_actor(self.text1)
        self.text2 = game.scene2d.MyLabel("pontot érjünk el úgy, hogy a fákat kerülgetjük.")
        self.text2.set_color(0, 0, 0)
        self.text2.x = 100
        self.text2.y = 0 + self.text2.get_height() * 2
        self.add_actor(self.text2)
        self.text3 = game.scene2d.MyLabel("A játék ezek mellet tartalmaz még egyéb tárgyakat,")
        self.text3.set_color(0, 0, 0)
        self.text3.x = 100
        self.text3.y = 0 + self.text3.get_height() * 3
        self.add_actor(self.text3)
        self.text31 = game.scene2d.MyLabel("mint például az energiaital, vagy a trap:")
        self.text31.set_color(0, 0, 0)
        self.text31.x = 100
        self.text31.y = 0 + self.text31.get_height() * 4
        self.add_actor(self.text31)
        self.energy = Energy()
        self.energy.x = self.text3.get_x()
        self.energy.y = 0 + self.text3.get_height() * 5
        self.energy.width = 150
        self.energy.height = 150
        self.add_actor(self.energy)
        self.trap = Trap()
        self.trap.x = self.text3.get_x() + self.text3.get_x() * 3
        self.trap.y = 0 + self.text3.get_height() * 5
        self.add_actor(self.trap)
        self.text4 = game.scene2d.MyLabel("A játékost a billentyűzeten lévő 'w', 'a', 's', 'd' gombokkal")
        self.text4.set_color(0, 0, 0)
        self.text4.x = 100
        self.text4.y = self.height / 2 + self.text4.get_height()
        self.add_actor(self.text4)
        self.text5 = game.scene2d.MyLabel("tudjuk irányítani. Multiplayer módban az egyik karaktert")
        self.text5.set_color(0, 0, 0)
        self.text5.x = 100
        self.text5.y = self.height / 2 + self.text5.get_height() * 2
        self.add_actor(self.text5)
        self.text6 = game.scene2d.MyLabel("szintén ezekkel a gombokkal, míg a másikat pedig a ")
        self.text6.set_color(0, 0, 0)
        self.text6.x = 100
        self.text6.y = self.height / 2 + self.text5.get_height() * 3
        self.add_actor(self.text6)
        self.text7 = game.scene2d.MyLabel("billyentyűzeten lévő nyilakkal. Visszalépés ")
        self.text7.set_color(0, 0, 0)
        self.text7.x = 100
        self.text7.y = self.height / 2 + self.text7.get_height() * 4
        self.add_actor(self.text7)
        self.text71 = game.scene2d.MyLabel("az 'ESC' gombbal történik, valamint ez a gomb a menüben")
        self.text71.set_color(0, 0, 0)
        self.text71.x = 100
        self.text71.y = self.height / 2 + self.text71.get_height() * 5
        self.add_actor(self.text71)
        self.text72 = game.scene2d.MyLabel("a kilépést szolgálja.")
        self.text72.set_color(0, 0, 0)
        self.text72.x = 100
        self.text72.y = self.height / 2 + self.text72.get_height() * 6
        self.add_actor(self.text72)
        self.back = Back()
        self.add_actor(self.back)
        self.back.x = self.width - self.back.get_width()
        self.back.y = self.height - self.back.get_height()

        self.set_on_key_down_listener(self.katt)
        self.back.set_on_mouse_down_listener(self.Back)

    def katt(self, sender, event):
        if event.key == pygame.K_ESCAPE:
            self.screen.game.set_screen(kuposztok.Menu.MenuScreen.MenuScreen())

    def Back(self, sender, event):
        if event.button == 1:
            self.screen.game.set_screen(kuposztok.Menu.MenuScreen.MenuScreen())


from django.urls import path
from django.views.generic import TemplateView
from . import views

urlpatterns = [
    path('', views.index, name='index'),
    path('register/', views.register, name='register'),
    path('requirements/', views.requirements, name='requirements'),
    path('types/', views.types, name='types'),
    path('tips/', views.tips, name='tips'),
    path('login/', views.log, name='login'),
    path('donor_register/', views.donor_register.as_view(), name='donor_register'),
    path('center_register/', views.center_register.as_view(), name='center_register'),
    path('quiz/', views.quiz, name='quiz'),
    path('edit_profile/<str:edit>',views.edit_profile,name ='edit_profile') ,
    path('profiles/<str:email>',views.profiles,name ='profiles') ,
    path('search/', views.search_results, name='search_results'),
    path('create_event/', views.create_event.as_view(), name='create_event'),
    path('appointment/', views.create_appointment.as_view(), name='appointment'),
    path('events/', views.events, name='events'),
    path('center/<str:name>', views.cent, name='center'),
    path('logout/', views.pagelogout, name='logout'),
    # path( "/dashboard",
    #     TemplateView.as_view(template_name="layouts/dashboard.html"),
    #     name="dashboard",),
    # path('accounts/logout/', views.LogoutView.as_view(template_name="post_list.html"), name='logout'),
    # path("<int:myid>/", views.quiz, name="quiz"),
    # path("quiz/", views.quiz, name="quiz"),
    # path('<int:myid>/data/', views.quiz_data_view, name='quiz-data'),
    # path('<int:myid>/save/', views.save_quiz_view, name='quiz-save'),
]


from rest_framework.views import exception_handler
from rest_framework.exceptions import (
    AuthenticationFailed,
    NotFound,
    PermissionDenied,
    ValidationError,
    MethodNotAllowed
)
from rest_framework.response import Response
from rest_framework import status

def custom_exception_handler(exc, context):
    response = exception_handler(exc, context)
    if isinstance(exc, AuthenticationFailed):
        response = Response(
            {
                'status_code': status.HTTP_401_UNAUTHORIZED,
                'message': 'Authentication failed',
                'errors': []
            },
            status=status.HTTP_401_UNAUTHORIZED
        )
    elif isinstance(exc, ValidationError):
        print(exc)
        response = Response(
            {
                'status_code': status.HTTP_400_BAD_REQUEST,
                'message': 'Bad request',
                'errors': str(exc.detail)
            },
            status=status.HTTP_400_BAD_REQUEST
        )
    elif isinstance(exc, PermissionDenied):
        response = Response(
            {
                'status_code': status.HTTP_403_FORBIDDEN,
                'message': 'Permission denied',
                'errors': []
            },
            status=status.HTTP_403_FORBIDDEN
        )
    elif isinstance(exc, NotFound):
        response = Response(
            {
                'status_code': status.HTTP_404_NOT_FOUND,
                'message': 'Not found',
                'errors': []
            },
            status=status.HTTP_404_NOT_FOUND
        )
    elif isinstance(exc, MethodNotAllowed):
        response = Response(
            {
                'status_code': status.HTTP_405_METHOD_NOT_ALLOWED,
                'message': 'Method not allowed',
                'errors': []
            },
            status=status.HTTP_405_METHOD_NOT_ALLOWED
        )

    return response


# -*- coding: utf-8 -*-
"""
Prediction on video using
pytorch model
"""
# import the necessary packages
from __future__ import print_function
import argparse
import json
from collections import deque
import torch
from torchvision import transforms
from torch.autograd import Variable
import torch.nn.functional as F
import numpy as np
import cv2

device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
print(device)
# construct the argument parser and parse the arguments
ap = argparse.ArgumentParser()
ap.add_argument("-m", "--model", required=True,
                help="path to trained serialized model")
ap.add_argument("-w", "--weights", required=True,
                help="path to the parameters of the model")
ap.add_argument("-i", "--input", required=True,
                help="path to our input video")
ap.add_argument("-l", "--labels", required=True,
                help="path to the labels")
ap.add_argument("-o", "--output", required=True,
                help="path to our output video")
ap.add_argument("-s", "--size", type=int, default=1,
                help="size of queue for averaging")
args = vars(ap.parse_args())

# load dictionary of labels
print("[INFO] loading labels...")
json_file = open(args['labels'])
label_dict = json.load(json_file)

# load the trained model
print("[INFO] loading model...")
loc = torch.load(args["weights"], map_location=device)
model = torch.load(args["model"], map_location=device)
model.load_state_dict(loc["model"])
model.eval()

data_transforms = {
    'val': transforms.Compose([transforms.ToTensor(),
                               transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])]),
}

def predict(image, model):
    # Pass the image through our model
    image_tensor = data_transforms['val'](image).float()
    image_tensor = image_tensor.unsqueeze_(0)
    images = Variable(image_tensor)
    image_tensor = images.to(device)
    torch.no_grad()
    predict = F.softmax(model(image_tensor))
    print(predict.detach().cpu().numpy())
    return predict.cpu().detach().numpy()

# initialize the image mean for mean subtraction along with the
# predictions queue
Q = deque(maxlen=args["size"])

# initialize the video stream, pointer to output video file, and
# frame dimensions
vs = cv2.VideoCapture(args["input"])
writer = None
(W, H) = (None, None)

# loop over frames from the video file stream
while True:
    # read the next frame from the file
    (grabbed, frame) = vs.read()

    # if the frame was not grabbed, then we have reached the end
    # of the stream
    if not grabbed:
        break

    # if the frame dimensions are empty, grab them
    if W is None or H is None:
        (H, W) = frame.shape[:2]

    # clone the output frame, then convert it from BGR to RGB
    # ordering, resize the frame to a fixed 224x224, and then
    # perform mean subtraction
    output = frame.copy()
    frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)

    # make predictions on the frame and then update the predictions
    # queue
    preds = predict(frame, model)
    Q.append(preds)

    # perform prediction averaging over the current history of
    # previous predictions
    results = np.array(Q).mean(axis=0)
    i = np.argmax(results)
    label = label_dict[str(i)]

    # draw the condition on the output frame
    text = "{}".format(label)
    cv2.putText(output, text, (35, 50), cv2.FONT_HERSHEY_SIMPLEX,
                1.25, (0, 255, 0), 5)

    # check if the video writer is None
    if writer is None:
        # initialize our video writer
        fourcc = cv2.VideoWriter_fourcc(*"MJPG")
        writer = cv2.VideoWriter(args["output"], fourcc, 30,
                                 (W, H), True)

    # write the output frame to disk
    writer.write(output)

    # show the output image
    cv2.imshow("Output", output)
    key = cv2.waitKey(1) & 0xFF

    # if the `q` key was pressed, break from the loop
    if key == ord("q"):
        break

# release the file pointers
print("[INFO] cleaning up...")
writer.release()
vs.release()


from roughsets.roughsets import *
import pandas as pd
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
import numpy as np

'''
Determina le soglie utilizzando tre metodi: OPTIMIZER, FORZA BRUTA e VISUALE (scrivo nel file rfilename le prime due coppie)
'''
def determineThresholds(dtable, ncol, concept, rfilename, output, path, kind):

    f = open(rfilename, "w")
    f.write("beta,alpha\n")

    nrs = RoughSets()
    print(dtable)
    output.write(str(dtable) + '\n')

    nrs.setInformationSystem(dtable, np.array([i for i in range(ncol)]))

    b1 = (0.0, 0.5)
    b2 = (0.5, 1.0)
    xinit = [0.3, 0.7]
    beta, alpha = nrs.calculateAlphaBeta(concept, b1, b2, xinit)

    print("* Optimizer")
    output.write("* Optimizer \n")

    print("BETA, ALFA: ", beta, alpha)
    output.write(str(beta) + "," + str(alpha) + "\n")
    f.write(str(beta)+","+str(alpha)+"\n")

    b = np.linspace(0.0, 0.5, 5)
    a = np.linspace(0.5, 1.0, 5)

    A, B = np.meshgrid(a, b)

    HM=list()
    for i in range(len(B)):
        HR = list()
        for j in range(len(A)):
            HR.append(nrs.Entropy((B[i][j], A[i][j])))
        HM.append(HR)

    print(HM)
    output.write(str(HM)+'\n')

    ind = np.unravel_index(np.argmin(np.array(HM), axis=None), np.array(HM).shape)
    print("ARGMIN: ",ind)
    output.write("* ARGMIN: "+str(ind)+ '\n')

    print("MIN: ", np.min(HM))
    output.write("MIN: "+str(np.min(HM))+'\n')

    print("* Brute Force")
    output.write("* Brute Force \n")

    print("BETA, ALFA: ", b[ind[0]], a[ind[1]])
    output.write("BETA, ALFA: "+str([ind[0]])+','+str(a[ind[1]])+"\n")

    f.write(str(b[ind[0]])+","+str(a[ind[1]])+"\n")
    f.close()

    print("* Visual")
    output.write('* Visual\n')
    ax = plt.axes(projection='3d')
    ax.plot_surface(A, B, np.array(HM), rstride=1, cstride=1,
                    cmap='viridis', edgecolor='none')
    ax.set_title('surface')
    ax.set_xlabel('alpha')
    ax.set_ylabel('beta')
    ax.set_zlabel('entropy')
    #plt.show()
    plt.savefig(path+'/'+str(kind)+'.png')

'''
Applica il 3WD probabilistico con le soglie date e calcola l'esagono di opposizione
'''
def exagonOpposition(dtable, ncol, concept, beta, alpha, output):
    nrs = RoughSets()
    nrs.setInformationSystem(dtable, np.array([i for i in range(ncol)]))

    regions = nrs.pcalculate3WD(concept, alpha, beta)
    regions = list(regions)

    POS = regions[0]
    BND = regions[1]
    NEG = regions[2]

    print("POS: ", POS)
    output.write("POS: " + str(POS) + '\n')
    print("BND: ", BND)
    output.write("BND: " + str(BND) + '\n')
    print("NEG: ", NEG)
    output.write("NEG: " + str(NEG) + '\n')
    regs = {"POS": POS, "BND": BND, "NEG": NEG}
    points = {"A": {}, "B": {}, "C": {}, "D": {}, "E": {}, "F": {}}

    points["A"] = POS
    points["B"] = NEG
    points["F"] = BND
    points["D"] = NEG.union(BND)
    points["C"] = POS.union(BND)
    points["E"] = POS.union(NEG)

    return regs, points

def mainNew(path,i):

    print("------------------------------"+str(i)+"/"+str(i+1)+"-------------------------------------")
    with open(path+'regions.txt', 'w') as output:
        # Numero di attributi condizionali del primo dataset
        NCOL1=6

        data1 = pd.read_csv(path+"decision_table_base.csv")
        data1=data1.drop(columns="usr_index")
        infoTable1 = data1.values

        # filtro sulla colonna decisionale (il concetto da studiare deve essere cambiato in questa riga)
        decisions_d1 = data1[data1.decision == 2]
        decisions_d1 = np.array(list(decisions_d1.index))

        determineThresholds(infoTable1, NCOL1, decisions_d1, path+"thresholds_step1.csv", output, path, 'base')

        f=open(path+"thresholds_step1.csv", "r")
        lfile=list()

        for line in f:
            lfile.append(line.strip())
        f.close()

        # uso solo i valori calcolati con forza bruta
        beta1 = float(lfile[2].split(",")[0])
        alpha1 = float(lfile[2].split(",")[1])

        regions1, points1 = exagonOpposition(infoTable1, NCOL1, decisions_d1, beta1, alpha1, output)

        for k, v in points1.items():
            print("POINT: ", k, "   VALUE: ", v)
            output.write("POINT: "+str(k)+" VALUE "+str(v)+'\n')

        for k, v in regions1.items():
            print("REGION: ", k, "   VALUE: ", v)
            output.write("REGION: " + str(k) + " VALUE " + str(v) + '\n')
        # Numero di attributi condizionali del secondo dataset
        NCOL2=7

        data2 = pd.read_csv(path+"decision_table_t1.csv")
        data2=data2.drop(columns="usr_index")
        infoTable2 = data2.values

        # filtro sulla colonna decisionale (il concetto da studiare deve essere cambiato in questa riga)
        decisions_d2 = data2[data2.decision == 2]
        decisions_d2 = np.array(list(decisions_d2.index))

        determineThresholds(infoTable2, NCOL2, decisions_d2, path+"thresholds_step2.csv", output, path, 't1')

        f=open(path+"thresholds_step2.csv", "r")
        lfile=list()

        for line in f:
            lfile.append(line.strip())
        f.close()

        # uso solo i valori calcolati con forza bruta
        beta2 = float(lfile[2].split(",")[0])
        alpha2 = float(lfile[2].split(",")[1])

        regions2, points2 = exagonOpposition(infoTable2, NCOL2, decisions_d2, beta2, alpha2, output)

        for k, v in points2.items():
            print("POINT: ", k, "   VALUE: ", v)
            output.write("POINT: " + str(k) + " VALUE " + str(v) + '\n')

        for k, v in regions2.items():
            print("REGION: ", str(k), "   VALUE: ", str(v))
            output.write("REGION: " + str(k) + " VALUE " + str(v) + '\n')
        mu, xi = utilityMeasures((regions1, regions2), (points1, points2), ((beta1, alpha1),(beta2, alpha2)), (data1, data2), (decisions_d1, decisions_d2))

        print("MU: ", mu)
        output.write("MU: " + str(mu) + '\n')
        print("XI: ", xi)
        output.write("XI: " + str(xi) + '\n')

def communityProbability(community, concept):
    return len(set(community).intersection(set(concept)))/len(set(concept))

def calculateX(POS_T1, POS_T2, concept, data):
    data1=data[0]
    data2=data[1]

    if len(POS_T2) > len(POS_T1):
        probabilities = list()
        communities = set()
        irange=set(POS_T2).difference(set(POS_T1))
        for i in list(irange):
            ci=data1.iloc[i]["community"]
            communities.add(ci)
        for i in list(communities):
            datax=data1[data1.community==i]
            datax = list(datax.index)
            prob=communityProbability(datax, concept)
            probabilities.append(prob)
        mini=0
        if len(probabilities)!=0:
            mini=min(probabilities)
        return mini
    else:
        probabilities = list()
        communities = set()
        irange=set(POS_T1).difference(set(POS_T2))
        for i in list(irange):
            ci=data1.iloc[i]["community"]
            communities.add(ci)
        for i in list(communities):
            datax=data1[data1.community==i]
            datax = list(datax.index)
            prob=communityProbability(datax, concept)
            probabilities.append(prob)
        maxi=0
        if len(probabilities)!=0:
            maxi=max(probabilities)
        return maxi

def calculateY(NEG_T1, NEG_T2, concept, data):
    data1=data[0]
    data2=data[1]

    if len(NEG_T2) > len(NEG_T1):
        probabilities = list()
        communities = set()
        irange=set(NEG_T2).difference(set(NEG_T1))
        for i in irange:
            ci=data1.iloc[i]["community"]
            communities.add(ci)
        for i in list(communities):
            datax=data1[data1.community==i]
            datax = list(datax.index)
            prob=communityProbability(datax, concept)
            probabilities.append(prob)
        maxi=0
        if len(probabilities)!=0:
            maxi=max(probabilities)
        return maxi
    else:
        probabilities = list()
        communities = set()
        irange=set(NEG_T1).difference(set(NEG_T2))
        for i in irange:
            ci=data1.iloc[i]["community"]
            communities.add(ci)
        for i in list(communities):
            datax=data1[data1.community==i]
            datax = list(datax.index)
            prob=communityProbability(datax, concept)
            probabilities.append(prob)
        mini=0
        if len(probabilities)!=0:
            mini=min(probabilities)
        return mini

def utilityMeasures(regions, points, thresholds, datas, concepts):
    regions1=regions[0]
    regions2=regions[1]
    points1=points[0]
    points2=points[1]
    beta1 = thresholds[0][0]
    alpha1 = thresholds[0][1]
    beta2 = thresholds[1][0]
    alpha2 = thresholds[1][1]
    data1 = datas[0]
    data2 = datas[1]
    concept1 = concepts[0]
    concept2 = concepts[1]

    x=calculateX(regions1["POS"], regions2["POS"], concept1, datas)
    y=calculateY(regions1["NEG"], regions2["NEG"], concept1, datas)

    mu = None
    xi = None
    if abs(alpha1-x) > 0:
        mu = (len(list(regions2["POS"]))-len(list(regions1["POS"])))/(abs(alpha1-x))
    if abs(y-beta1) > 0:
        xi = (len(list(regions2["NEG"]))-len(list(regions1["NEG"])))/(abs(y-beta1))

    return mu, xi

if __name__ == "__main__":
    for i in range(5,23):
        path = 'fludata_luigi/table_pairs/'+str(i)+'-'+str(i+1)+'/'
        mainNew(path, i)


###### import ######
from threading import Thread, Lock
from enum import Enum
import sys
from loguru import logger
from pyloramac import *
from ipaddress import IPv6Address, AddressValueError
from scapy.all import *
from time import *
###### LOG Configuration ######
LOG_FORMAT = "<green>("+str(perf_counter_ns())+"){time: HH:mm:ss.SSS}</green> | "+\
             "<level>{level: <8}</level> |"+\
             "{level.icon}  <cyan>{name}</cyan>"+\
             ":<cyan>{function}</cyan>:<cyan>{line}</cyan> "+\
             "- <level>{message}</level>"+\
             "\n{exception}"
LOG_CONFIG = {
    "handlers": [
        {"sink": sys.stdout, "format": lambda record: LOG_FORMAT, "level": "INFO"},
        #{"sink":"file_{time}.log", "format": lambda record: LOG_FORMAT, "level": "INFO"},
    ]
}
###### UDP Configuration ######
UDP_CLIENT_PORT = 8765
UDP_SERVER_PORT = 5678

SEND_INTERVAL = 5 # s
MAX_PAQUET_COUNT = 100
###### MAIN APP ######
class Mode(Enum):
    PINGPONG = 0
    RECEIVER = 1
    SENDER = 2

class Child:
    def __init__(self, mode: Mode, first_data: IPv6):
        self.mode = mode
        self.addr = first_data.src
        self.count = 0
        self.sender = None

        if mode == Mode.PINGPONG:
            self.receive(first_data)
        if mode == Mode.SENDER:
            self.sender = Thread(target=self.send)
            self.sender.start()

    def send(self):
        while self.count < MAX_PAQUET_COUNT:
            data = IPv6(src=NETWORK_STACK.node_ip_addr, dst=self.addr)/UDP(sport=UDP_SERVER_PORT, dport=UDP_CLIENT_PORT)/Raw(load=f"hello {self.count}")
            logger.log("APP", f"Send {data[UDP][Raw].load.decode()} to {data.dst}")
            NETWORK_STACK.send(data)
            sleep(SEND_INTERVAL)
            self.count += 1

    def receive(self, data):
        logger.log("APP", f"Receive {data[UDP][Raw].load.decode()} from {data.src}")
        new_data = IPv6(src=NETWORK_STACK.node_ip_addr, dst=self.addr)/UDP(sport=UDP_SERVER_PORT, dport=UDP_CLIENT_PORT)/Raw(load=f"PONG {self.count}")
        logger.log("APP", f"send {data[UDP][Raw].load.decode()} to {new_data.dst}")
        NETWORK_STACK.send(new_data)
        self.count +=1

class LoRaRoot:
    def __init__(self, mode:Mode):
        self.childs = {}
        self.mode = mode

    def init(self, port:str):
        NETWORK_STACK.init(port=port)
        NETWORK_STACK.register_listener(self.on_data)

    def on_data(self, data: IPv6):
        if mode == Mode.RECEIVER:
            logger.log("APP", f"Receive {data[UDP][Raw].load.decode()} from {data.src}")
        else:
            child = self.childs.get(data.src, None)
            if child is None:
                child = Child(self.mode, data)
                self.childs[data.src] = child
            elif self.mode == Mode.PINGPONG:
                child.receive(data)

if __name__ == "__main__":
    logger.configure(**LOG_CONFIG)
    logger.enable("pyloramac")
    logger.level("APP", no=26, color="<fg #00ABCC><i>", icon="\U0001F3D3")

    if len(sys.argv) != 3:
        logger.error("Usage: lora-root.py <port> <mode>")
        sys.exit(1)

    port = sys.argv[1]
    mode = Mode(int(sys.argv[2]))
    logger.log("APP", f"Mode: {mode}")

    try:
        LoRaRoot(mode).init(port)
    except KeyboardInterrupt:
        logger.info("KeyboardInterrupt")
        logger.info("Exiting...")
        sys.exit(0)


from utils.tree_util import create_binary_tree_from_array, inorder

def good_nodes(root):
    count = 0
    def util(head, max_till_now):
        if not head:
            return
        nonlocal count
        if head.val >= max_till_now:
            count += 1
            max_till_now = head.val
        util(head.left, max_till_now)
        util(head.right, max_till_now)
    util(root, float('-inf'))
    return count

array =  [3,1,4,3,None,1,5]
root = create_binary_tree_from_array(array)
print('--')
result = good_nodes(root)
print(result)