#!/usr/bin/env python3

import tempfile
import subprocess
from pathlib import Path
from typing import List
from dataclasses import dataclass
import hashlib
from ftplib import FTP
from urllib.parse import urlparse
import sys
import os
import json
import time
import argparse
import shutil
from packaging.version import parse as parse_version
import requests


CACHE_LIMIT = 3600 * 24
PYTHON_DEP_OVERRIDE = {
    "lxml": {
        "build-commands": [
            "python2 ./setup.py install --prefix=${FLATPAK_DEST} --optimize=1"
        ],
        "build-options": {
            "env": {
                "XSLT_CONFIG": "pkg-config libxslt"
            }
        },
    },
    "dbus-python": {
        "build-commands": [
            "python2 setup.py build",
            "python2 setup.py install --prefix=${FLATPAK_DEST}",
        ]
    },
    "urwid": {
        "sources_extend": [
            {
                "type": "patch",
                "path": "main_loop.patch"
            }
        ]
    },
    "kivy": {
        "post-install": [
            # this file is not copied normally when installing with
            # `python2 setup.py install`.
            # TODO: report upstream
            "cp kivy/setupconfig.py /app/lib/python2.7/site-packages/Kivy-*.egg/kivy/"
        ]
    },
}
PYTHON_DEP_REQUIREMENTS_UPDATE = {
    # service-identity is not seen as a twisted requirement, so it's sometimes misplaced
    'twisted': ['service-identity'],
}
PYTHON_SETUP_REQUIREMENTS = [
    'setuptools',  # to have an up-to-date version
    'setuptools_scm',
    'docutils',  # needed my m2r
    'mistune',  # needed my m2r
    'm2r',  # needed by automat
]
DEFAULT_MANIFEST = {
    "runtime": "org.freedesktop.Platform",
    "runtime-version": "1.6",
    "sdk": "org.freedesktop.Sdk",
    "sdk-extensions": [
        "org.freedesktop.Sdk.Debug",
        "org.freedesktop.Sdk.Locale",
        "org.freedesktop.Sdk.Docs"
    ],
    "platform-extensions": [
        "org.freedesktop.Platform.Locale"
    ],
    "command": "sat_wrapper",
    "finish-args": [
        "--socket=session-bus",
        "--share=network",
        "--filesystem=home"
    ]
}
SHOW_REQUIRES_HEADER = 'Requires: '
SETTINGS_KEY = '_build_settings'


@dataclass
class Package:
    name: str
    version: str
    hash_: str
    url: str
    requirements: List[str]

    def __hash__(self):
        return hash(self.name)


def print_step(step):
    print()
    print("┌" + "─" * (len(step) + 2) + "┐")
    print("│ {} │".format(step))
    print("└" + "─" * (len(step) + 2) + "┘")
    print()


def parse_args():
    parser = argparse.ArgumentParser(
        description='Flatpak manifest builder for Salut à Toi')

    build_group = parser.add_argument_group('building', 'options used to generate the manifest')
    export_group = parser.add_argument_group('export', 'otions used to building files')

    # build group
    build_group.add_argument('-f', '--force', action="store_true",
                        help="force overwritting of existing manifest")
    build_group.add_argument('--ignore-cache', action='append', default=[],
                        help='ignore the cache of this step ("all" to ignore all caches)')
    build_group.add_argument(
        '--deps-dir',
        help="use this directory to build_group python dependencies (it won't be deleted at "
             "the end, useful when you want to re-use it and avoir re-downloading)")
    build_group.add_argument('--no-binary', help="don't use binary packages")

    # export group
    export_group.add_argument('-s', '--symlink', choices=['no', 'all', 'cache'],
        default='no',
        help='"no" to copy all files, "all" to symlink all files, "cache" to copy files '
             'and symlink only cache (default: "no")')
    export_group.add_argument('-F', '--force-export', action='store_true',
        help='force overwritting of existing files/symlinks when exporting (DEFAULT: '
             'existing files are skipped)')
    export_group.add_argument('-e', '--export', type=Path,
        help="export build files to this directory (DEFAULT: don't export files)")

    # common
    parser.add_argument('name', type=str, help="name of the package to build_group")
    parser.add_argument('version', type=str, help="version of the package to build_group")

    args = parser.parse_args()

    if 'all' in args.ignore_cache and len(args.ignore_cache) != 1:
        parser.error('"all" cannot be used with step names in --ignore-cache, '
                     'use it alone')

    if args.export is not None and not args.export.is_dir():
        parser.error(f"{args.export} is not a directory!")

    return args


## useful methods ##

def get_cache(name):
    """Retrieve cache for a step

    cache will be ignored if too old, or explicitly ignored by user
    @param name(str): name of the step
    @return (object): cached data
    """
    # name of the cache without prefix, mainly used for python_deps
    shortname = name.split('__', 1)[0]

    if shortname in args.ignore_cache or 'all' in args.ignore_cache:
        print(f"ignoring cache for {shortname}")
        args.ignore_cache.remove(shortname)
        return None
    try:
        cache_name = cache[name]
        cache_updated = cache_name['updated']
    except KeyError:
        return None
    if time.time() - cache_updated > CACHE_LIMIT:
        print(f"cache for {name} is too old, we won't use it")
        return None
    print(f"cache found for {shortname}")
    return cache_name['data']


def set_cache(name, data):
    """Update cache for a step

    @param name(str): name of the step
    @param data(object): data to cache
    """
    cache_name = cache.setdefault(name, {})
    cache_name['updated'] = time.time()
    cache_name['data'] = data
    with cache_path.open('w') as f:
        json.dump(cache, f)


def get_python_package(package_name, step_name=None, step_message=None, with_pre=False,
                       version=None):
    """Generate module for a Python package

    @param package_name(str, list[str]): name of the Python package
        use list of str to get several packages at once
    @param step_name(None, str): name of the step, None to use package_name
    @param step_message(None, str): message of the step (None to use default one)
        use empty string to disable it
    @param with_pre(bool): if True retrieve also pre-releases
    @param version(str, None): indicate the version of the package to download
        if None, the most recent version compativle with `with_pre` will be used
    """
    single = isinstance(package_name, str)
    if step_name is None:
        step_name = package_name if single else ' - '.join(package_name)
    if step_message != "":
        print_step(step_message or f"retrieving latest version of {package_name}")
    cache = get_cache(step_name)
    if cache is not None:
        return cache
    package_names = [package_name] if single else package_name
    data = []
    for name in package_names:
        r = requests.get(f"https://pypi.org/pypi/{name}/json")
        r.raise_for_status()

        releases_data = []
        for version_raw, release_data in r.json()["releases"].items():
            if version is not None and version != version_raw:
                continue
            release_version = parse_version(version_raw)
            if release_version.is_prerelease and not with_pre:
                continue
            releases_data.append((release_version, release_data))

        # we sort releases by version to be sure to have latests one
        releases_data.sort(key=lambda d: d[0])

        try:
            release_version, release_data = releases_data[-1]
            package = next(r for r in release_data if r["packagetype"] == "sdist")
        except (IndexError, StopIteration):
            raise RuntimeError(f"Can't find a matching package for {name}")

        print(f"{name} {release_version} will be used")
        dep_data = {
            "name": name,
            "buildsystem": "simple",
            "build-commands": [
                "python2 setup.py install --prefix=${FLATPAK_DEST}"
            ],
            "sources": [
                {
                    "type": "archive",
                    "url": package["url"],
                    "sha256": package["digests"]["sha256"],
                }
            ],
            "ensure-writable": [
                "/lib/python2.7/site-packages/easy-install.pth",
                "/lib/python2.7/site-packages/setuptools.pth",
            ]
        }
        data.append(dep_data)

    set_cache(step_name, data)
    return data


def file_upload(filename, dest="/app/bin", src=None, replace=None, use_install=False):
    """Generate manifest modules to upload a local file

    @param filename(str): name of the local file to upload
        Note that it will be overwritted if replace is used
    @param dest(str, Path): path where the file must be installed in the container
    @param src(str, None): when replace is used, name of the source file
    @param replace(dict, None): mapping of text to replace
        Use when file must be dynamicly adapted to the container, note that src must be
        set to the name of the template file if this is used.
        e.g. {'##COMMAND##', 'cagou'} will replace all '##COMMAND##' in filename
        by 'cagou'
    @param use_install(bool): if True, install file with `install src dest`
        else, use `mv src dest`
    @return (list(dict)): modules to add to the manifest
    """
    assert isinstance(filename, str) and '/' not in filename
    print_step(f"generating {filename} module")
    # no cache for this one, to be sure to have always latest version
    filename = Path(filename)
    if src is None:
        file_to_test = filename
    else:
        src = Path(src)
        file_to_test = src
    if not file_to_test.exists():
        raise RuntimeError(
            f"{file_to_test} is missing, it is needed to build the manifest!")

    if replace:
        if src is None:
            raise ValueError(
                '"src" must be set to name of template file if replace is used')
        print(f'doing replacement in template "{src}" to dest "{filename}"')
        # there are strings to replace, we read file, do the changes and dump the
        # result in <filename>
        with open(src) as f:
            buff = f.read()

        for old, new in replace.items():
            buff = buff.replace(old, new)

        with open(filename, 'w') as f:
            f.write(buff)
    else:
        if src is not None:
            raise ValueError('"src" must not be used if replace is not used')

    with filename.open('rb') as f:
        hash_ = hashlib.sha256(f.read()).hexdigest()

    dest = Path(dest)

    dep_data = {
        "name": str(filename),
        "buildsystem": "simple",
        "build-commands": [
        ],
        "sources": [
            {
                "type": "file",
                "path": str(filename),
                "sha256": hash_,
            }
        ]
    }

    build_commands = dep_data['build-commands']
    if use_install:
        build_commands.append(f"install -Dv {filename} {dest}")
    else:
        if dest.as_posix() not in ('/app', '/app/bin'):
            # just in case the destination directory doesn't exist
            build_commands.append(f"mkdir -p {dest.parent}")
        build_commands.append(f"mv -v {filename} {dest}")

    return [dep_data]


def file_from_url(url, dest=None, step_name=None, step_message=None):
    """Generate manifest modules for a file either on a http(s) url or local

    @param url(str): url of the file to use, or local path
        if it starts with http, url will be used, else file_upload(url) will be used
    @param dest(str, None): path were the file should be copied
    """
    parsed = urlparse(url)
    if not parsed.scheme:
        return file_upload(url)
    if not parsed.scheme.startswith('http'):
        raise NotImplementedError (
            f'we can\'t use the URL "{url}", the scheme "{parsed.scheme}" is not managed')

    filepath = Path(parsed.path)
    stem = filepath.stem

    if step_name is None:
        # we use this name to easily ignore cache (with stem) while avoiding
        # conflict if we have 2 URLs with the same stem
        step_name = f"{stem}__{url}"

    if step_message is None:
        step_messate = f"generating module for {stem}"

    print_step(step_message)
    cache = get_cache(step_name)
    if cache is not None:
        return cache

    r = requests.get(url)
    r.raise_for_status()
    file_hash = hashlib.sha256(r.content).hexdigest()

    dep_data = {"name": stem}

    if dest is not None:
        dest = Path(dest)
        dep_data.update({
            'buildsystem': 'simple',
            'build-commands':[
                f'mkdir -p {dest.parent}',
                f'mv "{filepath.name}" "{dest}"',
            ]
        })

    dep_data['sources'] = [
        {
            "type": "file",
            "url": url,
            "sha256": file_hash,
        }
    ]

    data = [dep_data]
    set_cache(step_name, data)
    return data


def get_requirements(pip, package_name):
    """Retrieve requirements for an installed python package

    @param pip(Path): path to pip executable to use
        package must be installed in the environment of this pip
    @param package_name(str): name of the package to retrieve
    @return(list[str]): found requirements
    """
    show_cplted = subprocess.run([pip,  "show", package_name],
                                 capture_output=True, text=True)
    show_cplted.check_returncode()

    lines = show_cplted.stdout.split('\n')
    requirement_raw = next(l for l in lines if l.startswith(SHOW_REQUIRES_HEADER))
    requirement_raw = requirement_raw[len(SHOW_REQUIRES_HEADER):]
    requirements = [canonical(p) for p in requirement_raw.split(',') if p.strip()]
    requirements_update = [
        canonical(r) for r in PYTHON_DEP_REQUIREMENTS_UPDATE.get(package_name, [])]
    new_requirements = set(requirements_update).difference(requirements)
    if new_requirements:
        print("adding extra requirements to {}: {}".format(
            package_name, ', '.join(new_requirements)))
        requirements.extend(new_requirements)
    return requirements


def resolve_requirements(package, deps_map, deps, indent=0):
    """Recursively resolve requirements

    @param package(Package): retrieve dependencies of this package
    @param deps_map(dict): map from dependency name to Package instance
    @param deps(list[package]): ordered dependencies
        this list is updated in place
    @param indent(int): use internally to print dependencies tree
    """
    if package in deps:
        return
    print("    " * indent + package.name)
    reqs_data = [deps_map[r] for r in package.requirements]

    for data in reqs_data:
        resolve_requirements(data, deps_map, deps, indent+1)

    deps.append(package)


def get_cache_dir():
    """Return path to directory to use for cache"""
    return Path(f"cache_{app_id}")


def canonical(name):
    """Get canonical name of a package"""
    return name.lower().replace('_', '-').strip()


modules = []
cache_path = Path.home() / Path('.cache/sat_flatpak_cache.json')
if not cache_path.exists():
    cache = {}
else:
    with cache_path.open() as f:
        cache = json.load(f)


## steps ##

def get_libxslt():
    """Generate manifest module to install libxslt (needed for lxml)"""
    step_name = 'libxslt'
    print_step("retrieving latest version of libxslt")
    cache = get_cache(step_name)
    if cache is not None:
        return cache

    ftp = FTP("xmlsoft.org")
    ftp.login()
    ftp.cwd("libxml2")
    libxslt_archives = [l for l in ftp.nlst() if 'xslt' in l
                        and l.endswith('tar.gz')
                        and 'git' not in l
                        and 'rc' not in l]
    latest_libxslt = libxslt_archives[-1]
    print(f"latest libxslt found: {latest_libxslt}")

    with tempfile.TemporaryDirectory() as tmp_dirname:
        tmp_dir = Path(tmp_dirname)
        file_path = tmp_dir / latest_libxslt
        with file_path.open('wb+') as f:
            ftp.retrbinary('RETR ' + latest_libxslt, f.write)
            f.seek(0)
            libxslt_hash = hashlib.sha256(f.read()).hexdigest()

    ftp.quit()

    print(f"libxstl hash: {libxslt_hash}")

    data = [{
        "name": "libxslt",
        "sources": [
            {
                "type": "archive",
                "url": f"ftp://xmlsoft.org/libxml2/{latest_libxslt}",
                "sha256": libxslt_hash,
            }
        ]
    }]
    set_cache(step_name, data)
    return data


def get_python_deps():
    """Generate manifest modules for python dependencies of main package"""
    step_name = f'python_deps__{app_id}'
    print_step("retrieving python dependencies")
    cache = get_cache(step_name)
    if cache is not None:
        return cache

    with tempfile.TemporaryDirectory() as tmp_dirname:
        if args.deps_dir is not None:
            # we ignore the created temporary directory is we have an other one specified
            tmp_dirname = args.deps_dir
        tmp_dir = Path(tmp_dirname)
        env_dir = tmp_dir / 'env'
        pip = env_dir / 'bin' / 'pip'
        download_dir = tmp_dir / 'archives'
        if download_dir.exists() and env_dir.exists():
            print("dependencies are already downloaded and packages are already "
                  "installed")
        else:
            download_dir.mkdir()
            print(f"working in temporary directory {tmp_dirname}")
            venv_cplted = subprocess.run(["/usr/bin/env", "virtualenv2", env_dir])
            venv_cplted.check_returncode()
            print("\ndownloading packages")
            command_args = [pip,  "download", "-d", download_dir,
                            "--progress-bar", "emoji"]
            if args.no_binary:
                command_args.extend(["--no-binary", ":all:"])
            command_args.append(main_package_source)
            down_cplted = subprocess.run(command_args)
            down_cplted.check_returncode()
            print("\ninstalling package\n")
            # we install package to have a good map of requirements, pypi metadata
            # are incomplete. Packages should be retrieved from pip cache
            inst_cplted = subprocess.run([pip, "install", main_package_source])
            inst_cplted.check_returncode()

        print_step("analyzing python dependencies")
        deps_map = {}

        for archive_path in download_dir.iterdir():
            name, right_part = archive_path.name.rsplit('-', 1)
            name_canonical = canonical(name)
            if right_part.endswith('.tar.gz'):
                version = right_part[:-7]
            elif right_part.endswith('.tar.bz2'):
                version = right_part[:-8]
            elif right_part.endswith('.zip'):
                version = right_part[:-4]
            else:
                raise ValueError(
                    f"Archive are supposed to be .tar.gz, .tar.bz2 or .zip archives, but "
                    f"file found is {archive_path.name}, did something change on Pypy?")
            with open(archive_path, "rb") as f:
                dep_hash = hashlib.sha256(f.read()).hexdigest()
            r = requests.get(f"https://pypi.org/pypi/{name}/{version}/json")
            r.raise_for_status()
            dep_json = r.json()
            release_json = dep_json["releases"][version]
            try:
                version_json = next(v for v in release_json
                                    if v['digests']['sha256'] == dep_hash)
            except IndexError:
                raise ValueError(f"Can't find the version we downloaded for {name}")

            requirements = get_requirements(pip, name_canonical)

            dep = Package(name=name_canonical,
                          version=version,
                          hash_=dep_hash,
                          url=version_json['url'],
                          requirements=requirements,
                          # extra_requirements=extra_requirements,
                          )

            deps_map[name_canonical] = dep
            print(f"found dependency: {dep.name} {dep.version}")

    print_step("ordering dependencies")

    requirements = get_requirements(pip, main_package)
    main_pack = Package(name=main_package,
                        version=args.version,
                        hash_="",
                        url="",
                        requirements=requirements,
                        )

    deps = []
    print("resolving requirements\n")
    resolve_requirements(main_pack, deps_map, deps)
    missing_deps = set(deps_map.values()).difference(deps)
    if missing_deps:
        print("\n/!\\ some dependencies are missing this should not happen! "
              "Adding them\n")
        print("additional requirements")
        for pack in missing_deps:
            resolve_requirements(pack, deps_map, deps)

    # we remove deps already installed with PYTHON_SETUP_REQUIREMENTS
    for name in PYTHON_SETUP_REQUIREMENTS:
        package_data = deps_map.get(name)
        if package_data is not None:
            deps.remove(package_data)
            print(f"removed {name} which is already a setup requirement")

    # main package is installed at the end
    deps.remove(main_pack)

    print("\npackages are now ordered: {}".format(", ".join(d.name for d in deps)))

    print("\nwe now generate modules for python dependencies")
    data = []
    version_force = {
        canonical(n): v
        for n,v in build_settings.get('python_deps_version_force', {}).items()
    }

    for dep in deps:
        version_forced = version_force.get(dep.name)
        if version_forced == 'pre-release':
            print(f"using pre-release version for {dep.name} as requested in build "
                  "settings")
            dep_data = get_python_package(dep.name, step_message='', with_pre=True)[0]
        elif version_forced is not None:
            print(f"using version {version_forced} for {dep.name} as requested in "
                  "build settings")
            dep_data = get_python_package(dep.name, step_message='',
                                          version=version_forced)[0]
        else:
            dep_data = {
                "name": dep.name,
                "buildsystem": "simple",
                "build-commands": [
                    "python2 setup.py install --prefix=${FLATPAK_DEST}"
                ],
                "sources": [
                    {
                        "type": "archive",
                        "url": dep.url,
                        "sha256": dep.hash_,
                    }
                ],
                "ensure-writable": [
                    "/lib/python2.7/site-packages/easy-install.pth",
                    "/lib/python2.7/site-packages/setuptools.pth",
                ]
            }

        if dep.name in PYTHON_DEP_OVERRIDE:
            print(f"applying override for {dep.name}")
            override = PYTHON_DEP_OVERRIDE[dep.name]

            # key suffixed with "_extend" won't replace the data
            for key in list(override.keys()):
                if key.endswith('_extend'):
                    real_key = key[:-7]
                    extend_data = override.pop(key)
                    if real_key == 'sources':
                        for item_data in extend_data:
                            # we look for missing hashes and add them
                            if (item_data.get('type') in ('file', 'patch')
                                and 'sha256' not in item_data):
                                with open(item_data['path'], 'rb') as f:
                                    hash_ = hashlib.sha256(f.read()).hexdigest()
                                item_data['sha256'] = hash_

                    dep_data.setdefault(real_key, []).extend(extend_data)

            dep_data.update(override)

        data.append(dep_data)

    set_cache(step_name, data)
    return data


def cache_from_repos():
    """Get and cache locally a repository and returns cache path"""
    print_step("retrieving code from repository")
    dev_repos = build_settings['dev_repos']
    repos_type = dev_repos.get('type', 'hg')
    if repos_type != 'hg':
        raise NotImplementedError("only Mercurial is managed so far")
    url = dev_repos['url']
    dest_path = get_cache_dir() / main_package
    dest_path.mkdir(parents=True, exist_ok=True)
    if (dest_path / '.hg').is_dir():
        print("code is already in cache")
    else:
        try:
            print(f"retrieving code from repository {url} into {dest_path}")
            hg_cplted = subprocess.run(["hg",  "clone", url, dest_path])
            hg_cplted.check_returncode()
        except Exception as e:
            shutil.rmtree(dest_path)
            raise e
    return dest_path


def get_repos_module():
    """Generate manifest module for the repository"""
    dep_data = {
        "name": main_package,
        "buildsystem": "simple",
        "build-commands": [
            "python2 setup.py install --prefix=${FLATPAK_DEST}"
        ],
        "sources": [
            {
                "type": "dir",
                "path": str(main_package_source)
            }
        ],
        "ensure-writable": [
            "/lib/python2.7/site-packages/easy-install.pth",
            "/lib/python2.7/site-packages/setuptools.pth",
        ]
    }
    return [dep_data]


def get_sat_media():
    """Generate module for last version of sat_media available on the FTP"""
    step_name = 'sat_media'
    print_step("retrieving latest version of sat_media")
    cache = get_cache(step_name)
    if cache is not None:
        return cache
    url = "https://ftp.goffi.org/sat_media/sat_media.tar.bz2"
    r = requests.get(url)
    r.raise_for_status()
    hash_ = hashlib.sha256(r.content).hexdigest()
    dep_data = {
        "name": "sat-media",
        "buildsystem": "simple",
        "build-commands": [
            "cp -vr . ${FLATPAK_DEST}/share/sat-media"
        ],
        "sources": [
            {
                "type": "archive",
                "url": url,
                "sha256": hash_,
            }
        ],
    }

    data = [dep_data]
    set_cache(step_name, data)
    return data


def get_icon():
    icon = build_settings.get('icon')
    if icon is not None:
        if isinstance(icon, str):
            icon = {'url': icon}
        icon_path = Path(urlparse(icon['url']).path)
        suffix = icon_path.suffix[1:]
        if suffix not in ('svg', 'png'):
            raise ValueError("invalid icon, you must use a SVG or PNG image!")
        if 'size' not in icon:
            if suffix == 'svg':
                icon['size'] = 'scalable'
            else:
                raise ValueError('icon size is not specified, please add a "size" key')

        dest_path = f"/app/share/icons/hicolor/{icon['size']}/apps/{app_id}.{suffix}"

        return file_from_url(
            url = icon['url'],
            dest = dest_path,
            # we have common cache if several manifest use the same icon URL
            step_name = f"icon__{icon}",
            step_message = "retrieving application icon",
        )


def get_app_metadata():
    desktop_file = build_settings.get('desktop_file')
    appdata_file = build_settings.get('appdata_file')
    if desktop_file is None and app_data_file is None:
        return

    print_step("retrieving application metadata")
    # we don't use cache here to be sure to have always up-to-date files

    data = []

    if desktop_file is not None:
        print("generating module for desktop metadata")
        data.extend(file_upload(
            filename = desktop_file,
            dest=f"/app/share/applications/{app_id}.desktop",
            ))

    if appdata_file is not None:
        print("generating module for appdata metadata")
        data.extend(file_upload(
            filename = appdata_file,
            dest=f"/app/share/metainfo/{app_id}.appdata.xml",
            ))

    return data


## main_script ##


if __name__ == '__main__':

    args = parse_args()
    title = args.name.title()
    app_id = f"org.salutatoi.{title}"
    package_file = Path(f"{app_id}.json")

    print(f"generating manifest for {app_id} ({args.version})")

    if package_file.exists() and not args.force:
        confirm = input(
            f"{package_file} already exists, do you want to overwritte it (y/N)? ")
        if confirm != 'y':
            print("manifest building cancelled")
            sys.exit(0)

    tpl_file = Path(f"_tpl_{package_file}")

    if not tpl_file.exists():
        raise RuntimeError(f"Can't find template {tpl_file}, it is mandatory to build"
                           "the manifest!")

    with tpl_file.open() as f:
        template = json.load(f)

    build_settings = template.pop(SETTINGS_KEY, {})
    if "setup_requirements" in build_settings:
        PYTHON_SETUP_REQUIREMENTS.extend(build_settings["setup_requirements"])
    main_package = canonical(build_settings.get('package', args.name))
    if args.version == 'dev' and 'dev_repos' in build_settings:
        dev_repos = build_settings['dev_repos']
        main_package_source = cache_from_repos()
    else:
        main_package_source = main_package

    manifest = {}
    manifest['app-id'] = app_id
    manifest['default-branch'] = args.version
    # we update DEFAULT_MANIFEST only now to have "app-id" and "default-branch" on the top
    # of the manifest
    manifest.update(DEFAULT_MANIFEST)
    manifest.update(template)

    modules.extend(get_libxslt())

    # setup requirements
    modules.extend(get_python_package(
        PYTHON_SETUP_REQUIREMENTS,
        step_name="setup_requirements",
        step_message="generating modules for setup requirements")
    )

    # python dependencies
    modules.extend(get_python_deps())

    # at this point we add things specified in the template
    modules.extend(manifest.get('modules', []))
    manifest['modules'] = modules

    # sat common things
    existing_modules = {d['name'] for d in modules}
    if "sat_templates" not in existing_modules:
        modules.extend(get_python_package("sat_templates", with_pre=True))
    modules.extend(get_sat_media())
    modules.extend(file_upload('sat.conf', '/app'))

    # wrapper to launch the backend if needed
    wrapped_command = build_settings.get('wrapped_command')
    if wrapped_command:
        modules.extend(file_upload('sat_wrapper', src='sat_wrapper.py',
                                   replace={'##COMMAND##': wrapped_command},
                                   use_install=True))

    # icon
    modules.extend(get_icon())

    # desktop file and appdata file
    modules.extend(get_app_metadata())

    # now the app itself
    if args.version == 'dev' and 'dev_repos' in build_settings:
        # mercurial is needed for dev version to install but also to
        # retrieve revision used
        modules.extend(get_python_package("mercurial"))
        modules.extend(get_repos_module())
    else:
        modules.extend(get_python_package(main_package, version=args.version))

    print_step("writing manifest")
    with package_file.open('w') as f:
        json.dump(manifest, f, indent=4)

    if args.ignore_cache:
        print("/!\\ those --ignore-cache arguments don't correspond to any step: {}".
              format(', '.join(args.ignore_cache)))

    print(f"manifest generated successfully at {package_file}")

    if args.export is not None:
        print_step("exporting building files")
        print(f"exporting to {args.export}\n")
        to_export = [
            ("package file", package_file),
        ]

        for m in modules:
            for s in m.get('sources', []):
                s_type = s.get('type')
                if s_type in ('path', 'file'):
                    try:
                        path = s['path']
                    except KeyError:
                        if 'url' in s:
                            continue
                        else:
                            raise ValueError(f"path missing for module source:\n{s}")
                    to_export.append((s_type, Path(path)))

        for label, path in to_export:
            print(f"exporting {label}: {path}")
            dest = args.export / path
            if dest.exists():
                if args.force_export:
                    print(f"    replacing existing {dest}")
                    if path.is_dir():
                        shutil.rmtree(dest)
                    else:
                        dest.unlink()
                else:
                    print("    it already exists, skipping")
                    continue
            if args.symlink == 'all':
                os.symlink(path, dest)
            else:
                if path.is_dir():
                    shutil.copytree(path, dest)
                else:
                    shutil.copyfile(path, dest)

        if args.version == 'dev':
            print("exporting cache")
            dest = args.export / get_cache_dir()
            if args.force_export and os.path.lexists(dest):
                print(f"    replacing existing {dest}")
                if dest.is_symlink():
                    dest.unlink()
                else:
                    shutil.rmtree(dest)

            if dest.exists():
                print("    it already exists, skipping")
            else:
                if args.symlink in ('all', 'cache'):
                    os.symlink(get_cache_dir().resolve(), dest)
                else:
                    shutil.copytree(get_cache_dir(), dest)

        print("\nexport completed")