Mercurial > sat_docs

comparison flatpak/build_manifest.py @ 139:d36a68e396d5

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
flatpak: set appdata release version for dev version: appdata file can now use a template (which has the same name with "_tpl_" prefix), which is used in dev version to automatically set the release with the right Mercurial revision/date. Comment from org.salutatoi.Cagou.appdata.xml has been moved as first child of <component>, so it is not lost while parsing the file.
author Goffi <goffi@goffi.org>
date Sun, 23 Jun 2019 17:34:10 +0200
parents 274af514a5cf
children e23e414987d7
comparison
equal deleted inserted replaced
138:274af514a5cf 139:d36a68e396d5
6 from typing import List 6 from typing import List
7 from dataclasses import dataclass 7 from dataclasses import dataclass
8 import hashlib 8 import hashlib
9 from ftplib import FTP 9 from ftplib import FTP
10 from urllib.parse import urlparse 10 from urllib.parse import urlparse
11 from textwrap import dedent
11 import sys 12 import sys
12 import os 13 import os
13 import json 14 import json
14 import time 15 import time
15 import argparse 16 import argparse
16 import shutil 17 import shutil
17 from packaging.version import parse as parse_version 18 from packaging.version import parse as parse_version
18 import requests 19 import requests
20 from lxml import etree
19 21
20 22
21 CACHE_LIMIT = 3600 * 24 23 CACHE_LIMIT = 3600 * 24
22 PYTHON_DEP_OVERRIDE = { 24 PYTHON_DEP_OVERRIDE = {
23 "lxml": { 25 "lxml": {
83 "--filesystem=home" 85 "--filesystem=home"
84 ] 86 ]
85 } 87 }
86 SHOW_REQUIRES_HEADER = 'Requires: ' 88 SHOW_REQUIRES_HEADER = 'Requires: '
87 SETTINGS_KEY = '_build_settings' 89 SETTINGS_KEY = '_build_settings'
90 APPDATA_RELEASE_DEV_TEXT = dedent("""\
91 This is a development version, used as a preview.
92 Please note that it is incomplete and it probably contains bugs.
93 """)
94 OVERWRITE_WARNING = "{} already exists, do you want to overwrite it (y/N)? "
88 95
89 96
90 @dataclass 97 @dataclass
91 class Package: 98 class Package:
92 name: str 99 name: str
114 build_group = parser.add_argument_group('building', 'options used to generate the manifest') 121 build_group = parser.add_argument_group('building', 'options used to generate the manifest')
115 export_group = parser.add_argument_group('export', 'otions used to building files') 122 export_group = parser.add_argument_group('export', 'otions used to building files')
116 123
117 # build group 124 # build group
118 build_group.add_argument('-f', '--force', action="store_true", 125 build_group.add_argument('-f', '--force', action="store_true",
119 help="force overwritting of existing manifest") 126 help="force overwritting of existing manifest and appdata file")
120 build_group.add_argument('--ignore-cache', action='append', default=[], 127 build_group.add_argument('--ignore-cache', action='append', default=[],
121 help='ignore the cache of this step ("all" to ignore all caches)') 128 help='ignore the cache of this step ("all" to ignore all caches)')
122 build_group.add_argument( 129 build_group.add_argument(
123 '--deps-dir', 130 '--deps-dir',
124 help="use this directory to build_group python dependencies (it won't be deleted at " 131 help="use this directory to build_group python dependencies (it won't be deleted at "
363 # we use this name to easily ignore cache (with stem) while avoiding 370 # we use this name to easily ignore cache (with stem) while avoiding
364 # conflict if we have 2 URLs with the same stem 371 # conflict if we have 2 URLs with the same stem
365 step_name = f"{stem}__{url}" 372 step_name = f"{stem}__{url}"
366 373
367 if step_message is None: 374 if step_message is None:
368 step_messate = f"generating module for {stem}" 375 step_message = f"generating module for {stem}"
369 376
370 print_step(step_message) 377 print_step(step_message)
371 cache = get_cache(step_name) 378 cache = get_cache(step_name)
372 if cache is not None: 379 if cache is not None:
373 return cache 380 return cache
443 450
444 for data in reqs_data: 451 for data in reqs_data:
445 resolve_requirements(data, deps_map, deps, indent+1) 452 resolve_requirements(data, deps_map, deps, indent+1)
446 453
447 deps.append(package) 454 deps.append(package)
455
456
457 def get_hg_id_date(path):
458 """Get short identifier and date of current commit from given Mercurial repository
459
460 version is retrieve with `hg id`, a "+" is appended after shortrev if it has
461 been modified.
462 @param path(str, Path): path to the repository
463 @return(tuple(str, date)): found revision + iso date
464 """
465 hg_cplted = subprocess.run(
466 ["hg", "id", "--template", "{id|short}{dirty}\n{date|isodate}", path],
467 capture_output=True, text=True)
468 hg_cplted.check_returncode()
469 return hg_cplted.stdout.split('\n')
448 470
449 471
450 def get_cache_dir(): 472 def get_cache_dir():
451 """Return path to directory to use for cache""" 473 """Return path to directory to use for cache"""
452 return Path(f"cache_{app_id}") 474 return Path(f"cache_{app_id}")
583 dep = Package(name=name_canonical, 605 dep = Package(name=name_canonical,
584 version=version, 606 version=version,
585 hash_=dep_hash, 607 hash_=dep_hash,
586 url=version_json['url'], 608 url=version_json['url'],
587 requirements=requirements, 609 requirements=requirements,
588 # extra_requirements=extra_requirements,
589 ) 610 )
590 611
591 deps_map[name_canonical] = dep 612 deps_map[name_canonical] = dep
592 print(f"found dependency: {dep.name} {dep.version}") 613 print(f"found dependency: {dep.name} {dep.version}")
593 614
790 step_name = f"icon__{icon}", 811 step_name = f"icon__{icon}",
791 step_message = "retrieving application icon", 812 step_message = "retrieving application icon",
792 ) 813 )
793 814
794 815
816 def generate_appdata_from_template(template_file):
817 appdata_file = Path(f"{app_id}.appdata.xml")
818 if appdata_file.exists() and not args.force:
819 confirm = input(OVERWRITE_WARNING.format(appdata_file))
820 if confirm != 'y':
821 print("manifest building cancelled")
822 sys.exit(0)
823 parser = etree.XMLParser(remove_blank_text=True)
824 tree = etree.parse(template_file, parser)
825 root = tree.getroot()
826 if args.version == 'dev':
827 print("addind release data for dev version")
828 releases_elt = root.find('releases')
829 if releases_elt is None:
830 raise ValueError(
831 "<releases/> element is missing in appdata template, please add it")
832 release_elt = etree.SubElement(
833 releases_elt,
834 "release",
835 {'type': 'development',
836 'version': dev_version_rev,
837 'date': dev_version_date},
838 )
839 description_elt = etree.SubElement(release_elt, 'description')
840 text_lines = APPDATA_RELEASE_DEV_TEXT.strip().split('\n')
841 for text in text_lines:
842 etree.SubElement(description_elt, 'p').text = text
843 print(f"release data added for this version ({dev_version_rev})")
844
845 with open(appdata_file, "wb") as f:
846 f.write(etree.tostring(root, encoding="utf-8", xml_declaration=True,
847 pretty_print=True))
848
849 return appdata_file.as_posix()
850
851
795 def get_app_metadata(): 852 def get_app_metadata():
796 desktop_file = build_settings.get('desktop_file') 853 desktop_file = build_settings.get('desktop_file')
797 appdata_file = build_settings.get('appdata_file') 854 appdata_file = build_settings.get('appdata_file')
798 if desktop_file is None and app_data_file is None: 855 if desktop_file is None and appdata_file is None:
799 return 856 return
800 857
801 print_step("retrieving application metadata") 858 print_step("retrieving application metadata")
802 # we don't use cache here to be sure to have always up-to-date files 859 # we don't use cache here to be sure to have always up-to-date files
803 860
809 filename = desktop_file, 866 filename = desktop_file,
810 dest=f"/app/share/applications/{app_id}.desktop", 867 dest=f"/app/share/applications/{app_id}.desktop",
811 )) 868 ))
812 869
813 if appdata_file is not None: 870 if appdata_file is not None:
871 if appdata_file.startswith('_tpl_'):
872 print("found appdata template, we now use it to generate the file")
873 appdata_file = generate_appdata_from_template(appdata_file)
814 print("generating module for appdata metadata") 874 print("generating module for appdata metadata")
815 data.extend(file_upload( 875 data.extend(file_upload(
816 filename = appdata_file, 876 filename = appdata_file,
817 dest=f"/app/share/metainfo/{app_id}.appdata.xml", 877 dest=f"/app/share/metainfo/{app_id}.appdata.xml",
818 )) 878 ))
831 package_file = Path(f"{app_id}.json") 891 package_file = Path(f"{app_id}.json")
832 892
833 print(f"generating manifest for {app_id} ({args.version})") 893 print(f"generating manifest for {app_id} ({args.version})")
834 894
835 if package_file.exists() and not args.force: 895 if package_file.exists() and not args.force:
836 confirm = input( 896 confirm = input(OVERWRITE_WARNING.format(package_file))
837 f"{package_file} already exists, do you want to overwritte it (y/N)? ")
838 if confirm != 'y': 897 if confirm != 'y':
839 print("manifest building cancelled") 898 print("manifest building cancelled")
840 sys.exit(0) 899 sys.exit(0)
841 900
842 tpl_file = Path(f"_tpl_{package_file}") 901 tpl_file = Path(f"_tpl_{package_file}")
850 909
851 build_settings = template.pop(SETTINGS_KEY, {}) 910 build_settings = template.pop(SETTINGS_KEY, {})
852 if "setup_requirements" in build_settings: 911 if "setup_requirements" in build_settings:
853 PYTHON_SETUP_REQUIREMENTS.extend(build_settings["setup_requirements"]) 912 PYTHON_SETUP_REQUIREMENTS.extend(build_settings["setup_requirements"])
854 main_package = canonical(build_settings.get('package', args.name)) 913 main_package = canonical(build_settings.get('package', args.name))
855 if args.version == 'dev' and 'dev_repos' in build_settings: 914 if args.version == 'dev':
915 if 'dev_repos' not in build_settings:
916 raise NotImplementedError(
917 "dev version can currently only be handled with a dev repostiory "
918 "(dev_repos)")
856 dev_repos = build_settings['dev_repos'] 919 dev_repos = build_settings['dev_repos']
857 main_package_source = cache_from_repos() 920 main_package_source = cache_from_repos()
921 dev_version_rev, dev_version_date = get_hg_id_date(main_package_source)
858 else: 922 else:
859 main_package_source = main_package 923 main_package_source = main_package
860 924
861 manifest = {} 925 manifest = {}
862 manifest['app-id'] = app_id 926 manifest['app-id'] = app_id
901 965
902 # desktop file and appdata file 966 # desktop file and appdata file
903 modules.extend(get_app_metadata()) 967 modules.extend(get_app_metadata())
904 968
905 # now the app itself 969 # now the app itself
906 if args.version == 'dev' and 'dev_repos' in build_settings: 970 if args.version == 'dev':
907 # mercurial is needed for dev version to install but also to 971 # mercurial is needed for dev version to install but also to
908 # retrieve revision used 972 # retrieve revision used
909 modules.extend(get_python_package("mercurial")) 973 modules.extend(get_python_package("mercurial"))
910 modules.extend(get_repos_module()) 974 modules.extend(get_repos_module())
911 else: 975 else: