Mercurial > libervia-backend
diff sat_frontends/jp/cmd_blog.py @ 3040:fee60f17ebac
jp: jp asyncio port:
/!\ this commit is huge. Jp is temporarily not working with `dbus` bridge /!\
This patch implements the port of jp to asyncio, so it is now correctly using the bridge
asynchronously, and it can be used with bridges like `pb`. This also simplify the code,
notably for things which were previously implemented with many callbacks (like pagination
with RSM).
During the process, some behaviours have been modified/fixed, in jp and backends, check
diff for details.
author | Goffi <goffi@goffi.org> |
---|---|
date | Wed, 25 Sep 2019 08:56:41 +0200 |
parents | ab2696e34d29 |
children | d9f328374473 |
line wrap: on
line diff
--- a/sat_frontends/jp/cmd_blog.py Wed Sep 25 08:53:38 2019 +0200 +++ b/sat_frontends/jp/cmd_blog.py Wed Sep 25 08:56:41 2019 +0200 @@ -18,6 +18,16 @@ # along with this program. If not, see <http://www.gnu.org/licenses/>. +import json +import sys +import os.path +import os +import time +import tempfile +import subprocess +import asyncio +from asyncio.subprocess import DEVNULL +from pathlib import Path from . import base from sat.core.i18n import _ from sat_frontends.jp.constants import Const as C @@ -27,15 +37,6 @@ from sat.tools.common import uri from sat.tools import config from configparser import NoSectionError, NoOptionError -from functools import partial -import json -import sys -import os.path -import os -import time -import tempfile -import subprocess -import codecs from sat.tools.common import data_format __commands__ = ["Blog"] @@ -64,7 +65,7 @@ ) URL_REDIRECT_PREFIX = "url_redirect_" -INOTIFY_INSTALL = '"pip install inotify"' +AIONOTIFY_INSTALL = '"pip install aionotify"' MB_KEYS = ( "id", "url", @@ -86,7 +87,7 @@ OUTPUT_OPT_NO_HEADER = "no-header" -def guessSyntaxFromPath(host, sat_conf, path): +async def guessSyntaxFromPath(host, sat_conf, path): """Return syntax guessed according to filename extension @param sat_conf(ConfigParser.ConfigParser): instance opened on sat configuration @@ -101,19 +102,35 @@ return k # if not found, we use current syntax - return host.bridge.getParamA("Syntax", "Composition", "value", host.profile) + return await host.bridge.getParamA("Syntax", "Composition", "value", host.profile) class BlogPublishCommon(object): """handle common option for publising commands (Set and Edit)""" - @property - def current_syntax(self): - if self._current_syntax is None: - self._current_syntax = self.host.bridge.getParamA( + async def get_current_syntax(self): + """Retrieve current_syntax + + Use default syntax if --syntax has not been used, else check given syntax. + Will set self.default_syntax_used to True if default syntax has been used + """ + if self.args.syntax is None: + self.default_syntax_used = True + return await self.host.bridge.getParamA( "Syntax", "Composition", "value", self.profile ) - return self._current_syntax + else: + self.default_syntax_used = False + try: + syntax = await self.host.bridge.syntaxGet(self.current_syntax) + + self.current_syntax = self.args.syntax = syntax + except Exception as e: + if e.classname == "NotFound": + self.parser.error(_(f"unknown syntax requested ({self.args.syntax})")) + else: + raise e + return self.args.syntax def add_parser_options(self): self.parser.add_argument( @@ -143,14 +160,14 @@ help=_("syntax to use (default: get profile's default syntax)"), ) - def setMbDataContent(self, content, mb_data): - if self.args.syntax is None: + async def setMbDataContent(self, content, mb_data): + if self.default_syntax_used: # default syntax has been used mb_data["content_rich"] = content elif self.current_syntax == SYNTAX_XHTML: mb_data["content_xhtml"] = content else: - mb_data["content_xhtml"] = self.host.bridge.syntaxConvert( + mb_data["content_xhtml"] = await self.host.bridge.syntaxConvert( content, self.current_syntax, SYNTAX_XHTML, False, self.profile ) @@ -178,37 +195,35 @@ help=_("publish a new blog item or update an existing one"), ) BlogPublishCommon.__init__(self) - self.need_loop = True def add_parser_options(self): BlogPublishCommon.add_parser_options(self) - def mbSendCb(self): - self.disp("Item published") - self.host.quit(C.EXIT_OK) - - def start(self): - self._current_syntax = self.args.syntax + async def start(self): + self.current_syntax = await self.get_current_syntax() self.pubsub_item = self.args.item mb_data = {} self.setMbDataFromArgs(mb_data) if self.pubsub_item: mb_data["id"] = self.pubsub_item - content = codecs.getreader("utf-8")(sys.stdin).read() - self.setMbDataContent(content, mb_data) + content = sys.stdin.read() + await self.setMbDataContent(content, mb_data) - self.host.bridge.mbSend( - self.args.service, - self.args.node, - data_format.serialise(mb_data), - self.profile, - callback=self.exitCb, - errback=partial( - self.errback, - msg=_("can't send item: {}"), - exit_code=C.EXIT_BRIDGE_ERRBACK, - ), - ) + try: + await self.host.bridge.mbSend( + self.args.service, + self.args.node, + data_format.serialise(mb_data), + self.profile, + ) + except Exception as e: + self.disp( + f"can't send item: {e}", error=True + ) + self.host.quit(C.EXIT_BRIDGE_ERRBACK) + else: + self.disp("Item published") + self.host.quit(C.EXIT_OK) class Get(base.CommandBase): @@ -227,7 +242,6 @@ extra_outputs=extra_outputs, help=_("get blog item(s)"), ) - self.need_loop = True def add_parser_options(self): # TODO: a key(s) argument to select keys to display @@ -401,28 +415,25 @@ print(("\n" + sep + "\n")) - def mbGetCb(self, mb_result): - items, metadata = mb_result - items = [data_format.deserialise(i) for i in items] - mb_result = items, metadata - self.output(mb_result) - self.host.quit(C.EXIT_OK) - - def mbGetEb(self, failure_): - self.disp("can't get blog items: {reason}".format(reason=failure_), error=True) - self.host.quit(C.EXIT_BRIDGE_ERRBACK) - - def start(self): - self.host.bridge.mbGet( - self.args.service, - self.args.node, - self.args.max, - self.args.items, - self.getPubsubExtra(), - self.profile, - callback=self.mbGetCb, - errback=self.mbGetEb, - ) + async def start(self): + try: + mb_result = await self.host.bridge.mbGet( + self.args.service, + self.args.node, + self.args.max, + self.args.items, + self.getPubsubExtra(), + self.profile + ) + except Exception as e: + self.disp(f"can't get blog items: {e}", error=True) + self.host.quit(C.EXIT_BRIDGE_ERRBACK) + else: + items, metadata = mb_result + items = [data_format.deserialise(i) for i in items] + mb_result = items, metadata + await self.output(mb_result) + self.host.quit(C.EXIT_OK) class Edit(base.CommandBase, BlogPublishCommon, common.BaseEdit): @@ -452,25 +463,27 @@ def buildMetadataFile(self, content_file_path, mb_data=None): """Build a metadata file using json - The file is named after content_file_path, with extension replaced by _metadata.json - @param content_file_path(str): path to the temporary file which will contain the body + The file is named after content_file_path, with extension replaced by + _metadata.json + @param content_file_path(str): path to the temporary file which will contain the + body @param mb_data(dict, None): microblog metadata (for existing items) - @return (tuple[dict, str]): merged metadata put originaly in metadata file + @return (tuple[dict, Path]): merged metadata put originaly in metadata file and path to temporary metadata file """ # we first construct metadata from edited item ones and CLI argumments # or re-use the existing one if it exists - meta_file_path = os.path.splitext(content_file_path)[0] + common.METADATA_SUFF - if os.path.exists(meta_file_path): + meta_file_path = content_file_path.with_name( + content_file_path.stem + common.METADATA_SUFF) + if meta_file_path.exists(): self.disp("Metadata file already exists, we re-use it") try: - with open(meta_file_path, "rb") as f: + with meta_file_path.open("rb") as f: mb_data = json.load(f) except (OSError, IOError, ValueError) as e: self.disp( - "Can't read existing metadata file at {path}, aborting: {reason}".format( - path=meta_file_path, reason=e - ), + f"Can't read existing metadata file at {meta_file_path}, " + f"aborting: {e}", error=True, ) self.host.quit(1) @@ -491,7 +504,8 @@ with os.fdopen( os.open(meta_file_path, os.O_RDWR | os.O_CREAT | os.O_TRUNC, 0o600), "w+b" ) as f: - # we need to use an intermediate unicode buffer to write to the file unicode without escaping characters + # we need to use an intermediate unicode buffer to write to the file + # unicode without escaping characters unicode_dump = json.dumps( mb_data, ensure_ascii=False, @@ -503,19 +517,20 @@ return mb_data, meta_file_path - def edit(self, content_file_path, content_file_obj, mb_data=None): + async def edit(self, content_file_path, content_file_obj, mb_data=None): """Edit the file contening the content using editor, and publish it""" # we first create metadata file meta_ori, meta_file_path = self.buildMetadataFile(content_file_path, mb_data) + coroutines = [] + # do we need a preview ? if self.args.preview: self.disp("Preview requested, launching it", 1) # we redirect outputs to /dev/null to avoid console pollution in editor # if user wants to see messages, (s)he can call "blog preview" directly - DEVNULL = open(os.devnull, "wb") - subprocess.Popen( - [ + coroutines.append( + asyncio.create_subprocess_exec( sys.argv[0], "blog", "preview", @@ -523,30 +538,34 @@ "true", "-p", self.profile, - content_file_path, - ], - stdout=DEVNULL, - stderr=subprocess.STDOUT, + str(content_file_path), + stdout=DEVNULL, + stderr=DEVNULL, + ) ) # we launch editor - self.runEditor( - "blog_editor_args", - content_file_path, - content_file_obj, - meta_file_path=meta_file_path, - meta_ori=meta_ori, + coroutines.append( + self.runEditor( + "blog_editor_args", + content_file_path, + content_file_obj, + meta_file_path=meta_file_path, + meta_ori=meta_ori, + ) ) - def publish(self, content, mb_data): - self.setMbDataContent(content, mb_data) + await asyncio.gather(*coroutines) + + async def publish(self, content, mb_data): + await self.setMbDataContent(content, mb_data) if self.pubsub_item: mb_data["id"] = self.pubsub_item mb_data = data_format.serialise(mb_data) - self.host.bridge.mbSend( + await self.host.bridge.mbSend( self.pubsub_service, self.pubsub_node, mb_data, self.profile ) self.disp("Blog item published") @@ -555,22 +574,27 @@ # we get current syntax to determine file extension return SYNTAX_EXT.get(self.current_syntax, SYNTAX_EXT[""]) - def getItemData(self, service, node, item): + async def getItemData(self, service, node, item): items = [item] if item else [] - mb_data = self.host.bridge.mbGet(service, node, 1, items, {}, self.profile)[0][0] - mb_data = data_format.deserialise(mb_data) + + mb_data = await self.host.bridge.mbGet( + service, node, 1, items, {}, self.profile) + mb_data = data_format.deserialise(mb_data[0][0]) + try: content = mb_data["content_xhtml"] except KeyError: content = mb_data["content"] if content: - content = self.host.bridge.syntaxConvert( + content = await self.host.bridge.syntaxConvert( content, "text", SYNTAX_XHTML, False, self.profile ) + if content and self.current_syntax != SYNTAX_XHTML: - content = self.host.bridge.syntaxConvert( + content = await self.host.bridge.syntaxConvert( content, SYNTAX_XHTML, self.current_syntax, False, self.profile ) + if content and self.current_syntax == SYNTAX_XHTML: content = content.strip() if not content.startswith('<div>'): @@ -586,37 +610,16 @@ return content, mb_data, mb_data["id"] - def start(self): + async def start(self): # if there are user defined extension, we use them SYNTAX_EXT.update(config.getConfig(self.sat_conf, "jp", CONF_SYNTAX_EXT, {})) - self._current_syntax = self.args.syntax - if self._current_syntax is not None: - try: - self._current_syntax = self.args.syntax = self.host.bridge.syntaxGet( - self.current_syntax - ) - except Exception as e: - if "NotFound" in str( - e - ): # FIXME: there is not good way to check bridge errors - self.parser.error( - _("unknown syntax requested ({syntax})").format( - syntax=self.args.syntax - ) - ) - else: - raise e + self.current_syntax = await self.get_current_syntax() - ( - self.pubsub_service, - self.pubsub_node, - self.pubsub_item, - content_file_path, - content_file_obj, - mb_data, - ) = self.getItemPath() + (self.pubsub_service, self.pubsub_node, self.pubsub_item, content_file_path, + content_file_obj, mb_data,) = await self.getItemPath() - self.edit(content_file_path, content_file_obj, mb_data=mb_data) + await self.edit(content_file_path, content_file_obj, mb_data=mb_data) + self.host.quit() class Preview(base.CommandBase, common.BaseEdit): @@ -643,14 +646,14 @@ help=_("path to the content file"), ) - def showPreview(self): + async def showPreview(self): # we implement showPreview here so we don't have to import webbrowser and urllib # when preview is not used - url = "file:{}".format(self.urllib.quote(self.preview_file_path)) + url = "file:{}".format(self.urllib.parse.quote(self.preview_file_path)) self.webbrowser.open_new_tab(url) - def _launchPreviewExt(self, cmd_line, opt_name): - url = "file:{}".format(self.urllib.quote(self.preview_file_path)) + async def _launchPreviewExt(self, cmd_line, opt_name): + url = "file:{}".format(self.urllib.parse.quote(self.preview_file_path)) args = common.parse_args( self.host, cmd_line, url=url, preview_file=self.preview_file_path ) @@ -662,34 +665,34 @@ self.host.quit(1) subprocess.Popen(args) - def openPreviewExt(self): - self._launchPreviewExt(self.open_cb_cmd, "blog_preview_open_cmd") + async def openPreviewExt(self): + await self._launchPreviewExt(self.open_cb_cmd, "blog_preview_open_cmd") - def updatePreviewExt(self): - self._launchPreviewExt(self.update_cb_cmd, "blog_preview_update_cmd") + async def updatePreviewExt(self): + await self._launchPreviewExt(self.update_cb_cmd, "blog_preview_update_cmd") - def updateContent(self): - with open(self.content_file_path, "rb") as f: + async def updateContent(self): + with self.content_file_path.open("rb") as f: content = f.read().decode("utf-8-sig") if content and self.syntax != SYNTAX_XHTML: # we use safe=True because we want to have a preview as close as possible # to what the people will see - content = self.host.bridge.syntaxConvert( + content = await self.host.bridge.syntaxConvert( content, self.syntax, SYNTAX_XHTML, True, self.profile ) xhtml = ( - '<html xmlns="http://www.w3.org/1999/xhtml">' - '<head><meta http-equiv="Content-Type" content="text/html;charset=utf-8" />' - "</head>" - "<body>{}</body>" - "</html>" - ).format(content) + f'<html xmlns="http://www.w3.org/1999/xhtml">' + f'<head><meta http-equiv="Content-Type" content="text/html;charset=utf-8" />' + f'</head>' + f'<body>{content}</body>' + f'</html>' + ) with open(self.preview_file_path, "wb") as f: f.write(xhtml.encode("utf-8")) - def start(self): + async def start(self): import webbrowser import urllib.request, urllib.parse, urllib.error @@ -697,33 +700,24 @@ if self.args.inotify != "false": try: - import inotify.adapters - import inotify.constants - from inotify.calls import InotifyError + import aionotify + except ImportError: if self.args.inotify == "auto": - inotify = None + aionotify = None self.disp( - "inotify module not found, deactivating feature. You can install" - " it with {install}".format(install=INOTIFY_INSTALL) + f"aionotify module not found, deactivating feature. You can " + f"install it with {AIONOTIFY_INSTALL}" ) else: self.disp( - "inotify not found, can't activate the feature! Please install " - "it with {install}".format(install=INOTIFY_INSTALL), + f"aioinotify not found, can't activate the feature! Please " + f"install it with {AIONOTIFY_INSTALL}", error=True, ) self.host.quit(1) - else: - # we deactivate logging in inotify, which is quite annoying - try: - inotify.adapters._LOGGER.setLevel(40) - except AttributeError: - self.disp( - "Logger doesn't exists, inotify may have chanded", error=True - ) else: - inotify = None + aionotify = None sat_conf = config.parseMainConf() SYNTAX_EXT.update(config.getConfig(sat_conf, "jp", CONF_SYNTAX_EXT, {})) @@ -750,76 +744,89 @@ if self.args.file == "current": self.content_file_path = self.getCurrentFile(self.profile) else: - self.content_file_path = os.path.abspath(self.args.file) + try: + self.content_file_path = Path(self.args.file).resolve(strict=True) + except FileNotFoundError: + self.disp(_(f'File "{self.args.file}" doesn\'t exist!')) + self.host.quit(C.EXIT_NOT_FOUND) - self.syntax = guessSyntaxFromPath(self.host, sat_conf, self.content_file_path) + self.syntax = await guessSyntaxFromPath( + self.host, sat_conf, self.content_file_path) # at this point the syntax is converted, we can display the preview preview_file = tempfile.NamedTemporaryFile(suffix=".xhtml", delete=False) self.preview_file_path = preview_file.name preview_file.close() - self.updateContent() + await self.updateContent() - if inotify is None: - # XXX: we don't delete file automatically because browser need it + if aionotify is None: + # XXX: we don't delete file automatically because browser needs it # (and webbrowser.open can return before it is read) self.disp( - "temporary file created at {}\nthis file will NOT BE DELETED " - "AUTOMATICALLY, please delete it yourself when you have finished".format( - self.preview_file_path - ) + f"temporary file created at {self.preview_file_path}\nthis file will NOT " + f"BE DELETED AUTOMATICALLY, please delete it yourself when you have " + f"finished" ) - open_cb() + await open_cb() else: - open_cb() - i = inotify.adapters.Inotify( - block_duration_s=60 - ) # no need for 1 s duraction, inotify drive actions here + await open_cb() + watcher = aionotify.Watcher() + watcher_kwargs = { + # Watcher don't accept Path so we convert to string + "path": str(self.content_file_path), + "alias": 'content_file', + "flags": aionotify.Flags.CLOSE_WRITE + | aionotify.Flags.DELETE_SELF + | aionotify.Flags.MOVE_SELF, + } + watcher.watch(**watcher_kwargs) - def add_watch(): - i.add_watch( - self.content_file_path.encode('utf-8'), - mask=inotify.constants.IN_CLOSE_WRITE - | inotify.constants.IN_DELETE_SELF - | inotify.constants.IN_MOVE_SELF, - ) - - add_watch() + loop = asyncio.get_event_loop() + await watcher.setup(loop) try: - for event in i.event_gen(): - if event is not None: - self.disp("Content updated", 1) - if {"IN_DELETE_SELF", "IN_MOVE_SELF"}.intersection(event[1]): + while True: + event = await watcher.get_event() + self.disp("Content updated", 1) + if event.flags & (aionotify.Flags.DELETE_SELF + | aionotify.Flags.MOVE_SELF): + self.disp( + "DELETE/MOVE event catched, changing the watch", + 2, + ) + try: + watcher.unwatch('content_file') + except IOError as e: self.disp( - "{} event catched, changing the watch".format( - ", ".join(event[1]) - ), + f"Can't remove the watch: {e}", 2, ) - i.remove_watch(self.content_file_path) - try: - add_watch() - except InotifyError: - # if the new file is not here yet we can have an error - # as a workaround, we do a little rest - time.sleep(1) - add_watch() - self.updateContent() - update_cb() - except InotifyError: - self.disp( - "Can't catch inotify events, as the file been deleted?", error=True - ) + watcher = aionotify.Watcher() + watcher.watch(**watcher_kwargs) + try: + await watcher.setup(loop) + except OSError: + # if the new file is not here yet we can have an error + # as a workaround, we do a little rest and try again + await asyncio.sleep(1) + await watcher.setup(loop) + await self.updateContent() + await update_cb() + except FileNotFoundError: + self.disp("The file seems to have been deleted.", error=True) + self.host.quit(C.EXIT_NOT_FOUND) finally: os.unlink(self.preview_file_path) try: - i.remove_watch(self.content_file_path) - except InotifyError: - pass + watcher.unwatch('content_file') + except IOError as e: + self.disp( + f"Can't remove the watch: {e}", + 2, + ) -class Import(base.CommandAnswering): +class Import(base.CommandBase): def __init__(self, host): super(Import, self).__init__( host, @@ -828,7 +835,6 @@ use_progress=True, help=_("import an external blog"), ) - self.need_loop = True def add_parser_options(self): self.parser.add_argument( @@ -871,10 +877,10 @@ ), ) - def onProgressStarted(self, metadata): + async def onProgressStarted(self, metadata): self.disp(_("Blog upload started"), 2) - def onProgressFinished(self, metadata): + async def onProgressFinished(self, metadata): self.disp(_("Blog uploaded successfully"), 2) redirections = { k[len(URL_REDIRECT_PREFIX) :]: v @@ -897,42 +903,35 @@ ) self.disp( _( - "\nTo redirect old URLs to new ones, put the following lines in your" - " sat.conf file, in [libervia] section:\n\n{conf}".format(conf=conf) + f"\nTo redirect old URLs to new ones, put the following lines in your" + f" sat.conf file, in [libervia] section:\n\n{conf}" ) ) - def onProgressError(self, error_msg): - self.disp(_("Error while uploading blog: {}").format(error_msg), error=True) + async def onProgressError(self, error_msg): + self.disp(_(f"Error while uploading blog: {error_msg}"), error=True) - def error(self, failure): - self.disp( - _("Error while trying to upload a blog: {reason}").format(reason=failure), - error=True, - ) - self.host.quit(1) - - def start(self): + async def start(self): if self.args.location is None: for name in ("option", "service", "no_images_upload"): if getattr(self.args, name): self.parser.error( _( - "{name} argument can't be used without location argument" - ).format(name=name) + f"{name} argument can't be used without location argument" + ) ) if self.args.importer is None: self.disp( "\n".join( [ - "{}: {}".format(name, desc) - for name, desc in self.host.bridge.blogImportList() + f"{name}: {desc}" + for name, desc in await self.host.bridge.blogImportList() ] ) ) else: try: - short_desc, long_desc = self.host.bridge.blogImportDesc( + short_desc, long_desc = await self.host.bridge.blogImportDesc( self.args.importer ) except Exception as e: @@ -942,13 +941,7 @@ self.disp(msg) self.host.quit(1) else: - self.disp( - "{name}: {short_desc}\n\n{long_desc}".format( - name=self.args.importer, - short_desc=short_desc, - long_desc=long_desc, - ) - ) + self.disp(f"{self.args.importer}: {short_desc}\n\n{long_desc}") self.host.quit() else: # we have a location, an import is requested @@ -967,20 +960,23 @@ elif self.args.upload_ignore_host: options["upload_ignore_host"] = self.args.upload_ignore_host - def gotId(id_): - self.progress_id = id_ + try: + progress_id = await self.host.bridge.blogImport( + self.args.importer, + self.args.location, + options, + self.args.service, + self.args.node, + self.profile, + ) + except Exception as e: + self.disp( + _(f"Error while trying to import a blog: {e}"), + error=True, + ) + self.host.quit(1) - self.host.bridge.blogImport( - self.args.importer, - self.args.location, - options, - self.args.service, - self.args.node, - self.profile, - callback=gotId, - errback=self.error, - ) - + await self.set_progress_id(progress_id) class Blog(base.CommandBase): subcommands = (Set, Get, Edit, Preview, Import)