libervia-backend: sat_frontends/jp/common.py comparison

comparison sat_frontends/jp/common.py @ 3040:fee60f17ebac

jp: jp asyncio port: /!\ this commit is huge. Jp is temporarily not working with `dbus` bridge /!\ This patch implements the port of jp to asyncio, so it is now correctly using the bridge asynchronously, and it can be used with bridges like `pb`. This also simplify the code, notably for things which were previously implemented with many callbacks (like pagination with RSM). During the process, some behaviours have been modified/fixed, in jp and backends, check diff for details.

author	Goffi <goffi@goffi.org>
date	Wed, 25 Sep 2019 08:56:41 +0200
parents	d314d4181f30
children	d9f328374473

comparison

equal deleted inserted replaced

-:a1bc34f90fa5
+:fee60f17ebac
 # GNU Affero General Public License for more details.
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
+import json
+import os
+import os.path
+import time
+import tempfile
+import asyncio
+import shlex
+from pathlib import Path
 from sat_frontends.jp.constants import Const as C
 from sat.core.i18n import _
 from sat.core import exceptions
 from sat.tools.common import regex
 from sat.tools.common.ansi import ANSI as A
 from sat.tools.common import uri as xmpp_uri
 from sat.tools import config
 from configparser import NoSectionError, NoOptionError
 from collections import namedtuple
-from functools import partial
-import json
-import os
-import os.path
-import time
-import tempfile
-import subprocess
-import glob
-import shlex
 # defaut arguments used for some known editors (editing with metadata)
 VIM_SPLIT_ARGS = "-c 'set nospr|vsplit|wincmd w|next|wincmd w'"
 EMACS_SPLIT_ARGS = '--eval "(split-window-horizontally)"'
 EDITOR_ARGS_MAGIC = {
 def getTmpDir(sat_conf, cat_dir, sub_dir=None):
 """Return directory used to store temporary files
 @param sat_conf(ConfigParser.ConfigParser): instance opened on sat configuration
-@param cat_dir(unicode): directory of the category (e.g. "blog")
+@param cat_dir(str): directory of the category (e.g. "blog")
 @param sub_dir(str): sub directory where data need to be put
 profile can be used here, or special directory name
 sub_dir will be escaped to be usable in path (use regex.pathUnescape to find
 initial str)
-@return (str): path to the dir
+@return (Path): path to the dir
 """
 local_dir = config.getConfig(sat_conf, "", "local_dir", Exception)
-path = [local_dir.encode("utf-8"), cat_dir.encode("utf-8")]
+path_elts = [local_dir, cat_dir]
 if sub_dir is not None:
-path.append(regex.pathEscape(sub_dir))
+path_elts.append(regex.pathEscape(sub_dir))
-return os.path.join(*path)
+return Path(*path_elts)
 def parse_args(host, cmd_line, **format_kw):
 """Parse command arguments
 """
 @param sat_conf(ConfigParser.ConfigParser): instance opened on sat configuration
 @param cat_dir(unicode): directory to use for drafts
 this will be a sub-directory of SàT's local_dir
 @param use_metadata(bool): True is edition need a second file for metadata
-most of signature change with use_metadata with an additional metadata argument.
+most of signature change with use_metadata with an additional metadata
-This is done to raise error if a command needs metadata but forget the flag, and vice versa
+argument.
+This is done to raise error if a command needs metadata but forget the flag,
+and vice versa
 """
 self.host = host
 self.sat_conf = config.parseMainConf()
-self.cat_dir_str = cat_dir.encode("utf-8")
+self.cat_dir = cat_dir
 self.use_metadata = use_metadata
 def secureUnlink(self, path):
 """Unlink given path after keeping it for a while
 This method is used to prevent accidental deletion of a draft
 If there are more file in SECURE_UNLINK_DIR than SECURE_UNLINK_MAX,
 older file are deleted
-@param path(str): file to unlink
+@param path(Path, str): file to unlink
 """
-if not os.path.isfile(path):
+path = Path(path).resolve()
+if not path.is_file:
 raise OSError("path must link to a regular file")
-if not path.startswith(getTmpDir(self.sat_conf, self.cat_dir_str)):
+if path.parent != getTmpDir(self.sat_conf, self.cat_dir):
 self.disp(
-"File {} is not in SàT temporary hierarchy, we do not remove it".format(
+f"File {path} is not in SàT temporary hierarchy, we do not remove "
-path
+f"it",
-),
 2,
 )
 return
 # we have 2 files per draft with use_metadata, so we double max
 unlink_max = SECURE_UNLINK_MAX * 2 if self.use_metadata else SECURE_UNLINK_MAX
-backup_dir = getTmpDir(self.sat_conf, self.cat_dir_str, SECURE_UNLINK_DIR)
+backup_dir = getTmpDir(self.sat_conf, self.cat_dir, SECURE_UNLINK_DIR)
 if not os.path.exists(backup_dir):
 os.makedirs(backup_dir)
 filename = os.path.basename(path)
 backup_path = os.path.join(backup_dir, filename)
 # we move file to backup dir
 backup_files.sort(key=lambda path: os.stat(path).st_mtime)
 for path in backup_files[: len(backup_files) - unlink_max]:
 self.host.disp("Purging backup file {}".format(path), 2)
 os.unlink(path)
-def runEditor(
+async def runEditor(self, editor_args_opt, content_file_path, content_file_obj,
-self,
+meta_file_path=None, meta_ori=None):
-editor_args_opt,
+"""Run editor to edit content and metadata
-content_file_path,
-content_file_obj,
-meta_file_path=None,
-meta_ori=None,
-):
-"""run editor to edit content and metadata
 @param editor_args_opt(unicode): option in [jp] section in configuration for
 specific args
 @param content_file_path(str): path to the content file
 @param content_file_obj(file): opened file instance
-@param meta_file_path(str, None): metadata file path
+@param meta_file_path(str, Path, None): metadata file path
 if None metadata will not be used
 @param meta_ori(dict, None): original cotent of metadata
 can't be used if use_metadata is False
 """
 if not self.use_metadata:
 args = parse_args(self.host, editor_args, **parse_kwargs)
 if not args:
 args = [content_file_path]
 # actual editing
-editor_exit = subprocess.call([editor] + args)
+editor_process = await asyncio.create_subprocess_exec(
+editor, *[str(a) for a in args])
+editor_exit = await editor_process.wait()
 # edition will now be checked, and data will be sent if it was a success
 if editor_exit != 0:
 self.disp(
-"Editor exited with an error code, so temporary file has not be deleted, and item is not published.\nYou can find temporary file at {path}".format(
+f"Editor exited with an error code, so temporary file has not be "
-path=content_file_path
+f"deleted, and item is not published.\nYou can find temporary file "
-),
+f"at {content_file_path}",
 error=True,
 )
 else:
 # main content
 try:
-with open(content_file_path, "rb") as f:
+with content_file_path.open("rb") as f:
 content = f.read()
 except (OSError, IOError):
 self.disp(
-"Can read file at {content_path}, have it been deleted?\nCancelling edition".format(
+f"Can read file at {content_file_path}, have it been deleted?\n"
-content_path=content_file_path
+f"Cancelling edition",
-),
 error=True,
 )
 self.host.quit(C.EXIT_NOT_FOUND)
 # metadata
 if self.use_metadata:
 try:
-with open(meta_file_path, "rb") as f:
+with meta_file_path.open("rb") as f:
 metadata = json.load(f)
 except (OSError, IOError):
 self.disp(
-"Can read file at {meta_file_path}, have it been deleted?\nCancelling edition".format(
+f"Can read file at {meta_file_path}, have it been deleted?\n"
-content_path=content_file_path, meta_path=meta_file_path
+f"Cancelling edition",
-),
 error=True,
 )
 self.host.quit(C.EXIT_NOT_FOUND)
 except ValueError:
 self.disp(
-"Can't parse metadata, please check it is correct JSON format. Cancelling edition.\n"
+f"Can't parse metadata, please check it is correct JSON format. "
-+ "You can find tmp file at {content_path} and temporary meta file at {meta_path}.".format(
+f"Cancelling edition.\nYou can find tmp file at "
-content_path=content_file_path, meta_path=meta_file_path
+f"{content_file_path} and temporary meta file at "
-),
+f"{meta_file_path}.",
 error=True,
 )
 self.host.quit(C.EXIT_DATA_ERROR)
 if self.use_metadata and not metadata.get("publish", True):
 self.disp(
-'Publication blocked by "publish" key in metadata, cancelling edition.\n\n'
+f'Publication blocked by "publish" key in metadata, cancelling '
-+ "temporary file path:\t{content_path}\nmetadata file path:\t{meta_path}".format(
+f'edition.\n\ntemporary file path:\t{content_file_path}\nmetadata '
-content_path=content_file_path, meta_path=meta_file_path
+f'file path:\t{meta_file_path}',
-),
 error=True,
 )
 self.host.quit()
 if len(content) == 0:
 self.disp("Content is empty, cancelling the edition")
-if not content_file_path.startswith(
+if content_file_path.parent != getTmpDir(self.sat_conf, self.cat_dir):
-getTmpDir(self.sat_conf, self.cat_dir_str)
-):
 self.disp(
 "File are not in SàT temporary hierarchy, we do not remove them",
 2,
 )
 self.host.quit()
-self.disp("Deletion of {}".format(content_file_path), 2)
+self.disp(f"Deletion of {content_file_path}", 2)
 os.unlink(content_file_path)
 if self.use_metadata:
-self.disp("Deletion of {}".format(meta_file_path), 2)
+self.disp(f"Deletion of {meta_file_path}".format(meta_file_path), 2)
 os.unlink(meta_file_path)
 self.host.quit()
 # time to re-check the hash
 elif tmp_ori_hash == hashlib.sha1(content).digest() and (
 else:
 # we can now send the item
 content = content.decode("utf-8-sig")  # we use utf-8-sig to avoid BOM
 try:
 if self.use_metadata:
-self.publish(content, metadata)
+await self.publish(content, metadata)
 else:
-self.publish(content)
+await self.publish(content)
 except Exception as e:
 if self.use_metadata:
 self.disp(
-"Error while sending your item, the temporary files have been kept at {content_path} and {meta_path}: {reason}".format(
+f"Error while sending your item, the temporary files have "
-content_path=content_file_path,
+f"been kept at {content_file_path} and {meta_file_path}: "
-meta_path=meta_file_path,
+f"{e}",
-reason=e,
-),
 error=True,
 )
 else:
 self.disp(
-"Error while sending your item, the temporary file has been kept at {content_path}: {reason}".format(
+f"Error while sending your item, the temporary file has been "
-content_path=content_file_path, reason=e
+f"kept at {content_file_path}: {e}",
-),
 error=True,
 )
 self.host.quit(1)
 self.secureUnlink(content_file_path)
 if self.use_metadata:
 self.secureUnlink(meta_file_path)
-def publish(self, content):
+async def publish(self, content):
 # if metadata is needed, publish will be called with it last argument
 raise NotImplementedError
 def getTmpFile(self):
 """Create a temporary file
-@param suff (str): suffix to use for the filename
+@return (tuple(file, Path)): opened (w+b) file object and file path
-@return (tuple(file, str)): opened (w+b) file object and file path
 """
 suff = "." + self.getTmpSuff()
-cat_dir_str = self.cat_dir_str
+cat_dir_str = self.cat_dir
-tmp_dir = getTmpDir(self.sat_conf, self.cat_dir_str, self.profile.encode("utf-8"))
+tmp_dir = getTmpDir(self.sat_conf, self.cat_dir, self.profile)
-if not os.path.exists(tmp_dir):
+if not tmp_dir.exists():
 try:
-os.makedirs(tmp_dir)
+tmp_dir.mkdir(parents=True)
 except OSError as e:
 self.disp(
-"Can't create {path} directory: {reason}".format(
+f"Can't create {tmp_dir} directory: {e}",
-path=tmp_dir, reason=e
-),
 error=True,
 )
 self.host.quit(1)
 try:
 fd, path = tempfile.mkstemp(
-suffix=suff.encode("utf-8"),
+suffix=suff,
 prefix=time.strftime(cat_dir_str + "_%Y-%m-%d_%H:%M:%S_"),
 dir=tmp_dir,
 text=True,
 )
-return os.fdopen(fd, "w+b"), path
+return os.fdopen(fd, "w+b"), Path(path)
 except OSError as e:
 self.disp(
-"Can't create temporary file: {reason}".format(reason=e), error=True
+f"Can't create temporary file: {e}", error=True
 )
 self.host.quit(1)
 def getCurrentFile(self, profile):
 """Get most recently edited file
 @param profile(unicode): profile linked to the draft
-@return(str): full path of current file
+@return(Path): full path of current file
 """
 # we guess the item currently edited by choosing
 # the most recent file corresponding to temp file pattern
 # in tmp_dir, excluding metadata files
-cat_dir_str = self.cat_dir_str
+tmp_dir = getTmpDir(self.sat_conf, self.cat_dir, profile)
-tmp_dir = getTmpDir(self.sat_conf, self.cat_dir_str, profile.encode("utf-8"))
 available = [
-path
+p
-for path in glob.glob(os.path.join(tmp_dir, cat_dir_str + "_*"))
+for p in tmp_dir.glob(f'{self.cat_dir}_*')
-if not path.endswith(METADATA_SUFF)
+if not p.match(f"*{METADATA_SUFF}")
 ]
 if not available:
 self.disp(
-"Could not find any content draft in {path}".format(path=tmp_dir),
+f"Could not find any content draft in {tmp_dir}",
 error=True,
 )
 self.host.quit(1)
-return max(available, key=lambda path: os.stat(path).st_mtime)
+return max(available, key=lambda p: p.stat().st_mtime)
-def getItemData(self, service, node, item):
+async def getItemData(self, service, node, item):
 """return formatted content, metadata (or not if use_metadata is false), and item id"""
 raise NotImplementedError
 def getTmpSuff(self):
 """return suffix used for content file"""
 return "xml"
-def getItemPath(self):
+async def getItemPath(self):
-"""retrieve item path (i.e. service and node) from item argument
+"""Retrieve item path (i.e. service and node) from item argument
 This method is obviously only useful for edition of PubSub based features
 """
 service = self.args.service
 node = self.args.node
 if self.args.current:
 # user wants to continue current draft
 content_file_path = self.getCurrentFile(self.profile)
 self.disp("Continuing edition of current draft", 2)
-content_file_obj = open(content_file_path, "r+b")
+content_file_obj = content_file_path.open("r+b")
 # we seek at the end of file in case of an item already exist
 # this will write content of the existing item at the end of the draft.
 # This way no data should be lost.
 content_file_obj.seek(0, os.SEEK_END)
 elif self.args.draft_path:
 # there is an existing draft that we use
-content_file_path = os.path.expanduser(self.args.draft_path)
+content_file_path = self.args.draft_path.expanduser()
-content_file_obj = open(content_file_path, "r+b")
+content_file_obj = content_file_path.open("r+b")
 # we seek at the end for the same reason as above
 content_file_obj.seek(0, os.SEEK_END)
 else:
 # we need a temporary file
 content_file_obj, content_file_path = self.getTmpFile()
 if item or last_item:
 self.disp("Editing requested published item", 2)
 try:
 if self.use_metadata:
-content, metadata, item = self.getItemData(service, node, item)
+content, metadata, item = await self.getItemData(service, node, item)
 else:
-content, item = self.getItemData(service, node, item)
+content, item = await self.getItemData(service, node, item)
 except Exception as e:
 # FIXME: ugly but we have not good may to check errors in bridge
 if "item-not-found" in str(e):
 #  item doesn't exist, we create a new one with requested id
 metadata = None
 if last_item:
 self.disp(_("no item found at all, we create a new one"), 2)
 else:
 self.disp(
 _(
-'item "{item_id}" not found, we create a new item with this id'
+f'item "{item}" not found, we create a new item with'
-).format(item_id=item),
+f'this id'
+),
 2,
 )
 content_file_obj.seek(0)
 else:
-self.disp("Error while retrieving item: {}".format(e))
+self.disp(f"Error while retrieving item: {e}")
 self.host.quit(C.EXIT_ERROR)
 else:
 # item exists, we write content
 if content_file_obj.tell() != 0:
 # we already have a draft,
 # we copy item content after it and add an indicator
 content_file_obj.write("\n*****\n")
 content_file_obj.write(content.encode("utf-8"))
 content_file_obj.seek(0)
 self.disp(
-_('item "{item_id}" found, we edit it').format(item_id=item), 2
+_(f'item "{item}" found, we edit it'), 2
 )
 else:
 self.disp("Editing a new item", 2)
 if self.use_metadata:
 metadata = None
 kwargs_ = {"col_sep": " ", "head_line_sep": " ", "show_borders": False}
 kwargs_.update(kwargs)
 return self.display(**kwargs_)
-class URIFinder(object):
+async def fill_well_known_uri(command, path, key, meta_map=None):
-"""Helper class to find URIs in well-known locations"""
+"""Look for URIs in well-known location and fill appropriate args if suitable
-def __init__(self, command, path, key, callback, meta_map=None):
+@param command(CommandBase): command instance
-"""
+args of this instance will be updated with found values
-@param command(CommandBase): command instance
+@param path(unicode): absolute path to use as a starting point to look for URIs
-args of this instance will be updated with found values
+@param key(unicode): key to look for
-@param path(unicode): absolute path to use as a starting point to look for URIs
+@param meta_map(dict, None): if not None, map metadata to arg name
-@param key(unicode): key to look for
+key is metadata used attribute name
-@param callback(callable): method to call once URIs are found (or not)
+value is name to actually use, or None to ignore
-@param meta_map(dict, None): if not None, map metadata to arg name
+use empty dict to only retrieve URI
-key is metadata used attribute name
+possible keys are currently:
-value is name to actually use, or None to ignore
+- labels
-use empty dict to only retrieve URI
+"""
-possible keys are currently:
+args = command.args
-- labels
+if args.service or args.node:
-"""
+# we only look for URIs if a service and a node are not already specified
-if not command.args.service and not command.args.node:
+return
-self.host = command.host
-self.args = command.args
+host = command.host
-self.key = key
-self.callback = callback
+try:
-self.meta_map = meta_map
+uris_data = await host.bridge.URIFind(path, [key])
-self.host.bridge.URIFind(
+except Exception as e:
-path,
+host.disp(f"can't find {key} URI: {e}", error=True)
-[key],
+host.quit(C.EXIT_BRIDGE_ERRBACK)
-callback=self.URIFindCb,
-errback=partial(
+try:
-command.errback,
+uri_data = uris_data[key]
-msg=_("can't find " + key + " URI: {}"),
+except KeyError:
-exit_code=C.EXIT_BRIDGE_ERRBACK,
+host.disp(
-),
+_(f"No {key} URI specified for this project, please specify service and "
-)
+f"node"),
-else:
+error=True,
-callback()
+)
+host.quit(C.EXIT_NOT_FOUND)
-def setMetadataList(self, uri_data, key):
-"""Helper method to set list of values from metadata
+uri = uri_data["uri"]
-@param uri_data(dict): data of the found URI
+# set extra metadata if they are specified
-@param key(unicode): key of the value to retrieve
+for data_key in ['labels']:
-"""
+new_values_json = uri_data.get(data_key)
-new_values_json = uri_data.get(key)
 if uri_data is not None:
-if self.meta_map is None:
+if meta_map is None:
-dest = key
+dest = data_key
 else:
-dest = self.meta_map.get(key)
+dest = meta_map.get(data_key)
 if dest is None:
-return
+continue
 try:
-values = getattr(self.args, key)
+values = getattr(args, data_key)
 except AttributeError:
-raise exceptions.InternalError(
+raise exceptions.InternalError(f'there is no {data_key!r} arguments')
-'there is no "{key}" arguments'.format(key=key)
-)
 else:
 if values is None:
 values = []
 values.extend(json.loads(new_values_json))
-setattr(self.args, dest, values)
+setattr(args, dest, values)
-def URIFindCb(self, uris_data):
+parsed_uri = xmpp_uri.parseXMPPUri(uri)
 try:
-uri_data = uris_data[self.key]
+args.service = parsed_uri["path"]
-except KeyError:
+args.node = parsed_uri["node"]
-self.host.disp(
+except KeyError:
-_(
+host.disp(_(f"Invalid URI found: {uri}"), error=True)
-"No {key} URI specified for this project, please specify service and node"
+host.quit(C.EXIT_DATA_ERROR)
-).format(key=self.key),
-error=True,
-)
-self.host.quit(C.EXIT_NOT_FOUND)
-else:
-uri = uri_data["uri"]
-self.setMetadataList(uri_data, "labels")
-parsed_uri = xmpp_uri.parseXMPPUri(uri)
-try:
-self.args.service = parsed_uri["path"]
-self.args.node = parsed_uri["node"]
-except KeyError:
-self.host.disp(_("Invalid URI found: {uri}").format(uri=uri), error=True)
-self.host.quit(C.EXIT_DATA_ERROR)
-self.callback()

Mercurial > libervia-backend

comparison sat_frontends/jp/common.py @ 3040:fee60f17ebac