comparison sat/plugins/plugin_misc_text_syntaxes.py @ 4037:524856bd7b19

massive refactoring to switch from camelCase to snake_case: historically, Libervia (SàT before) was using camelCase as allowed by PEP8 when using a pre-PEP8 code, to use the same coding style as in Twisted. However, snake_case is more readable and it's better to follow PEP8 best practices, so it has been decided to move on full snake_case. Because Libervia has a huge codebase, this ended with a ugly mix of camelCase and snake_case. To fix that, this patch does a big refactoring by renaming every function and method (including bridge) that are not coming from Twisted or Wokkel, to use fully snake_case. This is a massive change, and may result in some bugs.
author Goffi <goffi@goffi.org>
date Sat, 08 Apr 2023 13:54:42 +0200
parents 33d75cd3c371
children 00dbc3370d35
comparison
equal deleted inserted replaced
4036:c4464d7ae97b 4037:524856bd7b19
194 "name": NAME, 194 "name": NAME,
195 "label": _(NAME), 195 "label": _(NAME),
196 "syntaxes": self.syntaxes, 196 "syntaxes": self.syntaxes,
197 } 197 }
198 198
199 self.addSyntax( 199 self.add_syntax(
200 self.SYNTAX_XHTML, 200 self.SYNTAX_XHTML,
201 lambda xhtml: defer.succeed(xhtml), 201 lambda xhtml: defer.succeed(xhtml),
202 lambda xhtml: defer.succeed(xhtml), 202 lambda xhtml: defer.succeed(xhtml),
203 TextSyntaxes.OPT_NO_THREAD, 203 TextSyntaxes.OPT_NO_THREAD,
204 ) 204 )
205 # TODO: text => XHTML should add <a/> to url like in frontends 205 # TODO: text => XHTML should add <a/> to url like in frontends
206 # it's probably best to move sat_frontends.tools.strings to sat.tools.common or similar 206 # it's probably best to move sat_frontends.tools.strings to sat.tools.common or similar
207 self.addSyntax( 207 self.add_syntax(
208 self.SYNTAX_TEXT, 208 self.SYNTAX_TEXT,
209 lambda text: escape(text), 209 lambda text: escape(text),
210 lambda xhtml: self._removeMarkups(xhtml), 210 lambda xhtml: self._remove_markups(xhtml),
211 [TextSyntaxes.OPT_HIDDEN], 211 [TextSyntaxes.OPT_HIDDEN],
212 ) 212 )
213 try: 213 try:
214 import markdown, html2text 214 import markdown, html2text
215 from markdown.extensions import Extension 215 from markdown.extensions import Extension
216 216
217 # XXX: we disable raw HTML parsing by default, to avoid parsing error 217 # XXX: we disable raw HTML parsing by default, to avoid parsing error
218 # when the user is not aware of markdown and HTML 218 # when the user is not aware of markdown and HTML
219 class EscapeHTML(Extension): 219 class EscapeHTML(Extension):
220 def extendMarkdown(self, md): 220 def extend_markdown(self, md):
221 md.preprocessors.deregister('html_block') 221 md.preprocessors.deregister('html_block')
222 md.inlinePatterns.deregister('html') 222 md.inlinePatterns.deregister('html')
223 223
224 def _html2text(html, baseurl=""): 224 def _html2text(html, baseurl=""):
225 h = html2text.HTML2Text(baseurl=baseurl) 225 h = html2text.HTML2Text(baseurl=baseurl)
226 h.body_width = 0 # do not truncate the lines, it breaks the long URLs 226 h.body_width = 0 # do not truncate the lines, it breaks the long URLs
227 return h.handle(html) 227 return h.handle(html)
228 228
229 self.addSyntax( 229 self.add_syntax(
230 self.SYNTAX_MARKDOWN, 230 self.SYNTAX_MARKDOWN,
231 partial(markdown.markdown, 231 partial(markdown.markdown,
232 extensions=[ 232 extensions=[
233 EscapeHTML(), 233 EscapeHTML(),
234 'nl2br', 234 'nl2br',
249 log.warning("markdown or html2text not found, can't use Markdown syntax") 249 log.warning("markdown or html2text not found, can't use Markdown syntax")
250 log.info( 250 log.info(
251 "You can download/install them from https://pythonhosted.org/Markdown/ " 251 "You can download/install them from https://pythonhosted.org/Markdown/ "
252 "and https://github.com/Alir3z4/html2text/" 252 "and https://github.com/Alir3z4/html2text/"
253 ) 253 )
254 host.bridge.addMethod( 254 host.bridge.add_method(
255 "syntaxConvert", 255 "syntax_convert",
256 ".plugin", 256 ".plugin",
257 in_sign="sssbs", 257 in_sign="sssbs",
258 out_sign="s", 258 out_sign="s",
259 async_=True, 259 async_=True,
260 method=self.convert, 260 method=self.convert,
261 ) 261 )
262 host.bridge.addMethod( 262 host.bridge.add_method(
263 "syntaxGet", ".plugin", in_sign="s", out_sign="s", method=self.getSyntax 263 "syntax_get", ".plugin", in_sign="s", out_sign="s", method=self.get_syntax
264 ) 264 )
265 if xml_tools.cleanXHTML is None: 265 if xml_tools.clean_xhtml is None:
266 log.debug("Installing cleaning method") 266 log.debug("Installing cleaning method")
267 xml_tools.cleanXHTML = self.cleanXHTML 267 xml_tools.clean_xhtml = self.clean_xhtml
268 268
269 def _updateParamOptions(self): 269 def _update_param_options(self):
270 data_synt = self.syntaxes 270 data_synt = self.syntaxes
271 default_synt = TextSyntaxes.default_syntax 271 default_synt = TextSyntaxes.default_syntax
272 syntaxes = [] 272 syntaxes = []
273 273
274 for syntax in list(data_synt.keys()): 274 for syntax in list(data_synt.keys()):
282 for syntax in syntaxes: 282 for syntax in syntaxes:
283 selected = 'selected="true"' if syntax == default_synt else "" 283 selected = 'selected="true"' if syntax == default_synt else ""
284 options.append('<option value="%s" %s/>' % (syntax, selected)) 284 options.append('<option value="%s" %s/>' % (syntax, selected))
285 285
286 self.params_data["options"] = "\n".join(options) 286 self.params_data["options"] = "\n".join(options)
287 self.host.memory.updateParams(self.params % self.params_data) 287 self.host.memory.update_params(self.params % self.params_data)
288 288
289 def getCurrentSyntax(self, profile): 289 def get_current_syntax(self, profile):
290 """ Return the selected syntax for the given profile 290 """ Return the selected syntax for the given profile
291 291
292 @param profile: %(doc_profile)s 292 @param profile: %(doc_profile)s
293 @return: profile selected syntax 293 @return: profile selected syntax
294 """ 294 """
295 return self.host.memory.getParamA(NAME, CATEGORY, profile_key=profile) 295 return self.host.memory.param_get_a(NAME, CATEGORY, profile_key=profile)
296 296
297 def _logError(self, failure, action="converting syntax"): 297 def _log_error(self, failure, action="converting syntax"):
298 log.error( 298 log.error(
299 "Error while {action}: {failure}".format(action=action, failure=failure) 299 "Error while {action}: {failure}".format(action=action, failure=failure)
300 ) 300 )
301 return failure 301 return failure
302 302
303 def cleanStyle(self, styles_raw: str) -> str: 303 def clean_style(self, styles_raw: str) -> str:
304 """"Clean unsafe CSS styles 304 """"Clean unsafe CSS styles
305 305
306 Remove styles not in the whitelist, or where the value doesn't match the regex 306 Remove styles not in the whitelist, or where the value doesn't match the regex
307 @param styles_raw: CSS styles 307 @param styles_raw: CSS styles
308 @return: cleaned styles 308 @return: cleaned styles
325 cleaned_styles.append((key, value)) 325 cleaned_styles.append((key, value))
326 return "; ".join( 326 return "; ".join(
327 ["%s: %s" % (key_, value_) for key_, value_ in cleaned_styles] 327 ["%s: %s" % (key_, value_) for key_, value_ in cleaned_styles]
328 ) 328 )
329 329
330 def cleanClasses(self, classes_raw: str) -> str: 330 def clean_classes(self, classes_raw: str) -> str:
331 """Remove any non whitelisted class 331 """Remove any non whitelisted class
332 332
333 @param classes_raw: classes set on an element 333 @param classes_raw: classes set on an element
334 @return: remaining classes (can be empty string) 334 @return: remaining classes (can be empty string)
335 """ 335 """
336 return " ".join(SAFE_CLASSES.intersection(classes_raw.split())) 336 return " ".join(SAFE_CLASSES.intersection(classes_raw.split()))
337 337
338 def cleanXHTML(self, xhtml): 338 def clean_xhtml(self, xhtml):
339 """Clean XHTML text by removing potentially dangerous/malicious parts 339 """Clean XHTML text by removing potentially dangerous/malicious parts
340 340
341 @param xhtml(unicode, lxml.etree._Element): raw HTML/XHTML text to clean 341 @param xhtml(unicode, lxml.etree._Element): raw HTML/XHTML text to clean
342 @return (unicode): cleaned XHTML 342 @return (unicode): cleaned XHTML
343 """ 343 """
358 cleaner = clean.Cleaner( 358 cleaner = clean.Cleaner(
359 style=False, add_nofollow=False, safe_attrs=SAFE_ATTRS 359 style=False, add_nofollow=False, safe_attrs=SAFE_ATTRS
360 ) 360 )
361 xhtml_elt = cleaner.clean_html(xhtml_elt) 361 xhtml_elt = cleaner.clean_html(xhtml_elt)
362 for elt in xhtml_elt.xpath("//*[@style]"): 362 for elt in xhtml_elt.xpath("//*[@style]"):
363 elt.set("style", self.cleanStyle(elt.get("style"))) 363 elt.set("style", self.clean_style(elt.get("style")))
364 for elt in xhtml_elt.xpath("//*[@class]"): 364 for elt in xhtml_elt.xpath("//*[@class]"):
365 elt.set("class", self.cleanClasses(elt.get("class"))) 365 elt.set("class", self.clean_classes(elt.get("class")))
366 # we remove self-closing elements for non-void elements 366 # we remove self-closing elements for non-void elements
367 for element in xhtml_elt.iter(tag=etree.Element): 367 for element in xhtml_elt.iter(tag=etree.Element):
368 if not element.text: 368 if not element.text:
369 if element.tag in VOID_ELEMENTS: 369 if element.tag in VOID_ELEMENTS:
370 element.text = None 370 element.text = None
387 # FIXME: convert should be abled to handle domish.Element directly 387 # FIXME: convert should be abled to handle domish.Element directly
388 # when dealing with XHTML 388 # when dealing with XHTML
389 # TODO: a way for parser to return parsing errors/warnings 389 # TODO: a way for parser to return parsing errors/warnings
390 390
391 if syntax_from == _SYNTAX_CURRENT: 391 if syntax_from == _SYNTAX_CURRENT:
392 syntax_from = self.getCurrentSyntax(profile) 392 syntax_from = self.get_current_syntax(profile)
393 else: 393 else:
394 syntax_from = syntax_from.lower().strip() 394 syntax_from = syntax_from.lower().strip()
395 if syntax_to == _SYNTAX_CURRENT: 395 if syntax_to == _SYNTAX_CURRENT:
396 syntax_to = self.getCurrentSyntax(profile) 396 syntax_to = self.get_current_syntax(profile)
397 else: 397 else:
398 syntax_to = syntax_to.lower().strip() 398 syntax_to = syntax_to.lower().strip()
399 syntaxes = self.syntaxes 399 syntaxes = self.syntaxes
400 if syntax_from not in syntaxes: 400 if syntax_from not in syntaxes:
401 raise exceptions.NotFound(syntax_from) 401 raise exceptions.NotFound(syntax_from)
409 d = deferToThread(syntaxes[syntax_from]["to"], text) 409 d = deferToThread(syntaxes[syntax_from]["to"], text)
410 410
411 # TODO: keep only body element and change it to a div here ? 411 # TODO: keep only body element and change it to a div here ?
412 412
413 if safe: 413 if safe:
414 d.addCallback(self.cleanXHTML) 414 d.addCallback(self.clean_xhtml)
415 415
416 if TextSyntaxes.OPT_NO_THREAD in syntaxes[syntax_to]["flags"]: 416 if TextSyntaxes.OPT_NO_THREAD in syntaxes[syntax_to]["flags"]:
417 d.addCallback(syntaxes[syntax_to]["from"]) 417 d.addCallback(syntaxes[syntax_to]["from"])
418 else: 418 else:
419 d.addCallback(lambda xhtml: deferToThread(syntaxes[syntax_to]["from"], xhtml)) 419 d.addCallback(lambda xhtml: deferToThread(syntaxes[syntax_to]["from"], xhtml))
420 420
421 # converters can add new lines that disturb the microblog change detection 421 # converters can add new lines that disturb the microblog change detection
422 d.addCallback(lambda text: text.rstrip()) 422 d.addCallback(lambda text: text.rstrip())
423 return d 423 return d
424 424
425 def addSyntax(self, name, to_xhtml_cb, from_xhtml_cb, flags=None): 425 def add_syntax(self, name, to_xhtml_cb, from_xhtml_cb, flags=None):
426 """Add a new syntax to the manager 426 """Add a new syntax to the manager
427 427
428 @param name: unique name of the syntax 428 @param name: unique name of the syntax
429 @param to_xhtml_cb: callback to convert from syntax to XHTML 429 @param to_xhtml_cb: callback to convert from syntax to XHTML
430 @param from_xhtml_cb: callback to convert from XHTML to syntax 430 @param from_xhtml_cb: callback to convert from XHTML to syntax
454 "flags": flags, 454 "flags": flags,
455 } 455 }
456 if TextSyntaxes.OPT_DEFAULT in flags: 456 if TextSyntaxes.OPT_DEFAULT in flags:
457 TextSyntaxes.default_syntax = key 457 TextSyntaxes.default_syntax = key
458 458
459 self._updateParamOptions() 459 self._update_param_options()
460 460
461 def getSyntax(self, name): 461 def get_syntax(self, name):
462 """get syntax key corresponding to a name 462 """get syntax key corresponding to a name
463 463
464 @raise exceptions.NotFound: syntax doesn't exist 464 @raise exceptions.NotFound: syntax doesn't exist
465 """ 465 """
466 key = name.lower().strip() 466 key = name.lower().strip()
467 if key in self.syntaxes: 467 if key in self.syntaxes:
468 return key 468 return key
469 raise exceptions.NotFound 469 raise exceptions.NotFound
470 470
471 def _removeMarkups(self, xhtml): 471 def _remove_markups(self, xhtml):
472 """Remove XHTML markups from the given string. 472 """Remove XHTML markups from the given string.
473 473
474 @param xhtml: the XHTML string to be cleaned 474 @param xhtml: the XHTML string to be cleaned
475 @return: the cleaned string 475 @return: the cleaned string
476 """ 476 """