Mercurial > libervia-backend
annotate src/test/test_plugin_misc_text_syntaxes.py @ 2109:85f3e12e984d
core (memory/cache): file caching handling, first draft:
instead of having file caching handled individually by plugins, a generic module has been added in memory.
- Cache can be global or associated to a profile. In the later case, client.cache can be used.
- Cache are managed with unique ids (which can be any unique unicode, hash uuid, or something else).
- To know if a file is in cache, getFilePath is used: if the file is in cache, its absolute path is returned, else None is returned.
- To cache a file, cacheData is used with at list the source of cache (most of time plugin import name), and unique id. The method return file opened in binary writing mode (so cacheData can - and should - be used with "with" statement).
- 2 files will be created: a metadata file (named after the unique id), and the actual file.
- each file has a end of life time, after it, the cache is invalidated and the file must be requested again.
author | Goffi <goffi@goffi.org> |
---|---|
date | Thu, 05 Jan 2017 20:23:38 +0100 |
parents | 2daf7b4c6756 |
children | 8b37a62336c3 |
rev | line source |
---|---|
1934
2daf7b4c6756
use of /usr/bin/env instead of /usr/bin/python in shebang
Goffi <goffi@goffi.org>
parents:
1910
diff
changeset
|
1 #!/usr/bin/env python2 |
694
4284b6ad8aa3
tests: plugin text syntaxes sanitisation tests
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
2 # -*- coding: utf-8 -*- |
4284b6ad8aa3
tests: plugin text syntaxes sanitisation tests
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
3 |
4284b6ad8aa3
tests: plugin text syntaxes sanitisation tests
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
4 # SAT: a jabber client |
1766 | 5 # Copyright (C) 2009-2016 Jérôme Poisson (goffi@goffi.org) |
694
4284b6ad8aa3
tests: plugin text syntaxes sanitisation tests
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
6 |
4284b6ad8aa3
tests: plugin text syntaxes sanitisation tests
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
7 # This program is free software: you can redistribute it and/or modify |
4284b6ad8aa3
tests: plugin text syntaxes sanitisation tests
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
8 # it under the terms of the GNU Affero General Public License as published by |
4284b6ad8aa3
tests: plugin text syntaxes sanitisation tests
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
9 # the Free Software Foundation, either version 3 of the License, or |
4284b6ad8aa3
tests: plugin text syntaxes sanitisation tests
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
10 # (at your option) any later version. |
4284b6ad8aa3
tests: plugin text syntaxes sanitisation tests
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
11 |
4284b6ad8aa3
tests: plugin text syntaxes sanitisation tests
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
12 # This program is distributed in the hope that it will be useful, |
4284b6ad8aa3
tests: plugin text syntaxes sanitisation tests
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of |
4284b6ad8aa3
tests: plugin text syntaxes sanitisation tests
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
4284b6ad8aa3
tests: plugin text syntaxes sanitisation tests
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
15 # GNU Affero General Public License for more details. |
4284b6ad8aa3
tests: plugin text syntaxes sanitisation tests
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
16 |
4284b6ad8aa3
tests: plugin text syntaxes sanitisation tests
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
17 # You should have received a copy of the GNU Affero General Public License |
4284b6ad8aa3
tests: plugin text syntaxes sanitisation tests
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
18 # along with this program. If not, see <http://www.gnu.org/licenses/>. |
4284b6ad8aa3
tests: plugin text syntaxes sanitisation tests
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
19 |
4284b6ad8aa3
tests: plugin text syntaxes sanitisation tests
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
20 """ Plugin text syntaxes tests """ |
4284b6ad8aa3
tests: plugin text syntaxes sanitisation tests
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
21 |
4284b6ad8aa3
tests: plugin text syntaxes sanitisation tests
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
22 from sat.test import helpers |
4284b6ad8aa3
tests: plugin text syntaxes sanitisation tests
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
23 from sat.plugins import plugin_misc_text_syntaxes |
856
7ea7053dda88
test: skip the test for markdown syntax if the module is not installed
souliane <souliane@mailoo.org>
parents:
841
diff
changeset
|
24 from twisted.trial.unittest import SkipTest |
832
c4b22aedb7d7
plugin groupblog, XEP-0071, XEP-0277, text_syntaxes: manage raw/rich/xhtml data for content/title:
souliane <souliane@mailoo.org>
parents:
811
diff
changeset
|
25 import re |
694
4284b6ad8aa3
tests: plugin text syntaxes sanitisation tests
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
26 |
4284b6ad8aa3
tests: plugin text syntaxes sanitisation tests
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
27 |
4284b6ad8aa3
tests: plugin text syntaxes sanitisation tests
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
28 class SanitisationTest(helpers.SatTestCase): |
4284b6ad8aa3
tests: plugin text syntaxes sanitisation tests
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
29 |
832
c4b22aedb7d7
plugin groupblog, XEP-0071, XEP-0277, text_syntaxes: manage raw/rich/xhtml data for content/title:
souliane <souliane@mailoo.org>
parents:
811
diff
changeset
|
30 EVIL_HTML1 = """ |
694
4284b6ad8aa3
tests: plugin text syntaxes sanitisation tests
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
31 <html> |
4284b6ad8aa3
tests: plugin text syntaxes sanitisation tests
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
32 <head> |
4284b6ad8aa3
tests: plugin text syntaxes sanitisation tests
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
33 <script type="text/javascript" src="evil-site"></script> |
4284b6ad8aa3
tests: plugin text syntaxes sanitisation tests
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
34 <link rel="alternate" type="text/rss" src="evil-rss"> |
4284b6ad8aa3
tests: plugin text syntaxes sanitisation tests
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
35 <style> |
4284b6ad8aa3
tests: plugin text syntaxes sanitisation tests
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
36 body {background-image: url(javascript:do_evil)}; |
4284b6ad8aa3
tests: plugin text syntaxes sanitisation tests
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
37 div {color: expression(evil)}; |
4284b6ad8aa3
tests: plugin text syntaxes sanitisation tests
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
38 </style> |
4284b6ad8aa3
tests: plugin text syntaxes sanitisation tests
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
39 </head> |
4284b6ad8aa3
tests: plugin text syntaxes sanitisation tests
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
40 <body onload="evil_function()"> |
4284b6ad8aa3
tests: plugin text syntaxes sanitisation tests
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
41 <!-- I am interpreted for EVIL! --> |
4284b6ad8aa3
tests: plugin text syntaxes sanitisation tests
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
42 <a href="javascript:evil_function()">a link</a> |
4284b6ad8aa3
tests: plugin text syntaxes sanitisation tests
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
43 <a href="#" onclick="evil_function()">another link</a> |
4284b6ad8aa3
tests: plugin text syntaxes sanitisation tests
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
44 <p onclick="evil_function()">a paragraph</p> |
4284b6ad8aa3
tests: plugin text syntaxes sanitisation tests
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
45 <div style="display: none">secret EVIL!</div> |
4284b6ad8aa3
tests: plugin text syntaxes sanitisation tests
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
46 <object> of EVIL! </object> |
4284b6ad8aa3
tests: plugin text syntaxes sanitisation tests
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
47 <iframe src="evil-site"></iframe> |
4284b6ad8aa3
tests: plugin text syntaxes sanitisation tests
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
48 <form action="evil-site"> |
4284b6ad8aa3
tests: plugin text syntaxes sanitisation tests
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
49 Password: <input type="password" name="password"> |
4284b6ad8aa3
tests: plugin text syntaxes sanitisation tests
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
50 </form> |
4284b6ad8aa3
tests: plugin text syntaxes sanitisation tests
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
51 <blink>annoying EVIL!</blink> |
4284b6ad8aa3
tests: plugin text syntaxes sanitisation tests
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
52 <a href="evil-site">spam spam SPAM!</a> |
4284b6ad8aa3
tests: plugin text syntaxes sanitisation tests
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
53 <image src="evil!"> |
4284b6ad8aa3
tests: plugin text syntaxes sanitisation tests
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
54 </body> |
780
9810f22ba733
test: store the constants in constants.py + better PEP8 compliance
souliane <souliane@mailoo.org>
parents:
694
diff
changeset
|
55 </html>""" # example from lxml: /usr/share/doc/python-lxml-doc/html/lxmlhtml.html#cleaning-up-html |
694
4284b6ad8aa3
tests: plugin text syntaxes sanitisation tests
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
56 |
832
c4b22aedb7d7
plugin groupblog, XEP-0071, XEP-0277, text_syntaxes: manage raw/rich/xhtml data for content/title:
souliane <souliane@mailoo.org>
parents:
811
diff
changeset
|
57 EVIL_HTML2 = """<p style='display: None; test: blah; background: url(: alert()); color: blue;'>test <strong>retest</strong><br><span style="background-color: (alert('bouh')); titi; color: #cf2828; font-size: 3px; direction: !important; color: red; color: red !important; font-size: 100px !important; font-size: 100px ! important; font-size: 100%; font-size: 100ox; font-size: 100px; font-size: 100;;;; font-size: 100 %; color: 100 px 1.7em; color: rgba(0, 0, 0, 0.1); color: rgb(35,79,255); background-color: no-repeat; background-color: :alert(1); color: (alert('XSS')); color: (window.location='http://example.org/'); color: url(:window.location='http://example.org/'); "> toto </span></p>""" |
c4b22aedb7d7
plugin groupblog, XEP-0071, XEP-0277, text_syntaxes: manage raw/rich/xhtml data for content/title:
souliane <souliane@mailoo.org>
parents:
811
diff
changeset
|
58 |
c4b22aedb7d7
plugin groupblog, XEP-0071, XEP-0277, text_syntaxes: manage raw/rich/xhtml data for content/title:
souliane <souliane@mailoo.org>
parents:
811
diff
changeset
|
59 def setUp(self): |
c4b22aedb7d7
plugin groupblog, XEP-0071, XEP-0277, text_syntaxes: manage raw/rich/xhtml data for content/title:
souliane <souliane@mailoo.org>
parents:
811
diff
changeset
|
60 self.host = helpers.FakeSAT() |
1910 | 61 reload(plugin_misc_text_syntaxes) # reload the plugin to avoid conflict error |
832
c4b22aedb7d7
plugin groupblog, XEP-0071, XEP-0277, text_syntaxes: manage raw/rich/xhtml data for content/title:
souliane <souliane@mailoo.org>
parents:
811
diff
changeset
|
62 self.text_syntaxes = plugin_misc_text_syntaxes.TextSyntaxes(self.host) |
c4b22aedb7d7
plugin groupblog, XEP-0071, XEP-0277, text_syntaxes: manage raw/rich/xhtml data for content/title:
souliane <souliane@mailoo.org>
parents:
811
diff
changeset
|
63 |
c4b22aedb7d7
plugin groupblog, XEP-0071, XEP-0277, text_syntaxes: manage raw/rich/xhtml data for content/title:
souliane <souliane@mailoo.org>
parents:
811
diff
changeset
|
64 def test_xhtml_sanitise(self): |
1412 | 65 expected = u"""<div> |
694
4284b6ad8aa3
tests: plugin text syntaxes sanitisation tests
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
66 <style>/* deleted */</style> |
4284b6ad8aa3
tests: plugin text syntaxes sanitisation tests
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
67 <body> |
4284b6ad8aa3
tests: plugin text syntaxes sanitisation tests
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
68 <a href="">a link</a> |
4284b6ad8aa3
tests: plugin text syntaxes sanitisation tests
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
69 <a href="#">another link</a> |
4284b6ad8aa3
tests: plugin text syntaxes sanitisation tests
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
70 <p>a paragraph</p> |
4284b6ad8aa3
tests: plugin text syntaxes sanitisation tests
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
71 <div style="">secret EVIL!</div> |
4284b6ad8aa3
tests: plugin text syntaxes sanitisation tests
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
72 of EVIL! |
4284b6ad8aa3
tests: plugin text syntaxes sanitisation tests
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
73 Password: |
4284b6ad8aa3
tests: plugin text syntaxes sanitisation tests
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
74 annoying EVIL! |
4284b6ad8aa3
tests: plugin text syntaxes sanitisation tests
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
75 <a href="evil-site">spam spam SPAM!</a> |
4284b6ad8aa3
tests: plugin text syntaxes sanitisation tests
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
76 <img src="evil!"> |
4284b6ad8aa3
tests: plugin text syntaxes sanitisation tests
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
77 </img></body> |
4284b6ad8aa3
tests: plugin text syntaxes sanitisation tests
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
78 </div>""" |
4284b6ad8aa3
tests: plugin text syntaxes sanitisation tests
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
79 |
1812
160b0d4c6520
plugin XEP-0071, XEP-0277: method clean_xhtml has been renamed to cleanXHTML
souliane <souliane@mailoo.org>
parents:
1809
diff
changeset
|
80 d = self.text_syntaxes.cleanXHTML(self.EVIL_HTML1) |
694
4284b6ad8aa3
tests: plugin text syntaxes sanitisation tests
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
81 d.addCallback(self.assertEqualXML, expected, ignore_blank=True) |
4284b6ad8aa3
tests: plugin text syntaxes sanitisation tests
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
82 return d |
4284b6ad8aa3
tests: plugin text syntaxes sanitisation tests
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
83 |
4284b6ad8aa3
tests: plugin text syntaxes sanitisation tests
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
84 def test_styles_sanitise(self): |
1412 | 85 expected = u"""<p style="color: blue">test <strong>retest</strong><br/><span style="color: #cf2828; font-size: 3px; color: red; color: red !important; font-size: 100px !important; font-size: 100%; font-size: 100px; font-size: 100; font-size: 100 %; color: rgba(0, 0, 0, 0.1); color: rgb(35,79,255); background-color: no-repeat"> toto </span></p>""" |
694
4284b6ad8aa3
tests: plugin text syntaxes sanitisation tests
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
86 |
1812
160b0d4c6520
plugin XEP-0071, XEP-0277: method clean_xhtml has been renamed to cleanXHTML
souliane <souliane@mailoo.org>
parents:
1809
diff
changeset
|
87 d = self.text_syntaxes.cleanXHTML(self.EVIL_HTML2) |
694
4284b6ad8aa3
tests: plugin text syntaxes sanitisation tests
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
88 d.addCallback(self.assertEqualXML, expected) |
4284b6ad8aa3
tests: plugin text syntaxes sanitisation tests
Goffi <goffi@goffi.org>
parents:
diff
changeset
|
89 return d |
832
c4b22aedb7d7
plugin groupblog, XEP-0071, XEP-0277, text_syntaxes: manage raw/rich/xhtml data for content/title:
souliane <souliane@mailoo.org>
parents:
811
diff
changeset
|
90 |
841
831f208b4ea3
plugin text_syntaxes: html2text was breaking the long URLs
souliane <souliane@mailoo.org>
parents:
832
diff
changeset
|
91 def test_html2text(self): |
831f208b4ea3
plugin text_syntaxes: html2text was breaking the long URLs
souliane <souliane@mailoo.org>
parents:
832
diff
changeset
|
92 """Check that html2text is not inserting \n in the middle of that link. |
831f208b4ea3
plugin text_syntaxes: html2text was breaking the long URLs
souliane <souliane@mailoo.org>
parents:
832
diff
changeset
|
93 By default lines are truncated after the 79th characters.""" |
831f208b4ea3
plugin text_syntaxes: html2text was breaking the long URLs
souliane <souliane@mailoo.org>
parents:
832
diff
changeset
|
94 source = "<img src=\"http://sat.goffi.org/static/images/screenshots/libervia/libervia_discussions.png\" alt=\"sat\"/>" |
831f208b4ea3
plugin text_syntaxes: html2text was breaking the long URLs
souliane <souliane@mailoo.org>
parents:
832
diff
changeset
|
95 expected = "![sat](http://sat.goffi.org/static/images/screenshots/libervia/libervia_discussions.png)" |
856
7ea7053dda88
test: skip the test for markdown syntax if the module is not installed
souliane <souliane@mailoo.org>
parents:
841
diff
changeset
|
96 try: |
7ea7053dda88
test: skip the test for markdown syntax if the module is not installed
souliane <souliane@mailoo.org>
parents:
841
diff
changeset
|
97 d = self.text_syntaxes.convert(source, self.text_syntaxes.SYNTAX_XHTML, self.text_syntaxes.SYNTAX_MARKDOWN) |
7ea7053dda88
test: skip the test for markdown syntax if the module is not installed
souliane <souliane@mailoo.org>
parents:
841
diff
changeset
|
98 except plugin_misc_text_syntaxes.UnknownSyntax: |
7ea7053dda88
test: skip the test for markdown syntax if the module is not installed
souliane <souliane@mailoo.org>
parents:
841
diff
changeset
|
99 raise SkipTest("Markdown syntax is not available.") |
841
831f208b4ea3
plugin text_syntaxes: html2text was breaking the long URLs
souliane <souliane@mailoo.org>
parents:
832
diff
changeset
|
100 d.addCallback(self.assertEqual, expected) |
831f208b4ea3
plugin text_syntaxes: html2text was breaking the long URLs
souliane <souliane@mailoo.org>
parents:
832
diff
changeset
|
101 return d |
831f208b4ea3
plugin text_syntaxes: html2text was breaking the long URLs
souliane <souliane@mailoo.org>
parents:
832
diff
changeset
|
102 |
832
c4b22aedb7d7
plugin groupblog, XEP-0071, XEP-0277, text_syntaxes: manage raw/rich/xhtml data for content/title:
souliane <souliane@mailoo.org>
parents:
811
diff
changeset
|
103 def test_removeXHTMLMarkups(self): |
1809
821c77574ad9
test, setup: requires lxml >= 3.1.0 (fixes html cleaning issue)
souliane <souliane@mailoo.org>
parents:
1766
diff
changeset
|
104 expected = u""" a link another link a paragraph secret EVIL! of EVIL! Password: annoying EVIL! spam spam SPAM! """ |
832
c4b22aedb7d7
plugin groupblog, XEP-0071, XEP-0277, text_syntaxes: manage raw/rich/xhtml data for content/title:
souliane <souliane@mailoo.org>
parents:
811
diff
changeset
|
105 result = self.text_syntaxes._removeMarkups(self.EVIL_HTML1) |
c4b22aedb7d7
plugin groupblog, XEP-0071, XEP-0277, text_syntaxes: manage raw/rich/xhtml data for content/title:
souliane <souliane@mailoo.org>
parents:
811
diff
changeset
|
106 self.assertEqual(re.sub(r"\s+", " ", result).rstrip(), expected.rstrip()) |
c4b22aedb7d7
plugin groupblog, XEP-0071, XEP-0277, text_syntaxes: manage raw/rich/xhtml data for content/title:
souliane <souliane@mailoo.org>
parents:
811
diff
changeset
|
107 |
1412 | 108 expected = u"""test retest toto""" |
832
c4b22aedb7d7
plugin groupblog, XEP-0071, XEP-0277, text_syntaxes: manage raw/rich/xhtml data for content/title:
souliane <souliane@mailoo.org>
parents:
811
diff
changeset
|
109 result = self.text_syntaxes._removeMarkups(self.EVIL_HTML2) |
c4b22aedb7d7
plugin groupblog, XEP-0071, XEP-0277, text_syntaxes: manage raw/rich/xhtml data for content/title:
souliane <souliane@mailoo.org>
parents:
811
diff
changeset
|
110 self.assertEqual(re.sub(r"\s+", " ", result).rstrip(), expected.rstrip()) |
c4b22aedb7d7
plugin groupblog, XEP-0071, XEP-0277, text_syntaxes: manage raw/rich/xhtml data for content/title:
souliane <souliane@mailoo.org>
parents:
811
diff
changeset
|
111 |