diff tests/e2e/run_e2e.py @ 3429:d4558f3cbf13

tests, docker(e2e): added e2e tests for Libervia: - moved jp tests to `e2e/jp` - new fixtures - adapted docker-compose - improved `run_e2e` with several flags + report on failure - doc to come
author Goffi <goffi@goffi.org>
date Fri, 27 Nov 2020 16:39:40 +0100
parents 814e118d9ef3
children be6d91572633
line wrap: on
line diff
--- a/tests/e2e/run_e2e.py	Fri Nov 27 16:32:40 2020 +0100
+++ b/tests/e2e/run_e2e.py	Fri Nov 27 16:39:40 2020 +0100
@@ -22,28 +22,85 @@
 from pathlib import Path
 import tempfile
 from textwrap import dedent
+from datetime import datetime
 import sh
+import io
+import re
+import sat_templates
+import libervia
 from sat.core import exceptions
+import yaml
+try:
+    from yaml import CLoader as Loader, CDumper as Dumper
+except ImportError:
+    from yaml import Loader, Dumper
 
-KEEP_OPT = "--keep"
+
+OPT_KEEP_CONTAINERS = "--keep-containers"
+OPT_KEEP_PROFILES = "--keep-profiles"
+OPT_KEEP_VNC = "--keep-vnc"
+OPT_KEEP_BROWSER = "--keep-browser"
+OPT_VISUAL = "--visual"
+OPT_DEV_MODE = "--dev-mode"
+
+dev_mode_inst = dedent("""\
+    Here is a short script to start working with a logged account:
+
+    from helium import *
+    start_firefox()
+    go_to("https://libervia.test:8443/login")
+    write("account1", "login")
+    write("test", "password")
+    click("log in")
+    """)
+report_buffer = io.StringIO()
 
 
 def live_out(data):
     sys.stdout.write(data)
     sys.stdout.flush()
+    report_buffer.write(data)
 
 
 def live_err(data):
     sys.stderr.write(data)
     sys.stderr.flush()
+    report_buffer.write(data)
+
+
+def get_opt(opt_name):
+    """Check is an option flag is set, and remove it for sys.argv
+
+    This allow to have simple flags without interfering with pytest options
+    """
+    if opt_name in sys.argv:
+        sys.argv.remove(opt_name)
+        return True
+    else:
+        return False
+
+
+def set_env(override, name, value="1"):
+    """Set environement variable"""
+    environment = override["services"]["sat"].setdefault("environment", {})
+    environment[name] = value
 
 
 def use_e2e_env():
-    if KEEP_OPT in sys.argv:
+    visual = get_opt(OPT_VISUAL)
+    keep_containers = get_opt(OPT_KEEP_CONTAINERS)
+    keep_profiles = get_opt(OPT_KEEP_PROFILES)
+    keep_vnc = get_opt(OPT_KEEP_VNC)
+    keep_browser = get_opt(OPT_KEEP_BROWSER)
+    if keep_browser:
         keep_containers = True
-        sys.argv.remove(KEEP_OPT)
-    else:
-        keep_containers = False
+        keep_vnc = True
+    if keep_vnc:
+        visual = True
+    dev_mode = get_opt(OPT_DEV_MODE)
+    if dev_mode:
+        keep_containers = keep_profiles = keep_vnc = visual = True
+
     for p in Path.cwd().parents:
         package_path = p / "sat"
         docker_path = p / "docker"
@@ -56,14 +113,32 @@
             "from the backend repository?"
         )
 
+    libervia_path = Path(libervia.__file__).parent.resolve()
+    libervia_root_path = libervia_path.parent
+    if (libervia_root_path / ".hg").is_dir():
+        libervia_source = libervia_root_path
+        libervia_target = "/src/libervia"
+    else:
+        libervia_source = libervia_path
+        libervia_target = "/src/libervia/libervia"
+
+    sat_templates_path = Path(sat_templates.__file__).parent.resolve()
+    sat_templates_root_path = sat_templates_path.parent
+    if (sat_templates_root_path / ".hg").is_dir():
+        sat_templates_source = sat_templates_root_path
+        sat_templates_target = "/src/sat_templates"
+    else:
+        sat_templates_source = sat_templates_path
+        sat_templates_target = "/src/sat_templates/sat_templates"
+
     compose_e2e_path = docker_path / "docker-compose_e2e.yml"
     if not compose_e2e_path.is_file():
         raise exceptions.NotFound('"docker-compose_e2e.yml" file can\'t be found')
 
-    with tempfile.TemporaryDirectory() as temp_dir:
+    with tempfile.TemporaryDirectory(prefix="sat_test_e2e_") as temp_dir:
         override_path = Path(temp_dir) / "test_override.yml"
-        with override_path.open("w") as f:
-            f.write(dedent(f"""\
+        override = yaml.load(
+            dedent(f"""\
                 version: "3.6"
                 services:
                   sat:
@@ -72,23 +147,89 @@
                         source: {sat_root_path}
                         target: /src/sat
                         read_only: true
-                """))
+                  libervia:
+                    volumes:
+                      - type: bind
+                        source: {sat_root_path}
+                        target: /src/sat
+                        read_only: true
+                      - type: bind
+                        source: {libervia_source}
+                        target: {libervia_target}
+                        read_only: true
+                      - type: bind
+                        source: {sat_templates_source}
+                        target: {sat_templates_target}
+                        read_only: true
+                """
+                   ),
+            Loader=Loader
+        )
+
+        if keep_profiles:
+            set_env(override, "SAT_TEST_E2E_KEEP_PROFILES")
+
+        if visual:
+            set_env(override, "SAT_TEST_E2E_LIBERVIA_NO_HEADLESS")
+
+        if keep_browser:
+            set_env(override, "SAT_TEST_E2E_LIBERVIA_KEEP_BROWSER")
+
+        with override_path.open("w") as f:
+            yaml.dump(override, f, Dumper=Dumper)
 
         docker_compose = sh.docker_compose.bake(
             "-f", compose_e2e_path, "-f", override_path)
         docker_compose.up("-d")
 
-        try:
-            docker_compose.exec(
+        p = docker_compose.exec(
             "-T", "--workdir", "/src/sat/tests", "sat",
             "pytest", "-o", "cache_dir=/tmp", *sys.argv[1:], color="yes",
-            _in=sys.stdin, _out=live_out, _out_bufsize=0, _err=live_err, _err_bufsize=0
+            _in=sys.stdin, _out=live_out, _out_bufsize=0, _err=live_err, _err_bufsize=0,
+            _bg=True
         )
+        if visual:
+            vnc_port = docker_compose.port("sat", "5900").split(':', 1)[1].strip()
+            p_vnc = sh.vncviewer(
+                f"localhost:{vnc_port}",
+                _bg=True,
+                # vncviewer exits with 1 when we send an SIGTERM to it, and it's printed
+                # before we can catch it (it's happening in a thread). Thus we set exit
+                # code 1 as OK to avoid the backtrace.
+                _ok_code=[0, 1]
+            )
+        else:
+            p_vnc = None
+
+        try:
+            p.wait()
         except sh.ErrorReturnCode as e:
+            sat_cont_id = docker_compose.ps("-q", "sat").strip()
+            report_dest = Path(f"reports_{datetime.now().isoformat()}/")
+            # we need to make `report_dest` explicitely local with "./", otherwise
+            # docker parse takes it as a container path due to the presence of ":"
+            # with `isoformat()`.
+            sh.docker.cp(f"{sat_cont_id}:/reports", f"./{report_dest}")
+            # we save 2 versions: one with ANSI escape codes
+            report_ansi = report_dest / "report.ansi"
+            with report_ansi.open('w') as f:
+                f.write(report_buffer.getvalue())
+            # and one without (cf. https://stackoverflow.com/a/14693789)
+            ansi_escape = re.compile(r'\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])')
+            report_log = report_dest / "report.log"
+            with report_log.open('w') as f:
+                f.write(ansi_escape.sub('', report_buffer.getvalue()))
+
+            print(f"report saved to {report_dest}")
             sys.exit(e.exit_code)
         finally:
+            if p_vnc is not None and p_vnc.is_alive() and not keep_vnc:
+                p_vnc.terminate()
             if not keep_containers:
                 docker_compose.down(volumes=True)
+            if dev_mode:
+                print(dev_mode_inst)
+
 
 if __name__ == "__main__":
     use_e2e_env()