Speedup tests

.github/workflows/auto_tests.yml

@@ -17,7 +17,7 @@ jobs:
       matrix:
         include:
           - suite: server
-            pytest_args: "-v --cov=mergin --cov-report=lcov mergin/tests"
+            pytest_args: "-v -n auto --cov=mergin --cov-report=lcov mergin/tests"
           - suite: migration
             pytest_args: "-v mergin/test_migrations"
 

development.md

@@ -142,3 +142,7 @@ cd server
 pipenv install --dev --deploy --verbose
 pipenv run pytest -v --cov=mergin mergin/tests
 ```
+
+Tests can run in parallel with [pytest-xdist](https://pytest-xdist.readthedocs.io), e.g. `pipenv run pytest -n 4 mergin/tests`.
+Each worker uses its own database (`<DB_DATABASE>_gw<N>`, created automatically) and its own temporary directory.
+To run several test sessions at the same time, give each a different `DB_DATABASE`, e.g. `DB_DATABASE=ce pipenv run pytest -n 4 mergin/tests`.

server/.test.env

@@ -26,4 +26,5 @@ SECURITY_PASSWORD_SALT='password'
 SECURITY_UNLOCK_SALT='unlock'
 DIAGNOSTIC_LOGS_DIR=/tmp/diagnostic_logs
 GEVENT_WORKER=0
-OTEL_ENABLED=0
+OTEL_ENABLED=0
+BCRYPT_LOG_ROUNDS=4

server/Pipfile

@@ -51,6 +51,7 @@ pytest-cov = "==5.0.0"
 pylint = "==3.2.6"
 responses = "==0.21.0"
 pytest-dotenv= "==0.5.2"
+pytest-xdist = "==3.8.0"
 # sqlite with loadable extensions for tests, pysqlite3-binary wheels are available only for linux
 pysqlite3-binary = {version = "==0.5.3", markers = "sys_platform == 'linux'"}
 pysqlite3 = {version = "==0.6.0", markers = "sys_platform != 'linux'"}

server/conftest.py

@@ -0,0 +1,31 @@
+# Copyright (C) Lutra Consulting Limited
+#
+# SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-MerginMaps-Commercial
+
+import os
+import tempfile
+
+
+def isolate_xdist_worker():
+    """Give each pytest-xdist worker its own database and temporary directories.
+
+    Must run before app configuration is imported as it is read from env variables,
+    this conftest is loaded right after test env file and before any tests modules.
+    """
+    worker = os.environ.get("PYTEST_XDIST_WORKER")
+    if not worker:
+        return
+
+    db_name = f"{os.environ.get('DB_DATABASE', 'postgres')}_{worker}"
+    os.environ["DB_DATABASE"] = db_name
+    tmp_dir = tempfile.gettempdir()
+    worker_tmp_dir = os.path.join(tmp_dir, db_name)
+    os.makedirs(worker_tmp_dir, exist_ok=True)
+    for key, value in list(os.environ.items()):
+        if value == tmp_dir or value.startswith(tmp_dir + os.sep):
+            os.environ[key] = worker_tmp_dir + value[len(tmp_dir) :]
+    os.environ["TMPDIR"] = worker_tmp_dir
+    tempfile.tempdir = None
+
+
+isolate_xdist_worker()

server/mergin/tests/fixtures.py

@@ -4,6 +4,7 @@
 
 import os
 import shutil
+import tempfile
 import sys
 import uuid
 from shutil import copy, move
@@ -17,16 +18,24 @@
 from ..stats.app import register
 from ..stats.models import MerginInfo
 from . import test_project, test_workspace_id, test_project_dir, TMP_DIR
-from .utils import login_as_admin, initialize, cleanup, file_info, ListSink
+from .utils import (
+    login_as_admin,
+    initialize,
+    cleanup,
+    file_info,
+    ListSink,
+    clean_db,
+    create_db,
+)
 from ..sync.files import files_changes_from_upload
 
 thisdir = os.path.dirname(os.path.realpath(__file__))
 sys.path.append(os.path.join(thisdir, os.pardir))
 
 
-@pytest.fixture(scope="function")
-def flask_app(request):
-    """Flask app with fresh db and initialized empty tables"""
+@pytest.fixture(scope="session")
+def session_app():
+    """Flask app shared by all tests, db tables are created only once"""
     from ..sync.db_events import remove_events
 
     application = create_app(
@@ -44,24 +53,34 @@ def flask_app(request):
     application.config["SERVER_NAME"] = "localhost.localdomain"
     application.config["SERVER_TYPE"] = "ce"
     application.config["SERVICE_ID"] = str(uuid.uuid4())
-    app_context = application.app_context()
-    app_context.push()
 
-    with app_context:
+    with application.app_context():
+        create_db()
+        # start from scratch in case of previous interrupted run
+        db.drop_all()
         db.create_all()
 
-    def teardown():
-        # clean up db
+    yield application
+
+    with application.app_context():
         db.session.remove()
         db.drop_all()
         db.engine.dispose()
+    # detach db hooks
+    remove_events()
 
-        app_context.pop()
-        # detach db hooks
-        remove_events()
 
-    request.addfinalizer(teardown)
-    return application
+@pytest.fixture(scope="function")
+def flask_app(session_app):
+    """Flask app with empty db tables"""
+    app_context = session_app.app_context()
+    app_context.push()
+    clean_db()
+
+    yield session_app
+
+    db.session.remove()
+    app_context.pop()
 
 
 @pytest.fixture(scope="function")
@@ -127,41 +146,40 @@ def diff_project(app):
     """
     from .test_project_controller import create_diff_meta
 
-    test_gpkg_file = os.path.join(test_project_dir, "test.gpkg")
+    # work on a copy of testing project files, the source dir is shared by all tests (and xdist workers)
+    project_dir = tempfile.mkdtemp(dir=TMP_DIR)
+    shutil.copytree(test_project_dir, project_dir, dirs_exist_ok=True)
+    test_gpkg_file = os.path.join(project_dir, "test.gpkg")
     try:
         geodiff = GeoDiff()
         project = Project.query.filter_by(
             name=test_project, workspace_id=test_workspace_id
         ).first()
 
-        update_meta = file_info(test_project_dir, "base.gpkg")
-        diff_meta_A = create_diff_meta(
-            "base.gpkg", "inserted_1_A.gpkg", test_project_dir
-        )
+        update_meta = file_info(project_dir, "base.gpkg")
+        diff_meta_A = create_diff_meta("base.gpkg", "inserted_1_A.gpkg", project_dir)
         diff_meta_mod = create_diff_meta(
-            "base.gpkg", "modified_1_geom.gpkg", test_project_dir
+            "base.gpkg", "modified_1_geom.gpkg", project_dir
         )
 
         patch = os.path.join(TMP_DIR, "patch")
 
-        basefile = os.path.join(test_project_dir, "base.gpkg")
+        basefile = os.path.join(project_dir, "base.gpkg")
         copy(basefile, patch)
         copy(basefile, test_gpkg_file)
         geodiff.apply_changeset(
             patch, os.path.join(TMP_DIR, diff_meta_mod["diff"]["path"])
         )
-        diff_meta_B = create_diff_meta(
-            "base.gpkg", "inserted_1_B.gpkg", test_project_dir
-        )
+        diff_meta_B = create_diff_meta("base.gpkg", "inserted_1_B.gpkg", project_dir)
 
         changes = [
             {
                 "added": [],
-                "removed": [file_info(test_project_dir, "base.gpkg")],
+                "removed": [file_info(project_dir, "base.gpkg")],
                 "updated": [],
             },
             {
-                "added": [file_info(test_project_dir, "base.gpkg")],
+                "added": [file_info(project_dir, "base.gpkg")],
                 "removed": [],
                 "updated": [],
             },
@@ -182,8 +200,8 @@ def diff_project(app):
             },
             # file renamed, by removing old and upload new - break of history
             {
-                "added": [file_info(test_project_dir, "test.gpkg")],
-                "removed": [file_info(test_project_dir, "base.gpkg")],
+                "added": [file_info(project_dir, "test.gpkg")],
+                "removed": [file_info(project_dir, "base.gpkg")],
                 "updated": [],
             },
             {"added": [], "removed": [], "updated": []},
@@ -196,7 +214,7 @@ def diff_project(app):
                     project.storage.project_dir, ver, file_meta["path"]
                 )
                 os.makedirs(os.path.dirname(new_file), exist_ok=True)
-                copy(os.path.join(test_project_dir, file_meta["path"]), new_file)
+                copy(os.path.join(project_dir, file_meta["path"]), new_file)
             elif change["updated"]:
                 file_meta = change["updated"][0]
                 f_updated = next(
@@ -220,7 +238,7 @@ def diff_project(app):
                         ),
                     )
                 else:
-                    copy(os.path.join(test_project_dir, f_updated.path), patchedfile)
+                    copy(os.path.join(project_dir, f_updated.path), patchedfile)
             else:
                 # no files uploaded, hence no action needed
                 pass
@@ -248,7 +266,7 @@ def diff_project(app):
         db.session.add(project)
         db.session.commit()
     finally:
-        os.remove(test_gpkg_file)
+        shutil.rmtree(project_dir)
     return project
 
 

server/mergin/tests/test_auth.py

@@ -435,7 +435,7 @@ def login_attempt(login, password="dummy"):
         assert mock_hashpw.call_count + mock_checkpw.call_count == 4
 
 
-def test_bcrypt_lazy_rehash(app):
+def test_bcrypt_lazy_rehash(app, monkeypatch):
     """Password is transparently rehashed on login when the cost factor changes."""
     import bcrypt
     from ..auth.app import authenticate
@@ -448,7 +448,7 @@ def test_bcrypt_lazy_rehash(app):
     user.passwd = low_rounds_hash
     db.session.commit()
 
-    app.config["BCRYPT_LOG_ROUNDS"] = 5
+    monkeypatch.setitem(app.config, "BCRYPT_LOG_ROUNDS", 5)
     result = authenticate("rehashuser", "rehashpassword")
     assert result is not None
 
@@ -963,7 +963,7 @@ def test_api_user_profile(client):
         assert key in resp.json
 
 
-def test_update_user(client):
+def test_update_user(client, monkeypatch):
     login_as_admin(client)
     user = User.query.filter_by(username="mergin").first()
     data = {"active": True, "is_admin": True}
@@ -976,7 +976,9 @@ def test_update_user(client):
     assert user.active
     assert user.is_admin
 
-    client.application.config["ENABLE_SUPERADMIN_ASSIGNMENT"] = False
+    monkeypatch.setitem(
+        client.application.config, "ENABLE_SUPERADMIN_ASSIGNMENT", False
+    )
     data = {"active": False, "is_admin": False}
     resp = client.patch(
         url_for("/.mergin_auth_controller_update_user", username=user.username),
@@ -994,7 +996,7 @@ def test_update_user(client):
     assert resp.status_code == 200
     assert not user.active
 
-    client.application.config["ENABLE_SUPERADMIN_ASSIGNMENT"] = True
+    monkeypatch.setitem(client.application.config, "ENABLE_SUPERADMIN_ASSIGNMENT", True)
     user.is_admin = False
     db.session.add(user)
     db.session.commit()

server/mergin/tests/test_celery.py

@@ -49,7 +49,7 @@
 from ..auth.models import User
 
 
-def test_send_email(app):
+def test_send_email(app, monkeypatch):
     """Test celery is actually sending emails."""
     mail = Mail()
     email_data = {
@@ -59,7 +59,7 @@ def test_send_email(app):
         "sender": "no_reply@cloudmergin.com",
     }
     with mail.record_messages() as outbox:
-        Configuration.MERGIN_TESTING = True
+        monkeypatch.setattr(Configuration, "MERGIN_TESTING", True)
         task = send_email_async.s(**email_data).apply()
         assert len(outbox) == 1
         assert task.status == "SUCCESS"
@@ -70,13 +70,13 @@ def test_send_email(app):
         assert "foo@bar.com" in outbox[0].send_to
 
         # turn off testing mode
-        Configuration.MERGIN_TESTING = False
+        monkeypatch.setattr(Configuration, "MERGIN_TESTING", False)
         task = send_email_async.s(**email_data).apply()
         assert len(outbox) == 2
         assert task.status == "SUCCESS"
         assert current_app.config["MAIL_BCC"] in outbox[1].bcc
 
-    Configuration.MERGIN_TESTING = True
+    monkeypatch.setattr(Configuration, "MERGIN_TESTING", True)
     del email_data["recipients"]
     task = send_email_async.s(**email_data).apply()
     assert task.status == "FAILURE"