Fix 142

dbsync.py

@@ -18,6 +18,7 @@
 import re
 import pathlib
 import logging
+import typing
 
 import psycopg2
 import psycopg2.extensions
@@ -495,8 +496,10 @@ def _get_project_version(work_path) -> str:
     return mp.version()
 
 
-def _get_project_id(mp: MerginProject):
+def _get_project_id(mp: typing.Union[MerginProject, str]):
     """Returns the project ID"""
+    if isinstance(mp, str):
+        mp = _get_mergin_project(mp)
     try:
         project_id = uuid.UUID(mp.project_id())
     except (
@@ -523,7 +526,7 @@ def _set_db_project_comment(
         "version": version,
     }
     if project_id:
-        comment["project_id"] = project_id
+        comment["project_id"] = str(project_id)
     if error:
         comment["error"] = error
     cur = conn.cursor()
@@ -535,7 +538,7 @@ def _set_db_project_comment(
     conn.commit()
 
 
-def _get_db_project_comment(conn, schema):
+def _get_db_project_comment(conn, schema) -> typing.Optional[typing.Dict]:
     """Get Mergin Maps project name and its current version in db schema"""
     cur = conn.cursor()
     schema = _add_quotes_to_schema_name(schema)
@@ -821,12 +824,12 @@ def pull(conn_cfg, mc):
 
     os.remove(gpkg_basefile_old)
     conn = psycopg2.connect(conn_cfg.conn_info)
-    version = _get_project_version(work_dir)
     _set_db_project_comment(
         conn,
         conn_cfg.base,
         conn_cfg.mergin_project,
-        version,
+        version=_get_project_version(work_dir),
+        project_id=_get_project_id(work_dir),
     )
 
 
@@ -1043,7 +1046,13 @@ def push(conn_cfg, mc):
     _geodiff_apply_changeset(
         conn_cfg.driver, conn_cfg.conn_info, conn_cfg.base, tmp_changeset_file, ignored_tables, include_tables
     )
-    _set_db_project_comment(conn, conn_cfg.base, conn_cfg.mergin_project, version)
+    _set_db_project_comment(
+        conn,
+        conn_cfg.base,
+        conn_cfg.mergin_project,
+        version,
+        project_id=_get_project_id(work_dir),
+    )
 
 
 def init(
@@ -1124,18 +1133,26 @@ def init(
                 f"Downloading version {db_proj_info['version']} of Mergin Maps project {conn_cfg.mergin_project} "
                 f"to {work_dir}"
             )
+            # project ID is missing in comments created by older versions of db-sync - skip the check in that case,
+            # it will be stored in the comment by next pull/push
+            db_project_id_str = db_proj_info.get("project_id", None)
+            if db_project_id_str:
+                project_info = mc.project_info(conn_cfg.mergin_project)
+                if uuid.UUID(db_project_id_str) != uuid.UUID(project_info["id"]):
+                    raise DbSyncError(
+                        "Mergin Maps project ID doesn't match Mergin Maps project ID stored in the database. "
+                        "Did you change configuration from one Mergin Maps project to another? "
+                        f"You either need to remove schema `{conn_cfg.base}` from Database or use `--force-init` option. "
+                        f"{FORCE_INIT_MESSAGE}"
+                    )
             mc.download_project(conn_cfg.mergin_project, work_dir, db_proj_info["version"])
         else:
             # Get project ID from DB if available
             try:
                 local_version = _get_project_version(work_dir)
                 logging.debug(f"Working directory {work_dir} already exists, with project version {local_version}")
                 # Compare local and database project version
-                db_project_id_str = getattr(
-                    db_proj_info,
-                    "project_id",
-                    None,
-                )
+                db_project_id_str = db_proj_info.get("project_id", None)
                 db_project_id = uuid.UUID(db_project_id_str) if db_project_id_str else None
                 mp = _get_mergin_project(work_dir)
                 local_project_id = _get_project_id(mp)
@@ -1298,6 +1315,7 @@ def init(
             conn_cfg.base,
             conn_cfg.mergin_project,
             local_version,
+            project_id=_get_project_id(work_dir),
         )
     else:
         if not modified_schema_exists:
@@ -1417,12 +1435,12 @@ def init(
         mc.push_project(work_dir)
 
         # mark project version into db schema
-        version = _get_project_version(work_dir)
         _set_db_project_comment(
             conn,
             conn_cfg.base,
             conn_cfg.mergin_project,
-            version,
+            version=_get_project_version(work_dir),
+            project_id=_get_project_id(work_dir),
         )
 
 

test/conftest.py

@@ -18,7 +18,7 @@
 API_USER = os.environ.get("TEST_API_USERNAME")
 USER_PWD = os.environ.get("TEST_API_PASSWORD")
 WORKSPACE = os.environ.get("TEST_API_WORKSPACE")
-TMP_DIR = tempfile.gettempdir()
+TMP_DIR = os.path.join(tempfile.gettempdir(), "dbsync_test")
 TEST_DATA_DIR = os.path.join(
     os.path.dirname(os.path.realpath(__file__)),
     "test_data",
@@ -178,7 +178,7 @@ def init_sync_from_geopackage(
     dbsync_init(mc)
 
 
-@pytest.fixture(scope="function")
+@pytest.fixture(scope="session")
 def mc():
     assert SERVER_URL and API_USER and USER_PWD
     # assert SERVER_URL and SERVER_URL.rstrip('/') != 'https://app.merginmaps.com/' and API_USER and USER_PWD
@@ -310,3 +310,30 @@ def init_sync_from_db(mc: MerginClient, project_name: str, path_sql_file: str, i
     )
 
     dbsync_init(mc)
+
+
+def _clean_workspace(mc: MerginClient, workspace: str) -> None:
+    """Immediately delete all projects in the test workspace."""
+
+    projects = mc.projects_list(only_namespace=workspace)
+
+    for project in projects:
+        mc.delete_project_now(f"{workspace}/{project['name']}")
+
+
+def _remove_dir(dir_path: str) -> None:
+    """Remove directory if it exists."""
+
+    if os.path.exists(dir_path):
+        shutil.rmtree(dir_path)
+
+
+@pytest.fixture(autouse=True, scope="session")
+def clean_dir_and_workspace(mc: MerginClient):
+    """Remove and recreate temporary directory for test files. Remove it after test session."""
+    _clean_workspace(mc, WORKSPACE)
+    _remove_dir(TMP_DIR)
+    os.makedirs(TMP_DIR)
+    yield
+    _remove_dir(TMP_DIR)
+    _clean_workspace(mc, WORKSPACE)

test/test_init_db.py

@@ -13,7 +13,15 @@
     MerginClient,
 )
 
-from dbsync import dbsync_pull, dbsync_push, config, DbSyncError
+from dbsync import (
+    dbsync_pull,
+    dbsync_push,
+    config,
+    DbSyncError,
+    dbsync_init,
+    _get_db_project_comment,
+    _set_db_project_comment,
+)
 
 from .conftest import (
     GEODIFF_EXE,
@@ -148,3 +156,108 @@ def test_missing_table(mc: MerginClient):
         init_sync_from_db(mc, project_name, path_test_data("create_another_schema.sql"))
 
     assert "The 'modified' schema does not exist" in str(err.value)
+
+
+def test_mm_project_change(mc: MerginClient, db_connection):
+    """Test that after init and local changes the changes are correctly pushed to database"""
+    project_name = "test_project_change"
+    project_full_name = complete_project_name(project_name)
+    project_dir = name_project_dir(project_name)
+    db_schema_main = "test_init_from_db_main"
+    db_schema_base = "test_init_from_db_base"
+
+    path_synced_gpkg = project_dir + "/" + filename_sync_gpkg()
+
+    init_sync_from_db(mc, project_name, path_test_data("create_base.sql"))
+
+    cur = db_connection.cursor()
+
+    # check that there are 3 features prior to changes
+    cur.execute(f'SELECT COUNT(*) from {db_schema_main}."simple"')
+    assert cur.fetchone()[0] == 3
+
+    mc.download_project(project_full_name, project_dir)
+
+    # make changes in GPKG to create new version of the project
+    shutil.copy(path_test_data("inserted_point_from_db.gpkg"), path_synced_gpkg)
+
+    # push project
+    mc.push_project(project_dir)
+
+    # run sync
+    dbsync_pull(mc)
+    dbsync_push(mc)
+
+    # check that new feature was added
+    cur.execute(f'SELECT COUNT(*) from {db_schema_main}."simple"')
+    assert cur.fetchone()[0] == 4
+
+    project_name = "test_project_change_2"
+    project_full_name = complete_project_name(project_name)
+    project_dir = name_project_dir(project_name)
+    mc.create_project_and_push(project_full_name, project_dir)
+
+    # change config to new project
+    config.update(
+        {
+            "CONNECTIONS": [
+                {
+                    "driver": "postgres",
+                    "conn_info": DB_CONNINFO,
+                    "modified": db_schema_main,
+                    "base": db_schema_base,
+                    "mergin_project": project_full_name,
+                    "sync_file": filename_sync_gpkg(),
+                }
+            ]
+        }
+    )
+
+    # run init
+    with pytest.raises(
+        DbSyncError, match="Mergin Maps project ID doesn't match Mergin Maps project ID stored in the database"
+    ):
+        dbsync_init(mc)
+
+
+def test_init_with_legacy_comment_without_project_id(mc: MerginClient, db_connection):
+    """Test that init does not fail on base schema comment without project ID (created by older versions of db-sync)
+    and that the project ID is added to the comment by next pull"""
+    project_name = "test_legacy_comment"
+    project_full_name = complete_project_name(project_name)
+    project_dir = name_project_dir(project_name)
+    sync_project_dir = name_project_sync_dir(project_name)
+    db_schema_main = "test_init_from_db_main"
+    db_schema_base = "test_init_from_db_base"
+
+    init_sync_from_db(mc, project_name, path_test_data("create_base.sql"))
+
+    # comment created by the current version contains project ID
+    comment = _get_db_project_comment(db_connection, db_schema_base)
+    server_project_id = mc.project_info(project_full_name)["id"]
+    assert comment["project_id"] == server_project_id
+
+    # simulate comment created by older version of db-sync - without project ID
+    _set_db_project_comment(db_connection, db_schema_base, comment["name"], comment["version"])
+    comment = _get_db_project_comment(db_connection, db_schema_base)
+    assert "project_id" not in comment
+
+    # init with existing working directory must not fail
+    dbsync_init(mc)
+
+    # init without working directory (e.g. after upgrade of docker container) must not fail
+    shutil.rmtree(sync_project_dir)
+    dbsync_init(mc)
+    assert os.path.exists(sync_project_dir)
+    comment = _get_db_project_comment(db_connection, db_schema_base)
+    assert "project_id" not in comment
+
+    # create new version of the project on server
+    mc.download_project(project_full_name, project_dir)
+    shutil.copy(path_test_data("inserted_point_from_db.gpkg"), os.path.join(project_dir, filename_sync_gpkg()))
+    mc.push_project(project_dir)
+
+    # after pulling the latest changes, the project ID should be existing
+    dbsync_pull(mc)
+    comment = _get_db_project_comment(db_connection, db_schema_base)
+    assert comment["project_id"] == server_project_id