diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs new file mode 100644 index 00000000..e3889197 --- /dev/null +++ b/.git-blame-ignore-revs @@ -0,0 +1,2 @@ +# Reformat code with black +107452539ec5a0e1d6e31f0697d215cc3cf3be11 diff --git a/.github/workflows/code_style.yml b/.github/workflows/code_style.yml new file mode 100644 index 00000000..2c384dc8 --- /dev/null +++ b/.github/workflows/code_style.yml @@ -0,0 +1,27 @@ +name: Code Layout + +on: push + +jobs: + code_style_python: + name: Python code convention check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - uses: psf/black@stable + with: + options: "--check --verbose --diff" + src: "./server" + + code_style_js: + name: JavaScript code convention check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + - name: Check JS + run: | + cd web-app + yarn install + yarn run lint:all diff --git a/README.md b/README.md index 56ef4dd6..b0d6fc4f 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,8 @@

- +
+[![Code Layout](https://github.com/MerginMaps/mergin/actions/workflows/code_style.yml/badge.svg)](https://github.com/MerginMaps/mergin/actions/workflows/code_style.yml)

## About diff --git a/server/mergin/app.py b/server/mergin/app.py index cb5edad0..4277b74f 100644 --- a/server/mergin/app.py +++ b/server/mergin/app.py @@ -18,6 +18,7 @@ from flask_mail import Mail from connexion.apps.flask_app import FlaskJSONEncoder from flask_wtf import FlaskForm +from wtforms import StringField from pathlib import Path import sys import time @@ -433,3 +434,19 @@ class ResponseError: def to_dict(self) -> Dict: return dict(code=self.code, detail=self.detail + f" ({self.code})") + + +def whitespace_filter(obj): + return obj.strip() if isinstance(obj, str) else obj + + +class CustomStringField(StringField): + """Custom class for string form fields""" + + def __init__(self, *args, **kwargs): + filters = kwargs.get("filters") + # add whitespace filter + kwargs["filters"] = ( + (*filters, whitespace_filter) if filters else (whitespace_filter,) + ) + super().__init__(*args, **kwargs) diff --git a/server/mergin/auth/forms.py b/server/mergin/auth/forms.py index 17937b2e..30a615fa 100644 --- a/server/mergin/auth/forms.py +++ b/server/mergin/auth/forms.py @@ -16,11 +16,7 @@ ) from .models import User -from ..app import UpdateForm - - -def whitespace_filter(obj): - return obj.strip() if isinstance(obj, str) else obj +from ..app import UpdateForm, CustomStringField def username_validation(form, field): @@ -51,20 +47,18 @@ def __call__(self, form, field): class LoginForm(FlaskForm): """Form with username and password fields for user to sign in.""" - login = StringField(validators=[DataRequired(), Length(max=80)]) + login = CustomStringField(validators=[DataRequired(), Length(max=80)]) password = PasswordField(validators=[DataRequired()]) class RegisterUserForm(FlaskForm): - username = StringField( + username = CustomStringField( "Username", validators=[validators.Length(min=4, max=25), username_validation], - filters=(whitespace_filter,), ) - email = StringField( + email = CustomStringField( "Email Address", validators=[DataRequired(), Email()], - filters=(whitespace_filter,), ) def validate(self): @@ -85,8 +79,9 @@ def validate(self): class ResetPasswordForm(FlaskForm): - email = StringField( - "Email Address", [DataRequired(), Email()], filters=(whitespace_filter,) + email = CustomStringField( + "Email Address", + [DataRequired(), Email()], ) @@ -119,9 +114,9 @@ class UserProfileDataForm(UpdateForm): """This form is for user profile update""" receive_notifications = BooleanField("Receive notifications", [Optional()]) - first_name = StringField("First Name", [Optional()], filters=(whitespace_filter,)) - last_name = StringField("Last Name", [Optional()], filters=(whitespace_filter,)) - email = StringField("Email", [Optional(), Email()], filters=(whitespace_filter,)) + first_name = CustomStringField("First Name", [Optional()]) + last_name = CustomStringField("Last Name", [Optional()]) + email = CustomStringField("Email", [Optional(), Email()]) class ApiLoginForm(LoginForm): diff --git a/server/mergin/sync/models.py b/server/mergin/sync/models.py index 1a222af9..b0544218 100644 --- a/server/mergin/sync/models.py +++ b/server/mergin/sync/models.py @@ -212,8 +212,11 @@ def file_diffs_chain(self, file, version): meta = history[history_end] # we have either full history of changes or v_x = v_x+n => no basefile in way, it is 'diffable' from the end if "diff" in meta: + # omit diff for target version as it would lead to previous version if reconstructed backward diffs = [ - value["diff"] for value in list(reversed(history.values()))[1:] + value["diff"] + for key, value in reversed(history.items()) + if key != v_x ] base_meta = history[next(iter(history))] base_meta["version"] = next(iter(history)) @@ -249,7 +252,9 @@ def file_diffs_chain(self, file, version): else: diffs = [ value["diff"] - for value in list(reversed(history.values()))[1:] + for value in list(reversed(history.values()))[ + 1: + ] # basefile has no diff ] # file was removed (or renamed for backward compatibility) else: @@ -417,6 +422,11 @@ def __init__(self, project, name, author, changes, files, ip, user_agent=None): if "diff" in change: change["diff"].pop("chunks", None) + @property + def int_name(self) -> int: + """Parsed version name as integer (v5 -> 5)""" + return int(self.name.replace("v", "")) + def diff_summary(self): """Calculate diff summary for versioned files updated with geodiff diff --git a/server/mergin/sync/permissions.py b/server/mergin/sync/permissions.py index 2d3adb4f..125b6ffd 100644 --- a/server/mergin/sync/permissions.py +++ b/server/mergin/sync/permissions.py @@ -8,6 +8,7 @@ from flask import abort, current_app from flask_login import current_user +from .utils import is_valid_uuid from ..auth.models import User from .models import ProjectAccess, Project, Upload @@ -138,6 +139,9 @@ def require_project(ws, project_name, permission): def require_project_by_uuid(uuid, permission): + if not is_valid_uuid(uuid): + abort(404) + project = ( Project.query.filter_by(id=uuid) .filter(Project.removed_at.is_(None)) diff --git a/server/mergin/sync/private_api_controller.py b/server/mergin/sync/private_api_controller.py index f867c48c..b87f83f8 100644 --- a/server/mergin/sync/private_api_controller.py +++ b/server/mergin/sync/private_api_controller.py @@ -276,7 +276,9 @@ def update_project_access(id: str): :rtype: None """ project = require_project_by_uuid(id, ProjectPermissions.Update) - user = User.query.filter_by(id=request.json["user_id"], active=True).first_or_404("User does not exist") + user = User.query.filter_by(id=request.json["user_id"], active=True).first_or_404( + "User does not exist" + ) # prevent to remove ownership of project creator if user.id == project.creator_id: abort(400, "Ownership of project creator cannot be removed") diff --git a/server/mergin/sync/public_api.yaml b/server/mergin/sync/public_api.yaml index 7843e442..9d66ab9b 100644 --- a/server/mergin/sync/public_api.yaml +++ b/server/mergin/sync/public_api.yaml @@ -460,6 +460,41 @@ paths: '404': $ref: '#/components/responses/NotFoundResp' x-openapi-router-controller: mergin.sync.public_api_controller + '/project/version/{project_id}/{version}': + get: + tags: + - project + summary: Get project version by its name (e.g. v3) + operationId: get_project_version + parameters: + - name: project_id + in: path + required: true + schema: + type: string + example: ac70181b5-7143-491b-91a6-4f533021c9a2 + - name: version + in: path + required: true + schema: + type: string + example: v3 + responses: + '200': + description: Project version detail + content: + application/json: + schema: + $ref: "#/components/schemas/ProjectVersion" + '400': + $ref: '#/components/responses/BadStatusResp' + '401': + $ref: '#/components/responses/UnauthorizedError' + '403': + $ref: '#/components/responses/Forbidden' + '404': + $ref: '#/components/responses/NotFoundResp' + x-openapi-router-controller: mergin.sync.public_api_controller /project/download/{namespace}/{project_name}: get: tags: diff --git a/server/mergin/sync/public_api_controller.py b/server/mergin/sync/public_api_controller.py index 669a8f7c..026e85ea 100644 --- a/server/mergin/sync/public_api_controller.py +++ b/server/mergin/sync/public_api_controller.py @@ -1335,3 +1335,14 @@ def get_workspace_by_id(id): ctx = {"user": current_user} data = UserWorkspaceSchema(context=ctx).dump(ws) return data, 200 + + +@auth_required +def get_project_version(project_id: str, version: str): + """Get project version by its name (e.g. v3)""" + project = require_project_by_uuid(project_id, ProjectPermissions.Read) + pv = ProjectVersion.query.filter_by( + project_id=project.id, name=version + ).first_or_404() + data = ProjectVersionSchema(exclude=["files"]).dump(pv) + return data, 200 diff --git a/server/mergin/sync/workspace.py b/server/mergin/sync/workspace.py index 6152bf0e..6d356c83 100644 --- a/server/mergin/sync/workspace.py +++ b/server/mergin/sync/workspace.py @@ -58,7 +58,11 @@ def disk_usage(self): # get only what is necessary from db to calculate final usage from .models import Project - projects_usage_list = db.session.query(Project.disk_usage).filter(Project.removed_at.is_(None)).all() + projects_usage_list = ( + db.session.query(Project.disk_usage) + .filter(Project.removed_at.is_(None)) + .all() + ) return sum(p.disk_usage for p in projects_usage_list) def user_has_permissions(self, user, permissions): diff --git a/server/mergin/tests/test_auth.py b/server/mergin/tests/test_auth.py index 2455efae..1f523813 100644 --- a/server/mergin/tests/test_auth.py +++ b/server/mergin/tests/test_auth.py @@ -24,9 +24,10 @@ def client(app): return client -# login tests: success, success with email login, invalid password, missing password, wrong headers +# login tests: success, success with trailing space, success with email login, invalid password, missing password, wrong headers test_login_data = [ ({"login": "mergin", "password": "ilovemergin"}, json_headers, 200), + ({"login": "mergin ", "password": "ilovemergin"}, json_headers, 200), ({"login": "mergin@mergin.com", "password": "ilovemergin"}, json_headers, 200), ({"login": "mergin", "password": "ilovemergi"}, json_headers, 401), ({"login": "mergin"}, json_headers, 401), @@ -37,7 +38,9 @@ def client(app): @pytest.mark.parametrize("data,headers,expected", test_login_data) def test_login(client, data, headers, expected): resp = client.post( - url_for("/.mergin_auth_controller_login"), data=json.dumps(data), headers=headers + url_for("/.mergin_auth_controller_login"), + data=json.dumps(data), + headers=headers, ) assert resp.status_code == expected if expected == 200: diff --git a/server/mergin/tests/test_config.py b/server/mergin/tests/test_config.py index 2fd6833e..318e7c0b 100644 --- a/server/mergin/tests/test_config.py +++ b/server/mergin/tests/test_config.py @@ -18,7 +18,7 @@ def test_config(client): "server_configured", "fix", "major", - "minor" + "minor", } resp = client.get("/config") assert resp.status_code == 200 diff --git a/server/mergin/tests/test_file_restore.py b/server/mergin/tests/test_file_restore.py new file mode 100644 index 00000000..7ca34684 --- /dev/null +++ b/server/mergin/tests/test_file_restore.py @@ -0,0 +1,202 @@ +# Copyright (C) Lutra Consulting Limited +# +# SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-MerginMaps-Commercial +import os +import pytest +import shutil +from sqlalchemy.orm.attributes import flag_modified + +from .. import db +from ..auth.models import User +from ..sync.models import ProjectVersion, Project, GeodiffActionHistory +from . import test_project_dir, TMP_DIR +from .utils import ( + create_project, + create_workspace, + gpkgs_are_equal, + create_blank_version, + execute_query, + push_change, +) + + +def _prepare_restore_project(working_dir: str) -> Project: + """Prepare new project with basefile called base.gpkg from testing files""" + user = User.query.filter_by(username="mergin").first() + test_workspace = create_workspace() + project = create_project("restore_test", test_workspace, user) + + # cleanup + if os.path.exists(working_dir): + shutil.rmtree(working_dir) + + os.makedirs(working_dir, exist_ok=True) + # add basefile + shutil.copy( + os.path.join(test_project_dir, "base.gpkg"), + os.path.join(working_dir, "base.gpkg"), + ) + push_change(project, "added", "base.gpkg", working_dir) + assert project.latest_version == "v1" + assert os.path.exists(os.path.join(project.storage.project_dir, "v1", "base.gpkg")) + return project + + +@pytest.mark.parametrize("forward_check", [True, False]) +def test_crud_in_version_file_restore(app, forward_check): + """Test to restore gpkg file where feature went through CRUD operations in subsequent versions""" + working_dir = os.path.join(TMP_DIR, "restore_from_diffs_with_crud") + basefile = os.path.join(working_dir, "base.gpkg") + p = _prepare_restore_project(working_dir) + + # introduce dummy changes to project before making any updated to force restore lookup from the end + if not forward_check: + for _ in range(3): + create_blank_version(p) + + # insert new feature + sql = "INSERT INTO simple (geometry, name) VALUES (GeomFromText('POINT(24.5, 38.2)', 4326), 'insert_test')" + execute_query(basefile, sql) + pv2 = push_change(p, "updated", "base.gpkg", working_dir) + assert p.latest_version == pv2.name + assert os.path.exists(os.path.join(p.storage.project_dir, pv2.name, "base.gpkg")) + # update feature + sql = "UPDATE simple SET rating=100 WHERE name='insert_test'" + execute_query(basefile, sql) + pv3 = push_change(p, "updated", "base.gpkg", working_dir) + assert p.latest_version == pv3.name + assert os.path.exists(os.path.join(p.storage.project_dir, pv3.name, "base.gpkg")) + # delete feature + sql = "DELETE FROM simple WHERE name='insert_test'" + execute_query(basefile, sql) + pv4 = push_change(p, "updated", "base.gpkg", working_dir) + assert p.latest_version == pv4.name + assert os.path.exists(os.path.join(p.storage.project_dir, pv4.name, "base.gpkg")) + + # introduce dummy changes before last project update to force restore lookup from beginning + if forward_check: + for _ in range(3): + create_blank_version(p) + + # create new version, latest is also 'basefile' from restore point of view + sql = "UPDATE simple SET rating=100 WHERE fid=1" + execute_query(basefile, sql) + pv5 = push_change(p, "updated", "base.gpkg", working_dir) + assert p.latest_version == pv5.name + assert os.path.exists(os.path.join(p.storage.project_dir, pv5.name, "base.gpkg")) + + # tests we can restore anything between pv1 and pv5 + for version in [pv2.name, pv3.name, pv4.name]: + test_file = os.path.join(p.storage.project_dir, version, "base.gpkg") + os.rename(test_file, test_file + "_backup") + p.storage.restore_versioned_file("base.gpkg", version) + assert os.path.exists(test_file) + assert gpkgs_are_equal(test_file, test_file + "_backup") + + +@pytest.mark.parametrize("forward_check", [True, False]) +def test_version_file_restore_with_no_changes(app, forward_check): + """Test to restore gpkg file from diffs where history contains some blank versions (no changes).""" + working_dir = os.path.join(TMP_DIR, "restore_from_diffs_with_gaps") + basefile = os.path.join(working_dir, "base.gpkg") + p = _prepare_restore_project(working_dir) + + if not forward_check: + for _ in range(6): + create_blank_version(p) + + base_version = p.get_latest_version().int_name + for i in range(3): + sql = "INSERT INTO simple (geometry, name) VALUES (GeomFromText('POINT(24.5, 38.2)', 4326), 'insert_test')" + execute_query(basefile, sql) + pv_latest = push_change(p, "updated", "base.gpkg", working_dir) + assert p.latest_version == pv_latest.name + assert os.path.exists( + os.path.join(p.storage.project_dir, pv_latest.name, "base.gpkg") + ) + # create "no changes" version + create_blank_version(p) + + latest_version = p.get_latest_version().int_name + # reconstruct all "diff" file versions including those where no changes were made + # but not first basefile and full latest version + for ver in range(base_version + 1, latest_version - 1): + pv = ProjectVersion.query.filter_by( + project_id=str(p.id), name=f"v{ver}" + ).first() + file = next(i for i in pv.files if i["path"] == "base.gpkg") + expected_file = os.path.join(p.storage.project_dir, file["location"]) + # pretend previous full file was removed due to storage optimization + os.rename(expected_file, expected_file + "_backup") + + p.storage.restore_versioned_file("base.gpkg", pv.name) + assert os.path.exists(expected_file) + assert gpkgs_are_equal(expected_file + "_backup", expected_file) + + +def test_version_file_restore(diff_project): + """Test restore gpkg file with history which contains deletion, force update and renaming (legacy option)""" + test_file = os.path.join(diff_project.storage.project_dir, "v4", "base.gpkg") + os.rename(test_file, test_file + "_backup") + diff_project.storage.restore_versioned_file("base.gpkg", "v4") + assert os.path.exists(test_file) + assert gpkgs_are_equal(test_file, test_file + "_backup") + + # we can restore version 7 (composed from multiple diffs from v6 and v7) + test_file = os.path.join(diff_project.storage.project_dir, "v7", "base.gpkg") + os.rename(test_file, test_file + "_backup") + diff_project.storage.restore_versioned_file("base.gpkg", "v7") + assert os.path.exists(test_file) + assert gpkgs_are_equal(test_file, test_file + "_backup") + # check we track performance of reconstruction + gh = GeodiffActionHistory.query.filter_by( + project_id=diff_project.id, target_version="v7" + ).first() + assert gh.base_version == "v5" + assert gh.geodiff_time + assert gh.copy_time + assert gh.action == "restore_file" + + # restore v6 from previous basefile v5 + test_file = os.path.join(diff_project.storage.project_dir, "v6", "base.gpkg") + os.rename(test_file, test_file + "_backup") + diff_project.storage.restore_versioned_file("base.gpkg", "v6") + assert os.path.exists(test_file) + assert gpkgs_are_equal(test_file, test_file + "_backup") + + # remove v9 and v10 to mimic that project history end with existing file + pv_8 = ProjectVersion.query.filter_by(project_id=diff_project.id, name="v8").first() + pv_9 = ProjectVersion.query.filter_by(project_id=diff_project.id, name="v9").first() + pv_10 = ProjectVersion.query.filter_by( + project_id=diff_project.id, name="v10" + ).first() + diff_project.latest_version = "v8" + diff_project.files = pv_8.files + flag_modified(diff_project, "files") + db.session.delete(pv_9) + db.session.delete(pv_10) + db.session.commit() + # restore v6 backward, from the latest file (v7=v8) + test_file = os.path.join(diff_project.storage.project_dir, "v6", "base.gpkg") + if os.path.exists(test_file): + os.remove(test_file) + diff_project.storage.restore_versioned_file("base.gpkg", "v6") + assert os.path.exists(test_file) + assert gpkgs_are_equal(test_file, test_file + "_backup") + gh = GeodiffActionHistory.query.filter_by( + project_id=diff_project.id, base_version="v7", target_version="v6" + ).first() + assert gh.geodiff_time + assert gh.copy_time + + # basefile can not be restored + test_file = os.path.join(diff_project.storage.project_dir, "v5", "base.gpkg") + os.remove(test_file) + diff_project.storage.restore_versioned_file("base.gpkg", "v5") + assert not os.path.exists(test_file) + + # no geodiff file can not be restored + test_file = os.path.join(diff_project.storage.project_dir, "v1", "test.txt") + os.remove(test_file) + diff_project.storage.restore_versioned_file("test.txt", "v1") + assert not os.path.exists(test_file) diff --git a/server/mergin/tests/test_private_project_api.py b/server/mergin/tests/test_private_project_api.py index 99bd753a..18389c44 100644 --- a/server/mergin/tests/test_private_project_api.py +++ b/server/mergin/tests/test_private_project_api.py @@ -197,7 +197,9 @@ def test_template_projects(client): u = add_user("TEMPLATES", "ilovemergin") test_workspace = create_workspace() p = create_project("my_template", test_workspace, u) - resp = client.get(url_for("/app.mergin_sync_private_api_controller_template_projects")) + resp = client.get( + url_for("/app.mergin_sync_private_api_controller_template_projects") + ) assert resp.status_code == 200 assert len(resp.json) == 1 assert ( diff --git a/server/mergin/tests/test_project_controller.py b/server/mergin/tests/test_project_controller.py index e5bb0c4b..4563909f 100644 --- a/server/mergin/tests/test_project_controller.py +++ b/server/mergin/tests/test_project_controller.py @@ -30,8 +30,8 @@ GeodiffActionHistory, ProjectRole, ) -from ..sync.schemas import ProjectSchema, ProjectListSchema -from ..sync.utils import generate_checksum, is_versioned_file, resolve_tags +from ..sync.schemas import ProjectListSchema +from ..sync.utils import generate_checksum, is_versioned_file from ..auth.models import User, UserProfile from . import ( @@ -1820,81 +1820,6 @@ def test_file_diffs_chain(diff_project): assert not diffs -def _gpkgs_are_equal(file1, file2): - """Check two geopackge files are equal by means there are no geodiff changes.""" - changes = os.path.join(TMP_DIR, "changeset" + str(uuid.uuid4())) - geodiff = GeoDiff() - geodiff.create_changeset(file1, file2, changes) - return not geodiff.has_changes(changes) - - -def test_version_file_restore(diff_project): - test_file = os.path.join(diff_project.storage.project_dir, "v4", "base.gpkg") - os.rename(test_file, test_file + "_backup") - diff_project.storage.restore_versioned_file("base.gpkg", "v4") - assert os.path.exists(test_file) - assert _gpkgs_are_equal(test_file, test_file + "_backup") - - # we can restore version 7 (composed from multiple diffs from v6 and v7) - test_file = os.path.join(diff_project.storage.project_dir, "v7", "base.gpkg") - os.rename(test_file, test_file + "_backup") - diff_project.storage.restore_versioned_file("base.gpkg", "v7") - assert os.path.exists(test_file) - assert _gpkgs_are_equal(test_file, test_file + "_backup") - # check we track performance of reconstruction - gh = GeodiffActionHistory.query.filter_by( - project_id=diff_project.id, target_version="v7" - ).first() - assert gh.base_version == "v5" - assert gh.geodiff_time - assert gh.copy_time - assert gh.action == "restore_file" - - # restore v6 from previous basefile v5 - test_file = os.path.join(diff_project.storage.project_dir, "v6", "base.gpkg") - os.rename(test_file, test_file + "_backup") - diff_project.storage.restore_versioned_file("base.gpkg", "v6") - assert os.path.exists(test_file) - assert _gpkgs_are_equal(test_file, test_file + "_backup") - - # remove v9 and v10 to mimic that project history end with existing file - pv_8 = ProjectVersion.query.filter_by(project_id=diff_project.id, name="v8").first() - pv_9 = ProjectVersion.query.filter_by(project_id=diff_project.id, name="v9").first() - pv_10 = ProjectVersion.query.filter_by( - project_id=diff_project.id, name="v10" - ).first() - diff_project.latest_version = "v8" - diff_project.files = pv_8.files - flag_modified(diff_project, "files") - db.session.delete(pv_9) - db.session.delete(pv_10) - db.session.commit() - # restore v6 backward, from the latest file (v7=v8) - test_file = os.path.join(diff_project.storage.project_dir, "v6", "base.gpkg") - if os.path.exists(test_file): - os.remove(test_file) - diff_project.storage.restore_versioned_file("base.gpkg", "v6") - assert os.path.exists(test_file) - assert _gpkgs_are_equal(test_file, test_file + "_backup") - gh = GeodiffActionHistory.query.filter_by( - project_id=diff_project.id, base_version="v7", target_version="v6" - ).first() - assert gh.geodiff_time - assert gh.copy_time - - # basefile can not be restored - test_file = os.path.join(diff_project.storage.project_dir, "v5", "base.gpkg") - os.remove(test_file) - diff_project.storage.restore_versioned_file("base.gpkg", "v5") - assert not os.path.exists(test_file) - - # no geodiff file can not be restored - test_file = os.path.join(diff_project.storage.project_dir, "v1", "test.txt") - os.remove(test_file) - diff_project.storage.restore_versioned_file("test.txt", "v1") - assert not os.path.exists(test_file) - - changeset_data = [ ("v1", "test.gpkg", 404), ("v1", "test.txt", 404), @@ -2040,186 +1965,6 @@ def test_project_conflict_files(diff_project, file): assert project_info["has_conflict"] -def _push_change(project, action, path, src_dir): - """Helper to create ProjectVersion incl. files changes based on change metadata - - :param project: project to push, Project - :param action: change action type, str - :param path: relative path of file inside project, str - :param src_dir: absolute path to directory with file upload, StrPath - - :returns: new project version, ProjectVersion - """ - current_files = project.files - new_version = project.next_version() - changes = {"added": [], "updated": [], "removed": []} - metadata = {**file_info(src_dir, path), "location": os.path.join(new_version, path)} - - if action == "added": - new_file = os.path.join(project.storage.project_dir, metadata["location"]) - os.makedirs(os.path.dirname(new_file), exist_ok=True) - shutil.copy(os.path.join(src_dir, metadata["path"]), new_file) - current_files.append(metadata) - elif action == "updated": - f_updated = next(f for f in current_files if f["path"] == path) - metadata = { - **file_info(src_dir, path), - "location": os.path.join(new_version, path), - } - patched_file = os.path.join(project.storage.project_dir, metadata["path"]) - os.makedirs(os.path.dirname(patched_file), exist_ok=True) - if ".gpkg" in path: - diff_id = str(uuid.uuid4()) - diff_name = path + "-diff-" + diff_id - basefile = os.path.join(project.storage.project_dir, f_updated["location"]) - modfile = os.path.join(src_dir, path) - changeset = os.path.join(src_dir, diff_name) - project.storage.geodiff.create_changeset(basefile, modfile, changeset) - metadata["diff"] = { - "path": diff_name, - "checksum": generate_checksum(changeset), - "size": os.path.getsize(changeset), - "chunks": [ - str(uuid.uuid4()) - for i in range( - math.ceil(file_info(src_dir, diff_name)["size"] / CHUNK_SIZE) - ) - ], - "location": os.path.join(new_version, diff_name), - } - diff_file = os.path.join( - project.storage.project_dir, metadata["diff"]["location"] - ) - os.makedirs(os.path.dirname(diff_file), exist_ok=True) - shutil.copy(changeset, diff_file) - - new_file = os.path.join(project.storage.project_dir, metadata["location"]) - os.makedirs(os.path.dirname(new_file), exist_ok=True) - shutil.copy(os.path.join(src_dir, metadata["path"]), new_file) - f_updated.update(metadata) - elif action == "removed": - f_removed = next(f for f in current_files if f["path"] == path) - current_files.remove(f_removed) - else: - return - - changes[action].append(metadata) - pv = ProjectVersion( - project, - new_version, - project.creator.username, - changes, - current_files, - "127.0.0.1", - ) - db.session.add(pv) - db.session.commit() - assert pv.project_size == sum(file["size"] for file in pv.files) - project.files = current_files - project.disk_usage = sum(file["size"] for file in project.files) - project.tags = resolve_tags(pv.files) - project.latest_version = new_version - db.session.add(project) - flag_modified(project, "files") - db.session.commit() - return pv - - -def _execute_query(file, sql): - """Open connection to gpkg file and execute SQL query""" - gpkg_conn = sqlite3.connect(file) - gpkg_conn.enable_load_extension(True) - gpkg_cur = gpkg_conn.cursor() - gpkg_cur.execute('SELECT load_extension("mod_spatialite")') - gpkg_cur.execute(sql) - gpkg_conn.commit() - gpkg_conn.close() - - -def _create_blank_version(project): - """Helper to create dummy project version with no changes to increase count""" - changes = {"added": [], "updated": [], "removed": []} - pv = ProjectVersion( - project, - project.next_version(), - project.creator.username, - changes, - project.files, - "127.0.0.1", - ) - db.session.add(pv) - project.latest_version = project.next_version() - flag_modified(project, "files") - db.session.commit() - - -@pytest.mark.parametrize("forward_check", [True, False]) -def test_crud_in_version_file_restore(app, forward_check): - """ " Test to restore gpkg file where feature went through CRUD operations in subsequent versions - Any version should be possible to restore from initial basefile or the latest file - """ - user = User.query.filter_by(username="mergin").first() - test_workspace = create_workspace() - p = create_project("restore_test", test_workspace, user) - working_dir = os.path.join(TMP_DIR, "restore_diffs") - # cleanup - if os.path.exists(working_dir): - shutil.rmtree(working_dir) - - # prepare: add insert/update/delete feature updates into base.gpkg file - os.makedirs(working_dir, exist_ok=True) - basefile = os.path.join(working_dir, "base.gpkg") - # add basefile - shutil.copy(os.path.join(test_project_dir, "base.gpkg"), basefile) - _push_change(p, "added", "base.gpkg", working_dir) - assert p.latest_version == "v1" - assert os.path.exists(os.path.join(p.storage.project_dir, "v1", "base.gpkg")) - - # introduce dummy changes to project before making any updated to force restore lookup from the end - if not forward_check: - for _ in range(3): - _create_blank_version(p) - - # insert new feature - sql = "INSERT INTO simple (geometry, name) VALUES (GeomFromText('POINT(24.5, 38.2)', 4326), 'insert_test')" - _execute_query(basefile, sql) - pv2 = _push_change(p, "updated", "base.gpkg", working_dir) - assert p.latest_version == pv2.name - assert os.path.exists(os.path.join(p.storage.project_dir, pv2.name, "base.gpkg")) - # update feature - sql = "UPDATE simple SET rating=100 WHERE name='insert_test'" - _execute_query(basefile, sql) - pv3 = _push_change(p, "updated", "base.gpkg", working_dir) - assert p.latest_version == pv3.name - assert os.path.exists(os.path.join(p.storage.project_dir, pv3.name, "base.gpkg")) - # delete feature - sql = "DELETE FROM simple WHERE name='insert_test'" - _execute_query(basefile, sql) - pv4 = _push_change(p, "updated", "base.gpkg", working_dir) - assert p.latest_version == pv4.name - assert os.path.exists(os.path.join(p.storage.project_dir, pv4.name, "base.gpkg")) - - # introduce dummy changes before last project update to force restore lookup from beginning - if forward_check: - for _ in range(3): - _create_blank_version(p) - - # create new version, latest is also 'basefile' from restore point of view - sql = "UPDATE simple SET rating=100 WHERE fid=1" - _execute_query(basefile, sql) - pv5 = _push_change(p, "updated", "base.gpkg", working_dir) - assert p.latest_version == pv5.name - assert os.path.exists(os.path.join(p.storage.project_dir, pv5.name, "base.gpkg")) - - # tests we can restore anything between pv1 and pv5 - for version in [pv2.name, pv3.name, pv4.name]: - test_file = os.path.join(p.storage.project_dir, version, "base.gpkg") - os.rename(test_file, test_file + "_backup") - p.storage.restore_versioned_file("base.gpkg", version) - assert os.path.exists(test_file) - assert _gpkgs_are_equal(test_file, test_file + "_backup") - - def test_orphan_project(client): """Test project whose creator was removed""" user = add_user("tests", "tests") @@ -2378,3 +2123,29 @@ def test_inactive_project(client, diff_project): ) assert resp.status_code == 409 assert "Project with the same name is scheduled for deletion" in resp.json["detail"] + + +def test_get_project_version(client, diff_project): + # success - latest version + resp = client.get( + f"/v1/project/version/{str(diff_project.id)}/{diff_project.latest_version}" + ) + assert resp.status_code == 200 + assert resp.json["name"] == diff_project.latest_version + + # success any older version + resp = client.get(f"/v1/project/version/{str(diff_project.id)}/v1") + assert resp.status_code == 200 + assert resp.json["name"] == "v1" + + # not existing version + resp = client.get(f"/v1/project/version/{str(diff_project.id)}/v100") + assert resp.status_code == 404 + + # invalid version identifier + resp = client.get(f"/v1/project/version/{str(diff_project.id)}/500") + assert resp.status_code == 404 + + # invalid project identifier + resp = client.get(f"/v1/project/version/1234/v10") + assert resp.status_code == 404 diff --git a/server/mergin/tests/test_workspace.py b/server/mergin/tests/test_workspace.py index ebfd7898..a12069c8 100644 --- a/server/mergin/tests/test_workspace.py +++ b/server/mergin/tests/test_workspace.py @@ -42,12 +42,14 @@ def test_workspace_implementation(client): Configuration.GLOBAL_ADMIN = True # create project with dummy file to count for workspace usage project = create_project("test_permissions", ws, user) - project.files = [{ - "path": "some_file.txt", - "location": "v1/some_file.txt", - "size": 1024, - "checksum": "89469a6482267de394c7c7270cb7ffafe694ea76", - }] + project.files = [ + { + "path": "some_file.txt", + "location": "v1/some_file.txt", + "size": 1024, + "checksum": "89469a6482267de394c7c7270cb7ffafe694ea76", + } + ] project.disk_usage = 1024 db.session.commit() assert ws.disk_usage() == 1024 diff --git a/server/mergin/tests/utils.py b/server/mergin/tests/utils.py index cfad3387..505b96cb 100644 --- a/server/mergin/tests/utils.py +++ b/server/mergin/tests/utils.py @@ -4,19 +4,24 @@ import json import shutil +import sqlite3 import uuid import math from datetime import datetime from flask import url_for, current_app import os from dateutil.tz import tzlocal +from pygeodiff import GeoDiff +from sqlalchemy.orm.attributes import flag_modified from ..auth.models import User, UserProfile -from ..sync.utils import generate_location, generate_checksum +from ..sync.utils import generate_location, generate_checksum, resolve_tags from ..sync.models import Project, ProjectAccess, ProjectVersion from ..sync.workspace import GlobalWorkspace from .. import db -from . import json_headers, DEFAULT_USER, test_project, test_project_dir +from . import json_headers, DEFAULT_USER, test_project, test_project_dir, TMP_DIR + +CHUNK_SIZE = 1024 def add_user(username, password, is_admin=False): @@ -196,7 +201,7 @@ def file_info(project_dir, path, chunk_size=1024): def upload_file_to_project(project, filename, client): - """ Add test file to project - start, upload and finish push process """ + """Add test file to project - start, upload and finish push process""" file = os.path.join(test_project_dir, filename) assert os.path.exists(file) changes = { @@ -222,3 +227,123 @@ def upload_file_to_project(project, filename, client): ) assert client.post(f"/v1/project/push/finish/{upload_id}").status_code == 200 + +def gpkgs_are_equal(file1, file2): + """Check two geopackge files are equal by means there are no geodiff changes.""" + changes = os.path.join(TMP_DIR, "changeset" + str(uuid.uuid4())) + geodiff = GeoDiff() + geodiff.create_changeset(file1, file2, changes) + return not geodiff.has_changes(changes) + + +def execute_query(file, sql): + """Open connection to gpkg file and execute SQL query""" + gpkg_conn = sqlite3.connect(file) + gpkg_conn.enable_load_extension(True) + gpkg_cur = gpkg_conn.cursor() + gpkg_cur.execute('SELECT load_extension("mod_spatialite")') + gpkg_cur.execute(sql) + gpkg_conn.commit() + gpkg_conn.close() + + +def create_blank_version(project): + """Helper to create dummy project version with no changes to increase count""" + changes = {"added": [], "updated": [], "removed": []} + pv = ProjectVersion( + project, + project.next_version(), + project.creator.username, + changes, + project.files, + "127.0.0.1", + ) + db.session.add(pv) + project.latest_version = project.next_version() + flag_modified(project, "files") + db.session.commit() + + +def push_change(project, action, path, src_dir): + """Helper to create ProjectVersion incl. files changes based on change metadata + + :param project: project to push, Project + :param action: change action type, str + :param path: relative path of file inside project, str + :param src_dir: absolute path to directory with file upload, StrPath + + :returns: new project version, ProjectVersion + """ + current_files = project.files + new_version = project.next_version() + changes = {"added": [], "updated": [], "removed": []} + metadata = {**file_info(src_dir, path), "location": os.path.join(new_version, path)} + + if action == "added": + new_file = os.path.join(project.storage.project_dir, metadata["location"]) + os.makedirs(os.path.dirname(new_file), exist_ok=True) + shutil.copy(os.path.join(src_dir, metadata["path"]), new_file) + current_files.append(metadata) + elif action == "updated": + f_updated = next(f for f in current_files if f["path"] == path) + metadata = { + **file_info(src_dir, path), + "location": os.path.join(new_version, path), + } + patched_file = os.path.join(project.storage.project_dir, metadata["path"]) + os.makedirs(os.path.dirname(patched_file), exist_ok=True) + if ".gpkg" in path: + diff_id = str(uuid.uuid4()) + diff_name = path + "-diff-" + diff_id + basefile = os.path.join(project.storage.project_dir, f_updated["location"]) + modfile = os.path.join(src_dir, path) + changeset = os.path.join(src_dir, diff_name) + project.storage.geodiff.create_changeset(basefile, modfile, changeset) + metadata["diff"] = { + "path": diff_name, + "checksum": generate_checksum(changeset), + "size": os.path.getsize(changeset), + "chunks": [ + str(uuid.uuid4()) + for i in range( + math.ceil(file_info(src_dir, diff_name)["size"] / CHUNK_SIZE) + ) + ], + "location": os.path.join(new_version, diff_name), + } + diff_file = os.path.join( + project.storage.project_dir, metadata["diff"]["location"] + ) + os.makedirs(os.path.dirname(diff_file), exist_ok=True) + shutil.copy(changeset, diff_file) + + new_file = os.path.join(project.storage.project_dir, metadata["location"]) + os.makedirs(os.path.dirname(new_file), exist_ok=True) + shutil.copy(os.path.join(src_dir, metadata["path"]), new_file) + f_updated.update(metadata) + elif action == "removed": + f_removed = next(f for f in current_files if f["path"] == path) + current_files.remove(f_removed) + else: + return + + changes[action].append(metadata) + pv = ProjectVersion( + project, + new_version, + project.creator.username, + changes, + current_files, + "127.0.0.1", + ) + db.session.add(pv) + db.session.commit() + assert pv.project_size == sum(file["size"] for file in pv.files) + project.files = current_files + project.disk_usage = sum(file["size"] for file in project.files) + project.tags = resolve_tags(pv.files) + project.latest_version = new_version + db.session.add(project) + flag_modified(project, "files") + db.session.commit() + return pv diff --git a/server/mergin/version.py b/server/mergin/version.py index b0e9d6a7..69dc6520 100644 --- a/server/mergin/version.py +++ b/server/mergin/version.py @@ -2,5 +2,6 @@ # # SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-MerginMaps-Commercial + def get_version(): return "2023.2.0" diff --git a/server/setup.py b/server/setup.py index 01181ac1..6d19a7f2 100644 --- a/server/setup.py +++ b/server/setup.py @@ -5,43 +5,43 @@ from setuptools import setup, find_packages setup( - name='mergin', + name="mergin", version="2023.2.0", - url='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/MerginMaps/mergin', - license='AGPL-3.0-only', - author='Lutra Consulting Limited', - author_email='info@merginmaps.com', - description='Mergin Maps server', - long_description='Mergin Maps server', + url="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/MerginMaps/mergin", + license="AGPL-3.0-only", + author="Lutra Consulting Limited", + author_email="info@merginmaps.com", + description="Mergin Maps server", + long_description="Mergin Maps server", packages=find_packages(), include_package_data=True, - platforms='any', + platforms="any", install_requires=[ - 'connexion', - 'flask', - 'python-dateutil', - 'flask-marshmallow', - 'marshmallow-sqlalchemy', - 'psycopg2-binary', - 'itsdangerous', - 'Flask-SQLAlchemy', - 'zipfly', - 'python-dotenv', - 'flask-login', - 'bcrypt', - 'flask-wtf', - 'flask-mail', - 'safe', - 'wtforms-json', - 'pytz', - 'pygeodiff', - 'pathvalidate', - 'celery', - 'result', - 'binaryornot', - 'python-decouple', - 'urllib3', - 'shapely', - 'psycogreen' + "connexion", + "flask", + "python-dateutil", + "flask-marshmallow", + "marshmallow-sqlalchemy", + "psycopg2-binary", + "itsdangerous", + "Flask-SQLAlchemy", + "zipfly", + "python-dotenv", + "flask-login", + "bcrypt", + "flask-wtf", + "flask-mail", + "safe", + "wtforms-json", + "pytz", + "pygeodiff", + "pathvalidate", + "celery", + "result", + "binaryornot", + "python-decouple", + "urllib3", + "shapely", + "psycogreen", ], ) diff --git a/web-app/babel.config.js b/web-app/babel.config.js new file mode 100644 index 00000000..716b0237 --- /dev/null +++ b/web-app/babel.config.js @@ -0,0 +1,3 @@ +module.exports = { + presets: ['@vue/cli-plugin-babel/preset'] +} diff --git a/web-app/package.json b/web-app/package.json index 22091a1d..2de18a63 100644 --- a/web-app/package.json +++ b/web-app/package.json @@ -26,13 +26,13 @@ "build:admin:dev": "yarn workspace @mergin/admin-app build:dev", "clean:libs": "yarn workspace @mergin/lib clean && yarn workspace @mergin/admin-lib clean", "clean:libs:noadmin": "yarn workspace @mergin/lib clean", - "lint": "yarn workspace @mergin/app lint", "i18n:report": "yarn workspace @mergin/app 18n:report", "types:libs": "yarn workspaces run build:types", "types:lib": "yarn workspace @mergin/lib build:types", "types:admin-lib": "yarn workspace @mergin/admin-lib build:types", "watch:lib": "yarn workspace @mergin/lib build:lib:watch", - "watch:admin-lib": "yarn workspace @mergin/admin-lib build:lib:watch" + "watch:admin-lib": "yarn workspace @mergin/admin-lib build:lib:watch", + "lint:all": "yarn workspaces run lint" }, "devDependencies": { "@babel/core": "^7.17.9", diff --git a/web-app/packages/admin-app/package.json b/web-app/packages/admin-app/package.json index d4565b8d..0e75b39d 100644 --- a/web-app/packages/admin-app/package.json +++ b/web-app/packages/admin-app/package.json @@ -13,7 +13,7 @@ "build:lib:all:prod": "yarn build:lib:prod && yarn build:types", "build:types": "echo 'Not defined script'", "clean": "echo 'Nothing to clean.'", - "lint": "vue-cli-service lint --fix", + "lint": "eslint . --config .eslintrc.js --ext .js,.vue,.ts", "i18n:report": "vue-cli-service i18n:report --src './src/admin/**/*.?(js|vue)' --locales './src/admin/locales/**/*.json'" }, "dependencies": { diff --git a/web-app/packages/admin-lib/.eslintrc.js b/web-app/packages/admin-lib/.eslintrc.js index 53487599..51dfd9e4 100644 --- a/web-app/packages/admin-lib/.eslintrc.js +++ b/web-app/packages/admin-lib/.eslintrc.js @@ -1,13 +1,11 @@ module.exports = { - root: true, env: { node: true }, extends: [ 'plugin:vue/essential', 'eslint:recommended', - '@vue/typescript/recommended', - '../../.eslintrc.js' + '@vue/typescript/recommended' ], parser: 'vue-eslint-parser', parserOptions: { diff --git a/web-app/packages/admin-lib/.eslintrc.lint.js b/web-app/packages/admin-lib/.eslintrc.lint.js new file mode 100644 index 00000000..defb45a1 --- /dev/null +++ b/web-app/packages/admin-lib/.eslintrc.lint.js @@ -0,0 +1,4 @@ +module.exports = { + root: true, + extends: '.eslintrc.js' +} diff --git a/web-app/packages/admin-lib/package.json b/web-app/packages/admin-lib/package.json index c87bd7da..c7c8175d 100644 --- a/web-app/packages/admin-lib/package.json +++ b/web-app/packages/admin-lib/package.json @@ -17,7 +17,7 @@ "clean:lib": "rimraf ./dist", "clean:types:temp": "rimraf ./**/*.vue.ts", "clean:types": "rimraf ./types ./tsconfig.tsbuildinfo && yarn clean:types:temp", - "lint": "vue-cli-service lint" + "lint": "eslint . --config .eslintrc.lint.js ../../.eslintrc.js --no-eslintrc --ext .js,.vue,.ts" }, "dependencies": { "@mergin/lib": "file:../lib", diff --git a/web-app/packages/admin-lib/src/modules/project/views/ProjectView.vue b/web-app/packages/admin-lib/src/modules/project/views/ProjectView.vue index f2d4817b..24b899b0 100644 --- a/web-app/packages/admin-lib/src/modules/project/views/ProjectView.vue +++ b/web-app/packages/admin-lib/src/modules/project/views/ProjectView.vue @@ -12,6 +12,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-MerginMaps-Commercial :asAdmin="asAdmin" :location="location" :show-settings="true" + :show-namespace="false" @open-clone-dialog="openCloneDialog" /> diff --git a/web-app/packages/app/package.json b/web-app/packages/app/package.json index 4adf89c5..c21986b6 100644 --- a/web-app/packages/app/package.json +++ b/web-app/packages/app/package.json @@ -13,7 +13,7 @@ "build:lib:all": "yarn build:lib && yarn build:types", "build:lib:all:prod": "yarn build:lib:prod && yarn build:types", "clean": "echo 'Nothing to clean.'", - "lint": "vue-cli-service lint --fix", + "lint": "eslint . --config .eslintrc.js --ext .js,.vue,.ts", "i18n:report": "vue-cli-service i18n:report --src './src/admin/**/*.?(js|vue)' --locales './src/admin/locales/**/*.json'" }, "dependencies": { diff --git a/web-app/packages/app/src/modules/project/views/ProjectView.vue b/web-app/packages/app/src/modules/project/views/ProjectView.vue index 92b1dac5..5638f0ca 100644 --- a/web-app/packages/app/src/modules/project/views/ProjectView.vue +++ b/web-app/packages/app/src/modules/project/views/ProjectView.vue @@ -6,7 +6,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-MerginMaps-Commercial