From 5798a1ec1acabcd9f247e7fdfdecfb4fb3fa1064 Mon Sep 17 00:00:00 2001 From: zhiren Date: Wed, 28 Feb 2024 10:20:18 -0500 Subject: [PATCH 1/4] update folder name restriction to 100 char --- app/resources/custom_error.py | 2 +- app/utils/aggregated.py | 2 +- tests/app/utils/test_aggregated.py | 7 +++++++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/app/resources/custom_error.py b/app/resources/custom_error.py index 8f862a6b..3f1b3691 100644 --- a/app/resources/custom_error.py +++ b/app/resources/custom_error.py @@ -45,7 +45,7 @@ class Error: 'INVALID_FOLDERNAME': ( 'The input folder name is not valid. Please follow the rule:\n' ' - cannot contains special characters.\n' - ' - the length should be smaller than 20 characters.' + ' - the length should be smaller than or equal to 100 characters.' ), 'INVALID_TOKEN': 'Your login session has expired. Please try again or log in again.', 'PERMISSION_DENIED': ( diff --git a/app/utils/aggregated.py b/app/utils/aggregated.py index 95ac1a97..83bd7520 100644 --- a/app/utils/aggregated.py +++ b/app/utils/aggregated.py @@ -133,7 +133,7 @@ def get_zone(zone): def validate_folder_name(folder_name): regex = re.compile('[/:?.\\*<>|”\']') contain_invalid_char = regex.search(folder_name) - if contain_invalid_char or len(folder_name) > 20 or not folder_name: + if contain_invalid_char or len(folder_name) > 100 or not folder_name: valid = False else: valid = True diff --git a/tests/app/utils/test_aggregated.py b/tests/app/utils/test_aggregated.py index 34a02646..00ec31e5 100644 --- a/tests/app/utils/test_aggregated.py +++ b/tests/app/utils/test_aggregated.py @@ -7,6 +7,7 @@ from app.configs.app_config import AppConfig from app.utils.aggregated import check_item_duplication from app.utils.aggregated import search_item +from app.utils.aggregated import validate_folder_name from tests.conftest import decoded_token test_project_code = 'testproject' @@ -122,3 +123,9 @@ def test_check_duplicate_fail_with_error_code(httpx_mock, mocker, capsys): check_item_duplication(['test_path'], 0, 'test_project_code') out, _ = capsys.readouterr() assert out.rstrip() == '{"error": "internal server error"}' + + +@pytest.mark.parametrize('folder_name', ['/:?.\\*<>|”\'', ''.join(['1' for _ in range(101)])]) +def test_validate_folder_name(folder_name): + valid = validate_folder_name(folder_name) + assert valid is False From 3f19c53e3fca15ea74d074d52d3568c91f146899 Mon Sep 17 00:00:00 2001 From: zhiren Date: Wed, 28 Feb 2024 11:02:56 -0500 Subject: [PATCH 2/4] add error handling when uploading with wrong format of tag/attribute files --- app/commands/file.py | 14 +++++++--- app/resources/custom_error.py | 1 + app/services/output_manager/error_handler.py | 1 + tests/app/commands/test_file.py | 29 ++++++++++++++++++++ 4 files changed, 41 insertions(+), 4 deletions(-) diff --git a/app/commands/file.py b/app/commands/file.py index f38b0f13..b3590f3d 100644 --- a/app/commands/file.py +++ b/app/commands/file.py @@ -132,10 +132,16 @@ def file_put(**kwargs): # noqa: C901 output_path = kwargs.get('output_path') # load tag json file to list, and attribute file to dict - tag = [] - for t_f in tag_files: - tag.extend(json.load(t_f)) - attribute = json.load(attribute_file) if attribute_file else None + try: + tag = [] + for t_f in tag_files: + tag.extend(json.load(t_f)) + except Exception: + SrvErrorHandler.customized_handle(ECustomizedError.INVALID_TAG_FILE, True) + try: + attribute = json.load(attribute_file) if attribute_file else None + except Exception: + SrvErrorHandler.customized_handle(ECustomizedError.INVALID_TEMPLATE, True) # Check zone and upload-message zone = get_zone(zone) if zone else AppConfig.Env.green_zone.lower() diff --git a/app/resources/custom_error.py b/app/resources/custom_error.py index 3f1b3691..893d676a 100644 --- a/app/resources/custom_error.py +++ b/app/resources/custom_error.py @@ -23,6 +23,7 @@ class Error: "Attribute validation failed. Please ensure mandatory attribute '%s' have value and try again." ), 'INVALID_TEMPLATE': 'Attribute validation failed. Please correct JSON format and try again.', + 'INVALID_TAG_FILE': 'Tag files validation failed. Please correct JSON format and try again.', 'LIMIT_TAG_ERROR': 'Tag limit has been reached. A maximum of 10 tags are allowed per file.', 'INVALID_TAG_ERROR': ( 'Invalid tag format. Tags must be between 1 and 32 characters long ' diff --git a/app/services/output_manager/error_handler.py b/app/services/output_manager/error_handler.py index a7e2c79f..53e5d480 100644 --- a/app/services/output_manager/error_handler.py +++ b/app/services/output_manager/error_handler.py @@ -22,6 +22,7 @@ class ECustomizedError(enum.Enum): TEXT_TOO_LONG = 'TEXT_TOO_LONG' FIELD_REQUIRED = 'FIELD_REQUIRED' INVALID_TEMPLATE = 'INVALID_TEMPLATE' + INVALID_TAG_FILE = 'INVALID_TAG_FILE' LIMIT_TAG_ERROR = 'LIMIT_TAG_ERROR' INVALID_TAG_ERROR = 'INVALID_TAG_ERROR' RESERVED_TAG = 'RESERVED_TAG' diff --git a/tests/app/commands/test_file.py b/tests/app/commands/test_file.py index 3b6adb0e..3f1f9477 100644 --- a/tests/app/commands/test_file.py +++ b/tests/app/commands/test_file.py @@ -55,6 +55,35 @@ def test_file_upload_command_success_with_attribute(mocker, cli_runner): attribute_mock.assert_called_once() +def test_file_upload_failed_with_invalid_tag_file(mocker, cli_runner): + # create invalid tag file with wrong format + runner = click.testing.CliRunner() + with runner.isolated_filesystem(): + with open('wrong_tag.json', 'w') as f: + f.write('wrong_tag.json') + + result = cli_runner.invoke( + file_put, ['--project-path', 'test', '--thread', 1, '--tag', 'wrong_tag.json', 'wrong_tag.json'] + ) + assert result.exit_code == 0 + assert result.output == customized_error_msg(ECustomizedError.INVALID_TAG_FILE) + '\n' + + +def test_file_upload_failed_with_invalid_attribute_file(mocker, cli_runner): + # create invalid attribute file with wrong format + runner = click.testing.CliRunner() + with runner.isolated_filesystem(): + with open('wrong_attribute.json', 'w') as f: + f.write('wrong_attribute.json') + + result = cli_runner.invoke( + file_put, + ['--project-path', 'test', '--thread', 1, '--attribute', 'wrong_attribute.json', 'wrong_attribute.json'], + ) + assert result.exit_code == 0 + assert result.output == customized_error_msg(ECustomizedError.INVALID_TEMPLATE) + '\n' + + def test_resumable_upload_command_success(mocker, cli_runner): mocker.patch('os.path.exists', return_value=True) # mock the open function From b975141001160ec9c1d5273aa1b19152f0c448ea Mon Sep 17 00:00:00 2001 From: zhiren Date: Wed, 28 Feb 2024 11:32:01 -0500 Subject: [PATCH 3/4] fixup the metadata download command cannt download item metadata/tags/attributes from project folder --- .../file_metadata/file_metadata_client.py | 10 ++- tests/app/commands/test_file.py | 4 +- .../test_file_metadata_client.py | 73 +++++++++++++++++++ 3 files changed, 84 insertions(+), 3 deletions(-) diff --git a/app/services/file_manager/file_metadata/file_metadata_client.py b/app/services/file_manager/file_metadata/file_metadata_client.py index 8fdbd77f..ad3833a3 100644 --- a/app/services/file_manager/file_metadata/file_metadata_client.py +++ b/app/services/file_manager/file_metadata/file_metadata_client.py @@ -109,7 +109,15 @@ def download_file_metadata(self) -> List[Dict[str, Any]]: """ project_code, object_path = self.file_path.split('/', 1) - item_res = search_item(project_code, self.zone, object_path).get('result', {}) + item_res = search_item(project_code, self.zone, object_path) + # double check if the file is in shared folder + if item_res.get('code') == 404: + item_res = search_item(project_code, self.zone, f'shared/{object_path}') + if item_res.get('code') == 404: + logger.error(f'Cannot find item {self.file_path} at {self.zone}.') + exit(1) + + item_res = item_res.get('result', {}) extra_info = item_res.pop('extended', {}).get('extra') tags = extra_info.get('tags', []) attributes = extra_info.get('attributes', {}) diff --git a/tests/app/commands/test_file.py b/tests/app/commands/test_file.py index 3f1f9477..d90912d1 100644 --- a/tests/app/commands/test_file.py +++ b/tests/app/commands/test_file.py @@ -55,7 +55,7 @@ def test_file_upload_command_success_with_attribute(mocker, cli_runner): attribute_mock.assert_called_once() -def test_file_upload_failed_with_invalid_tag_file(mocker, cli_runner): +def test_file_upload_failed_with_invalid_tag_file(cli_runner): # create invalid tag file with wrong format runner = click.testing.CliRunner() with runner.isolated_filesystem(): @@ -69,7 +69,7 @@ def test_file_upload_failed_with_invalid_tag_file(mocker, cli_runner): assert result.output == customized_error_msg(ECustomizedError.INVALID_TAG_FILE) + '\n' -def test_file_upload_failed_with_invalid_attribute_file(mocker, cli_runner): +def test_file_upload_failed_with_invalid_attribute_file(cli_runner): # create invalid attribute file with wrong format runner = click.testing.CliRunner() with runner.isolated_filesystem(): diff --git a/tests/app/services/file_manager/file_metadata/test_file_metadata_client.py b/tests/app/services/file_manager/file_metadata/test_file_metadata_client.py index 48c16216..dafeeaac 100644 --- a/tests/app/services/file_manager/file_metadata/test_file_metadata_client.py +++ b/tests/app/services/file_manager/file_metadata/test_file_metadata_client.py @@ -84,3 +84,76 @@ def test_file_metadata_client_get_detail_success_with_no_tag_attributes(mocker, assert item_info == item_info assert res_attributes == {} assert tags == tags + + +def test_metadata_download_from_project_folder(mocker, httpx_mock): + item_info = { + 'id': 'test', + 'parent_id': 'test_parent', + 'parent_path': 'shared/path', + 'name': 'admin', + 'zone': 0, + 'status': 'ACTIVE', + } + tags = ['test'] + attri_template_uid = 'template_uid' + attri_template_name = 'template_name' + attributes = {attri_template_uid: {'attr_1': 'value'}} + + mocker.patch( + 'app.services.user_authentication.token_manager.SrvTokenManager.decode_access_token', + return_value=decoded_token(), + ) + + search_mock = mocker.patch( + 'app.services.file_manager.file_metadata.file_metadata_client.search_item', + ) + search_mock.side_effect = [ + {'result': {}, 'code': 404}, + {'result': {**item_info, 'extended': {'extra': {'tags': tags, 'attributes': attributes}}}}, + ] + httpx_mock.add_response( + url=AppConfig.Connections.url_portal + f'/v1/data/manifest/{attri_template_uid}', + method='GET', + json={'result': {'id': attri_template_uid, 'name': attri_template_name}}, + ) + + mocker.patch( + 'app.services.file_manager.file_metadata.file_metadata_client.FileMetaClient.save_file_metadata', + return_value=None, + ) + + file_meta_client = FileMetaClient('zone', 'project_code/object_path', 'general', 'attr', 'tag') + assert file_meta_client.project_code == 'project_code' + assert file_meta_client.object_path == 'object_path' + + item_info, res_attributes, tags = file_meta_client.download_file_metadata() + assert item_info == item_info + assert res_attributes == {attri_template_name: attributes.get(attri_template_uid)} + assert tags == tags + assert search_mock.call_count == 2 + + +def test_metadata_download_fail_when_file_doesnot_exist(mocker, capfd): + mocker.patch( + 'app.services.user_authentication.token_manager.SrvTokenManager.decode_access_token', + return_value=decoded_token(), + ) + + search_mock = mocker.patch( + 'app.services.file_manager.file_metadata.file_metadata_client.search_item', + ) + search_mock.side_effect = [{'result': {}, 'code': 404}, {'result': {}, 'code': 404}] + + file_meta_client = FileMetaClient('zone', 'project_code/object_path', 'general', 'attr', 'tag') + + try: + file_meta_client.download_file_metadata() + except SystemExit: + assert search_mock.call_count == 2 + out, _ = capfd.readouterr() + + expect = 'Cannot find item project_code/object_path at zone.\n' + assert out == expect + else: + AssertionError('SystemExit not raised') From eb85c812e7b6ee7c98cf0f1bb3f65ca6c45cf97f Mon Sep 17 00:00:00 2001 From: zhiren Date: Wed, 28 Feb 2024 15:39:38 -0500 Subject: [PATCH 4/4] bumpup version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 5fa6eaeb..128e8386 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "app" -version = "2.9.9" +version = "2.9.10" description = "This service is designed to support pilot platform" authors = ["Indoc Systems"]