From 43623a2b32716fcc7d5f0188a9c19e8f9310ff38 Mon Sep 17 00:00:00 2001 From: Daniel Date: Tue, 17 Oct 2023 15:51:48 -0400 Subject: [PATCH 1/8] PILOT-3962: Port over changes from 2.7.3 to 2.7.4 --- app/resources/custom_help.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/resources/custom_help.py b/app/resources/custom_help.py index 3b6412ef..1b708a11 100644 --- a/app/resources/custom_help.py +++ b/app/resources/custom_help.py @@ -6,7 +6,7 @@ class HelpPage: page = { 'update': { - 'version': '2.7.2', + 'version': '2.7.4', '1': 'Add new feature for folder merging', '2': 'Secure the config file', '3': 'Optimize logic, input and error message', diff --git a/pyproject.toml b/pyproject.toml index f2b2b44d..7bf376f9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "app" -version = "2.7.3" +version = "2.7.4" description = "This service is designed to support pilot platform" authors = ["Indoc Systems"] From 6af654f62e281c85f672a4d311f5d04bc2a8fcdf Mon Sep 17 00:00:00 2001 From: zhiren Date: Thu, 23 Nov 2023 12:34:29 -0500 Subject: [PATCH 2/8] prepare the release branch --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index eab325d4..ce7a52f1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "app" -version = "2.8.5" +version = "2.9.0" description = "This service is designed to support pilot platform" authors = ["Indoc Systems"] From f377927a29005e01e0f9dcb7fb2ddd8d572f5216 Mon Sep 17 00:00:00 2001 From: zhiren Date: Thu, 23 Nov 2023 15:03:19 -0500 Subject: [PATCH 3/8] use correct version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index ce7a52f1..eab325d4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "app" -version = "2.9.0" +version = "2.8.5" description = "This service is designed to support pilot platform" authors = ["Indoc Systems"] From 4f55e2874ddfbb299990d5c988757c34fe8a4db5 Mon Sep 17 00:00:00 2001 From: zhiren Date: Wed, 21 Feb 2024 14:57:20 -0500 Subject: [PATCH 4/8] bumup version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index aba8fe43..07a4eeb4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "app" -version = "2.9.8" +version = "2.10.0" description = "This service is designed to support pilot platform" authors = ["Indoc Systems"] From b47255aa54df5a63eea4b1357d13725defe8411c Mon Sep 17 00:00:00 2001 From: zhiren Date: Wed, 21 Feb 2024 15:25:21 -0500 Subject: [PATCH 5/8] bump up to correct version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 07a4eeb4..5fa6eaeb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "app" -version = "2.10.0" +version = "2.9.9" description = "This service is designed to support pilot platform" authors = ["Indoc Systems"] From 3ca9f01f49185883f892c5ee88bbd12532f2ad90 Mon Sep 17 00:00:00 2001 From: Color Zhan Date: Wed, 28 Feb 2024 15:50:53 -0500 Subject: [PATCH 6/8] Hotfix20240228: hotfix to staging (#129) * update folder name restriction to 100 char * add error handling when uploading with wrong format of tag/attribute files * fixup the metadata download command cannt download item metadata/tags/attributes from project folder * bumpup version --------- Co-authored-by: zhiren --- app/commands/file.py | 14 +++- app/resources/custom_error.py | 3 +- .../file_metadata/file_metadata_client.py | 10 ++- app/services/output_manager/error_handler.py | 1 + app/utils/aggregated.py | 2 +- pyproject.toml | 2 +- tests/app/commands/test_file.py | 29 ++++++++ .../test_file_metadata_client.py | 73 +++++++++++++++++++ tests/app/utils/test_aggregated.py | 7 ++ 9 files changed, 133 insertions(+), 8 deletions(-) diff --git a/app/commands/file.py b/app/commands/file.py index f38b0f13..b3590f3d 100644 --- a/app/commands/file.py +++ b/app/commands/file.py @@ -132,10 +132,16 @@ def file_put(**kwargs): # noqa: C901 output_path = kwargs.get('output_path') # load tag json file to list, and attribute file to dict - tag = [] - for t_f in tag_files: - tag.extend(json.load(t_f)) - attribute = json.load(attribute_file) if attribute_file else None + try: + tag = [] + for t_f in tag_files: + tag.extend(json.load(t_f)) + except Exception: + SrvErrorHandler.customized_handle(ECustomizedError.INVALID_TAG_FILE, True) + try: + attribute = json.load(attribute_file) if attribute_file else None + except Exception: + SrvErrorHandler.customized_handle(ECustomizedError.INVALID_TEMPLATE, True) # Check zone and upload-message zone = get_zone(zone) if zone else AppConfig.Env.green_zone.lower() diff --git a/app/resources/custom_error.py b/app/resources/custom_error.py index 8f862a6b..893d676a 100644 --- a/app/resources/custom_error.py +++ b/app/resources/custom_error.py @@ -23,6 +23,7 @@ class Error: "Attribute validation failed. Please ensure mandatory attribute '%s' have value and try again." ), 'INVALID_TEMPLATE': 'Attribute validation failed. Please correct JSON format and try again.', + 'INVALID_TAG_FILE': 'Tag files validation failed. Please correct JSON format and try again.', 'LIMIT_TAG_ERROR': 'Tag limit has been reached. A maximum of 10 tags are allowed per file.', 'INVALID_TAG_ERROR': ( 'Invalid tag format. Tags must be between 1 and 32 characters long ' @@ -45,7 +46,7 @@ class Error: 'INVALID_FOLDERNAME': ( 'The input folder name is not valid. Please follow the rule:\n' ' - cannot contains special characters.\n' - ' - the length should be smaller than 20 characters.' + ' - the length should be smaller than or equal to 100 characters.' ), 'INVALID_TOKEN': 'Your login session has expired. Please try again or log in again.', 'PERMISSION_DENIED': ( diff --git a/app/services/file_manager/file_metadata/file_metadata_client.py b/app/services/file_manager/file_metadata/file_metadata_client.py index 8fdbd77f..ad3833a3 100644 --- a/app/services/file_manager/file_metadata/file_metadata_client.py +++ b/app/services/file_manager/file_metadata/file_metadata_client.py @@ -109,7 +109,15 @@ def download_file_metadata(self) -> List[Dict[str, Any]]: """ project_code, object_path = self.file_path.split('/', 1) - item_res = search_item(project_code, self.zone, object_path).get('result', {}) + item_res = search_item(project_code, self.zone, object_path) + # double check if the file is in shared folder + if item_res.get('code') == 404: + item_res = search_item(project_code, self.zone, f'shared/{object_path}') + if item_res.get('code') == 404: + logger.error(f'Cannot find item {self.file_path} at {self.zone}.') + exit(1) + + item_res = item_res.get('result', {}) extra_info = item_res.pop('extended', {}).get('extra') tags = extra_info.get('tags', []) attributes = extra_info.get('attributes', {}) diff --git a/app/services/output_manager/error_handler.py b/app/services/output_manager/error_handler.py index a7e2c79f..53e5d480 100644 --- a/app/services/output_manager/error_handler.py +++ b/app/services/output_manager/error_handler.py @@ -22,6 +22,7 @@ class ECustomizedError(enum.Enum): TEXT_TOO_LONG = 'TEXT_TOO_LONG' FIELD_REQUIRED = 'FIELD_REQUIRED' INVALID_TEMPLATE = 'INVALID_TEMPLATE' + INVALID_TAG_FILE = 'INVALID_TAG_FILE' LIMIT_TAG_ERROR = 'LIMIT_TAG_ERROR' INVALID_TAG_ERROR = 'INVALID_TAG_ERROR' RESERVED_TAG = 'RESERVED_TAG' diff --git a/app/utils/aggregated.py b/app/utils/aggregated.py index 95ac1a97..83bd7520 100644 --- a/app/utils/aggregated.py +++ b/app/utils/aggregated.py @@ -133,7 +133,7 @@ def get_zone(zone): def validate_folder_name(folder_name): regex = re.compile('[/:?.\\*<>|”\']') contain_invalid_char = regex.search(folder_name) - if contain_invalid_char or len(folder_name) > 20 or not folder_name: + if contain_invalid_char or len(folder_name) > 100 or not folder_name: valid = False else: valid = True diff --git a/pyproject.toml b/pyproject.toml index 5fa6eaeb..128e8386 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "app" -version = "2.9.9" +version = "2.9.10" description = "This service is designed to support pilot platform" authors = ["Indoc Systems"] diff --git a/tests/app/commands/test_file.py b/tests/app/commands/test_file.py index 3b6adb0e..d90912d1 100644 --- a/tests/app/commands/test_file.py +++ b/tests/app/commands/test_file.py @@ -55,6 +55,35 @@ def test_file_upload_command_success_with_attribute(mocker, cli_runner): attribute_mock.assert_called_once() +def test_file_upload_failed_with_invalid_tag_file(cli_runner): + # create invalid tag file with wrong format + runner = click.testing.CliRunner() + with runner.isolated_filesystem(): + with open('wrong_tag.json', 'w') as f: + f.write('wrong_tag.json') + + result = cli_runner.invoke( + file_put, ['--project-path', 'test', '--thread', 1, '--tag', 'wrong_tag.json', 'wrong_tag.json'] + ) + assert result.exit_code == 0 + assert result.output == customized_error_msg(ECustomizedError.INVALID_TAG_FILE) + '\n' + + +def test_file_upload_failed_with_invalid_attribute_file(cli_runner): + # create invalid attribute file with wrong format + runner = click.testing.CliRunner() + with runner.isolated_filesystem(): + with open('wrong_attribute.json', 'w') as f: + f.write('wrong_attribute.json') + + result = cli_runner.invoke( + file_put, + ['--project-path', 'test', '--thread', 1, '--attribute', 'wrong_attribute.json', 'wrong_attribute.json'], + ) + assert result.exit_code == 0 + assert result.output == customized_error_msg(ECustomizedError.INVALID_TEMPLATE) + '\n' + + def test_resumable_upload_command_success(mocker, cli_runner): mocker.patch('os.path.exists', return_value=True) # mock the open function diff --git a/tests/app/services/file_manager/file_metadata/test_file_metadata_client.py b/tests/app/services/file_manager/file_metadata/test_file_metadata_client.py index 48c16216..dafeeaac 100644 --- a/tests/app/services/file_manager/file_metadata/test_file_metadata_client.py +++ b/tests/app/services/file_manager/file_metadata/test_file_metadata_client.py @@ -84,3 +84,76 @@ def test_file_metadata_client_get_detail_success_with_no_tag_attributes(mocker, assert item_info == item_info assert res_attributes == {} assert tags == tags + + +def test_metadata_download_from_project_folder(mocker, httpx_mock): + item_info = { + 'id': 'test', + 'parent_id': 'test_parent', + 'parent_path': 'shared/path', + 'name': 'admin', + 'zone': 0, + 'status': 'ACTIVE', + } + tags = ['test'] + attri_template_uid = 'template_uid' + attri_template_name = 'template_name' + attributes = {attri_template_uid: {'attr_1': 'value'}} + + mocker.patch( + 'app.services.user_authentication.token_manager.SrvTokenManager.decode_access_token', + return_value=decoded_token(), + ) + + search_mock = mocker.patch( + 'app.services.file_manager.file_metadata.file_metadata_client.search_item', + ) + search_mock.side_effect = [ + {'result': {}, 'code': 404}, + {'result': {**item_info, 'extended': {'extra': {'tags': tags, 'attributes': attributes}}}}, + ] + httpx_mock.add_response( + url=AppConfig.Connections.url_portal + f'/v1/data/manifest/{attri_template_uid}', + method='GET', + json={'result': {'id': attri_template_uid, 'name': attri_template_name}}, + ) + + mocker.patch( + 'app.services.file_manager.file_metadata.file_metadata_client.FileMetaClient.save_file_metadata', + return_value=None, + ) + + file_meta_client = FileMetaClient('zone', 'project_code/object_path', 'general', 'attr', 'tag') + assert file_meta_client.project_code == 'project_code' + assert file_meta_client.object_path == 'object_path' + + item_info, res_attributes, tags = file_meta_client.download_file_metadata() + assert item_info == item_info + assert res_attributes == {attri_template_name: attributes.get(attri_template_uid)} + assert tags == tags + assert search_mock.call_count == 2 + + +def test_metadata_download_fail_when_file_doesnot_exist(mocker, capfd): + mocker.patch( + 'app.services.user_authentication.token_manager.SrvTokenManager.decode_access_token', + return_value=decoded_token(), + ) + + search_mock = mocker.patch( + 'app.services.file_manager.file_metadata.file_metadata_client.search_item', + ) + search_mock.side_effect = [{'result': {}, 'code': 404}, {'result': {}, 'code': 404}] + + file_meta_client = FileMetaClient('zone', 'project_code/object_path', 'general', 'attr', 'tag') + + try: + file_meta_client.download_file_metadata() + except SystemExit: + assert search_mock.call_count == 2 + out, _ = capfd.readouterr() + + expect = 'Cannot find item project_code/object_path at zone.\n' + assert out == expect + else: + AssertionError('SystemExit not raised') diff --git a/tests/app/utils/test_aggregated.py b/tests/app/utils/test_aggregated.py index 34a02646..00ec31e5 100644 --- a/tests/app/utils/test_aggregated.py +++ b/tests/app/utils/test_aggregated.py @@ -7,6 +7,7 @@ from app.configs.app_config import AppConfig from app.utils.aggregated import check_item_duplication from app.utils.aggregated import search_item +from app.utils.aggregated import validate_folder_name from tests.conftest import decoded_token test_project_code = 'testproject' @@ -122,3 +123,9 @@ def test_check_duplicate_fail_with_error_code(httpx_mock, mocker, capsys): check_item_duplication(['test_path'], 0, 'test_project_code') out, _ = capsys.readouterr() assert out.rstrip() == '{"error": "internal server error"}' + + +@pytest.mark.parametrize('folder_name', ['/:?.\\*<>|”\'', ''.join(['1' for _ in range(101)])]) +def test_validate_folder_name(folder_name): + valid = validate_folder_name(folder_name) + assert valid is False From 8f8cecd524f92d36cdb4f302da7fee09d563c188 Mon Sep 17 00:00:00 2001 From: Color Zhan Date: Tue, 5 Mar 2024 15:29:54 -0500 Subject: [PATCH 7/8] PILOT-4745: fixup the move command fails when creating a new folder under project folder (#130) * fixup the move command fails when creating a new folder under project folder * bumpup the version --------- Co-authored-by: zhiren --- .../file_manager/file_move/file_move_client.py | 6 ++++++ pyproject.toml | 2 +- .../file_move/test_file_move_client.py | 15 +++++++++++++++ 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/app/services/file_manager/file_move/file_move_client.py b/app/services/file_manager/file_move/file_move_client.py index 41833891..17478598 100644 --- a/app/services/file_manager/file_move/file_move_client.py +++ b/app/services/file_manager/file_move/file_move_client.py @@ -60,6 +60,12 @@ def create_object_path_if_not_exist(self, folder_path: str) -> dict: """ path_list = folder_path.split('/') + # first get the root folder to check if it is name folder + # or project folder + root_item = search_item(self.project_code, self.zone, path_list[0]).get('result') + if root_item.get('type') == 'project_folder': + path_list[0] = '/'.join([root_item.get('parent_path'), path_list[0]]) + # first check every folder in path exist or not # the loop start with index 1 since we assume cli will not # create any name folder or project folder diff --git a/pyproject.toml b/pyproject.toml index 128e8386..e91b9d00 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "app" -version = "2.9.10" +version = "2.9.11" description = "This service is designed to support pilot platform" authors = ["Indoc Systems"] diff --git a/tests/app/services/file_manager/file_move/test_file_move_client.py b/tests/app/services/file_manager/file_move/test_file_move_client.py index e22776e6..59d8cab5 100644 --- a/tests/app/services/file_manager/file_move/test_file_move_client.py +++ b/tests/app/services/file_manager/file_move/test_file_move_client.py @@ -18,6 +18,11 @@ def test_file_move_success(mocker, httpx_mock): return_value=decoded_token(), ) + mocker.patch( + 'app.services.file_manager.file_move.file_move_client.FileMoveClient.create_object_path_if_not_exist', + return_value=[], + ) + httpx_mock.add_response( url=AppConfig.Connections.url_bff + f'/v1/{project_code}/files', method='PATCH', @@ -37,6 +42,11 @@ def test_file_move_error_with_permission_denied_403(mocker, httpx_mock, capfd): return_value=decoded_token(), ) + mocker.patch( + 'app.services.file_manager.file_move.file_move_client.FileMoveClient.create_object_path_if_not_exist', + return_value=[], + ) + httpx_mock.add_response( url=AppConfig.Connections.url_bff + f'/v1/{project_code}/files', method='PATCH', @@ -60,6 +70,11 @@ def test_file_move_error_with_wrong_input_422(mocker, httpx_mock, capfd): return_value=decoded_token(), ) + mocker.patch( + 'app.services.file_manager.file_move.file_move_client.FileMoveClient.create_object_path_if_not_exist', + return_value=[], + ) + httpx_mock.add_response( url=AppConfig.Connections.url_bff + f'/v1/{project_code}/files', method='PATCH', From 17d211a6bc8a1190ef2f1aba729ef9022f5018da Mon Sep 17 00:00:00 2001 From: zhiren Date: Thu, 7 Mar 2024 11:12:30 -0500 Subject: [PATCH 8/8] bumpup to next preminor --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index e91b9d00..f04e7c59 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "app" -version = "2.9.11" +version = "2.10.0a0" description = "This service is designed to support pilot platform" authors = ["Indoc Systems"]