From 43623a2b32716fcc7d5f0188a9c19e8f9310ff38 Mon Sep 17 00:00:00 2001 From: Daniel Date: Tue, 17 Oct 2023 15:51:48 -0400 Subject: [PATCH 01/10] PILOT-3962: Port over changes from 2.7.3 to 2.7.4 --- app/resources/custom_help.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/resources/custom_help.py b/app/resources/custom_help.py index 3b6412ef..1b708a11 100644 --- a/app/resources/custom_help.py +++ b/app/resources/custom_help.py @@ -6,7 +6,7 @@ class HelpPage: page = { 'update': { - 'version': '2.7.2', + 'version': '2.7.4', '1': 'Add new feature for folder merging', '2': 'Secure the config file', '3': 'Optimize logic, input and error message', diff --git a/pyproject.toml b/pyproject.toml index f2b2b44d..7bf376f9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "app" -version = "2.7.3" +version = "2.7.4" description = "This service is designed to support pilot platform" authors = ["Indoc Systems"] From 6af654f62e281c85f672a4d311f5d04bc2a8fcdf Mon Sep 17 00:00:00 2001 From: zhiren Date: Thu, 23 Nov 2023 12:34:29 -0500 Subject: [PATCH 02/10] prepare the release branch --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index eab325d4..ce7a52f1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "app" -version = "2.8.5" +version = "2.9.0" description = "This service is designed to support pilot platform" authors = ["Indoc Systems"] From f377927a29005e01e0f9dcb7fb2ddd8d572f5216 Mon Sep 17 00:00:00 2001 From: zhiren Date: Thu, 23 Nov 2023 15:03:19 -0500 Subject: [PATCH 03/10] use correct version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index ce7a52f1..eab325d4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "app" -version = "2.9.0" +version = "2.8.5" description = "This service is designed to support pilot platform" authors = ["Indoc Systems"] From c5f8a845d467c4480cb04322e5e20788618fcd73 Mon Sep 17 00:00:00 2001 From: zhiren Date: Mon, 27 Nov 2023 14:49:17 -0500 Subject: [PATCH 04/10] add new command metadata to download file metadata/tags/attributes --- app/commands/file.py | 66 ++++++++++++++++++++++++ app/resources/custom_help.py | 5 ++ app/services/output_manager/help_page.py | 6 +++ 3 files changed, 77 insertions(+) diff --git a/app/commands/file.py b/app/commands/file.py index c3bacfaf..f25add83 100644 --- a/app/commands/file.py +++ b/app/commands/file.py @@ -435,3 +435,69 @@ def file_download(**kwargs): for item in item_res: srv_download = SrvFileDownload(zone, interactive) srv_download.simple_download_file(output_path, [item]) + + +@click.command(name='metadata') +@click.argument('file_path', type=click.STRING, nargs=-1) +@click.option( + '-z', + '--zone', + default=AppConfig.Env.green_zone, + required=True, + help=file_help.file_help_page(file_help.FileHELP.FILE_META_Z), + show_default=False, +) +@click.option( + '-g', + '--general', + default=None, + required=True, + help=file_help.file_help_page(file_help.FileHELP.FILE_META_G), + show_default=True, +) +@click.option( + '-a', + '--attribute', + default=None, + required=True, + help=file_help.file_help_page(file_help.FileHELP.FILE_META_A), + show_default=True, +) +@click.option( + '-t', + '--tag', + default=None, + required=True, + help=file_help.file_help_page(file_help.FileHELP.FILE_META_T), + show_default=True, +) +# @require_valid_token() +@doc(file_help.file_help_page(file_help.FileHELP.FILE_META)) +def file_metadata_download(**kwargs): + ''' + Summary: + Download metadata of a file including general, attribute and tag. + ''' + + # file_path = kwargs.get('file_path') + zone = kwargs.get('zone') + general_location = kwargs.get('general') + attribute_location = kwargs.get('attribute') + tag_location = kwargs.get('tag') + + # user = UserConfig() + # Check zone and upload-message + zone = get_zone(zone) if zone else AppConfig.Env.green_zone.lower() + + # check if the manifest file exists + try: + _ = {os.path.exists(x) for x in [general_location, attribute_location, tag_location]} + # print(files_exist) + + # if os.path.exists(output_path): + # click.confirm( + # customized_error_msg(ECustomizedError.MANIFEST_OF_FOLDER_FILE_EXIST) % (output_path), abort=True + # ) + except Abort: + message_handler.SrvOutPutHandler.cancel_upload() + exit(1) diff --git a/app/resources/custom_help.py b/app/resources/custom_help.py index 65b6aec3..a3d3122f 100644 --- a/app/resources/custom_help.py +++ b/app/resources/custom_help.py @@ -54,6 +54,11 @@ class HelpPage: "The processed pipeline of your processed files. [only used with '--source' option]" ), 'FILE_UPLOAD_ZIP': 'Upload folder as a compressed zip file.', + 'FILE_META': 'Download metadata file of a given file in target zone.', + 'FILE_META_Z': 'Target Zone (i.e., core/greenroom)', + 'FILE_META_G': 'The location of general metadata file', + 'FILE_META_A': 'The location of attribute metadata file', + 'FILE_META_T': 'The location of tag metadata file', }, 'config': { 'SET_CONFIG': 'Chose config file and set for cli.', diff --git a/app/services/output_manager/help_page.py b/app/services/output_manager/help_page.py index d48d0e2b..18b0702e 100644 --- a/app/services/output_manager/help_page.py +++ b/app/services/output_manager/help_page.py @@ -73,6 +73,12 @@ class FileHELP(enum.Enum): FILE_UPLOAD_PIPELINE = 'FILE_UPLOAD_PIPELINE' FILE_UPLOAD_ZIP = 'FILE_UPLOAD_ZIP' + FILE_META = 'FILE_META' + FILE_META_Z = 'FILE_META_Z' + FILE_META_G = 'FILE_META_G' + FILE_META_A = 'FILE_META_A' + FILE_META_T = 'FILE_META_T' + def file_help_page(FileHELP: FileHELP): helps = help_msg.get('file', 'default file help') From 07289e1ae633bfdc1bfd871173cba9c1d0c77419 Mon Sep 17 00:00:00 2001 From: zhiren Date: Mon, 27 Nov 2023 16:20:00 -0500 Subject: [PATCH 05/10] add new command for downloading file metadata --- app/commands/entry_point.py | 2 + app/commands/file.py | 33 +++++---- app/resources/custom_error.py | 2 + .../file_manager/file_metadata/__init__.py | 3 + .../file_metadata/file_metadata_client.py | 72 +++++++++++++++++++ app/services/output_manager/error_handler.py | 3 + .../output_manager/message_handler.py | 8 +++ 7 files changed, 111 insertions(+), 12 deletions(-) create mode 100644 app/services/file_manager/file_metadata/__init__.py create mode 100644 app/services/file_manager/file_metadata/file_metadata_client.py diff --git a/app/commands/entry_point.py b/app/commands/entry_point.py index b7830398..34c6b511 100644 --- a/app/commands/entry_point.py +++ b/app/commands/entry_point.py @@ -20,6 +20,7 @@ from .file import file_download from .file import file_export_manifest from .file import file_list +from .file import file_metadata_download from .file import file_put from .file import file_resume @@ -72,6 +73,7 @@ def user_group(): file_group.add_command(file_list) file_group.add_command(file_download) file_group.add_command(file_resume) +file_group.add_command(file_metadata_download) project_group.add_command(project_list_all) user_group.add_command(login) user_group.add_command(logout) diff --git a/app/commands/file.py b/app/commands/file.py index f25add83..85836fbd 100644 --- a/app/commands/file.py +++ b/app/commands/file.py @@ -16,6 +16,7 @@ from app.services.file_manager.file_download.download_client import SrvFileDownload from app.services.file_manager.file_list import SrvFileList from app.services.file_manager.file_manifests import SrvFileManifests +from app.services.file_manager.file_metadata.file_metadata_client import FileMetaClient from app.services.file_manager.file_upload.file_upload import assemble_path from app.services.file_manager.file_upload.file_upload import resume_upload from app.services.file_manager.file_upload.file_upload import simple_upload @@ -438,7 +439,7 @@ def file_download(**kwargs): @click.command(name='metadata') -@click.argument('file_path', type=click.STRING, nargs=-1) +@click.argument('file_path', type=click.STRING) @click.option( '-z', '--zone', @@ -471,7 +472,7 @@ def file_download(**kwargs): help=file_help.file_help_page(file_help.FileHELP.FILE_META_T), show_default=True, ) -# @require_valid_token() +@require_valid_token() @doc(file_help.file_help_page(file_help.FileHELP.FILE_META)) def file_metadata_download(**kwargs): ''' @@ -479,7 +480,7 @@ def file_metadata_download(**kwargs): Download metadata of a file including general, attribute and tag. ''' - # file_path = kwargs.get('file_path') + file_path = kwargs.get('file_path') zone = kwargs.get('zone') general_location = kwargs.get('general') attribute_location = kwargs.get('attribute') @@ -489,15 +490,23 @@ def file_metadata_download(**kwargs): # Check zone and upload-message zone = get_zone(zone) if zone else AppConfig.Env.green_zone.lower() - # check if the manifest file exists + # check if the manifest file exists and ask user whether to overwrite try: - _ = {os.path.exists(x) for x in [general_location, attribute_location, tag_location]} - # print(files_exist) - - # if os.path.exists(output_path): - # click.confirm( - # customized_error_msg(ECustomizedError.MANIFEST_OF_FOLDER_FILE_EXIST) % (output_path), abort=True - # ) + duplicate_error = customized_error_msg(ECustomizedError.LOCAL_METADATA_FILE_EXISTS) + overwrite_check = False + for x in ['general', 'attribute', 'tag']: + if os.path.exists(kwargs.get(x)): + overwrite_check = True + duplicate_error = duplicate_error + f'\n - {x}: {kwargs.get(x)}' + + if overwrite_check: + duplicate_error = duplicate_error + '\nDo you want to overwrite the existing file?' + click.confirm(duplicate_error, abort=True) except Abort: - message_handler.SrvOutPutHandler.cancel_upload() + message_handler.SrvOutPutHandler.cancel_metadata_download() exit(1) + + file_meta_client = FileMetaClient(zone, file_path, general_location, attribute_location, tag_location) + file_meta_client.download_file_metadata() + + message_handler.SrvOutPutHandler.metadata_download_success() diff --git a/app/resources/custom_error.py b/app/resources/custom_error.py index 16a1c4f4..c6233054 100644 --- a/app/resources/custom_error.py +++ b/app/resources/custom_error.py @@ -93,6 +93,8 @@ class Error: 'INVALID_FOLDER': 'Provided folder does not exist', 'INVALID_NAMEFOLDER': 'User name folder is missing or provided user name folder does not exist', 'INVALID_DOWNLOAD': 'Invalid download, file/folder not exist or folder is empty: %s', + # file metadata related error + 'LOCAL_METADATA_FILE_EXISTS': 'Following metadata file already exists in the local directory: ', 'VERSION_NOT_EXIST': 'Version not available: %s', 'DATASET_NOT_EXIST': 'Dataset not found in your dataset list', 'DATASET_PERMISSION': 'You do not have permission to access this dataset', diff --git a/app/services/file_manager/file_metadata/__init__.py b/app/services/file_manager/file_metadata/__init__.py new file mode 100644 index 00000000..96b7c430 --- /dev/null +++ b/app/services/file_manager/file_metadata/__init__.py @@ -0,0 +1,3 @@ +# Copyright (C) 2022-2023 Indoc Systems +# +# Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/app/services/file_manager/file_metadata/file_metadata_client.py b/app/services/file_manager/file_metadata/file_metadata_client.py new file mode 100644 index 00000000..c7c9ced7 --- /dev/null +++ b/app/services/file_manager/file_metadata/file_metadata_client.py @@ -0,0 +1,72 @@ +# Copyright (C) 2022-2023 Indoc Systems +# +# Contact Indoc Systems for any questions regarding the use of this source code. + +import json +from typing import Union + +from app.utils.aggregated import search_item + + +class FileMetaClient: + """ + Summary: + A client for interacting with file metadata. currently support to download + file metadata from metadata service. + """ + + def __init__( + self, + zone: str, + file_path: str, + general_location: str, + attribute_location: str, + tag_location: str, + ) -> None: + """ + Summary: + Initialize file metadata client. + Parameters: + zone (str): zone. + file_path (str): file path. + general_location (str): local file location of general metadata eg. item_id, project_code. + attribute_location (str): local file location of attribute metadata. + tag_location (str): local file location of tag metadata. + """ + + self.zone = zone + self.file_path = file_path + self.project_code, self.object_path = self.file_path.split('/', 1) + + # location of metadata files + self.general_location = general_location + self.attribute_location = attribute_location + self.tag_location = tag_location + + def save_file_metadata(self, file_loc: str, metadata: Union[dict, list]) -> None: + """ + Summary: + Save file metadata to local file. + """ + + with open(file_loc, 'w') as f: + json.dump(metadata, f, indent=4) + + def download_file_metadata(self) -> None: + """ + Summary: + Download file metadata from metadata service, including. + """ + + project_code, object_path = self.file_path.split('/', 1) + item_res = search_item(project_code, self.zone, object_path, 'file').get('result', {}) + extra_info = item_res.pop('extended', {}).get('extra') + tags = extra_info.get('tags', []) + attributes = extra_info.get('attributes', []) + # remove the uuid of attribute template + attribute_detail = attributes.get(next(iter(attributes))) + + # save metadata into files + self.save_file_metadata(self.general_location, item_res) + self.save_file_metadata(self.attribute_location, attribute_detail) + self.save_file_metadata(self.tag_location, tags) diff --git a/app/services/output_manager/error_handler.py b/app/services/output_manager/error_handler.py index d855013f..4fa9586c 100644 --- a/app/services/output_manager/error_handler.py +++ b/app/services/output_manager/error_handler.py @@ -63,6 +63,9 @@ class ECustomizedError(enum.Enum): INVALID_NAMEFOLDER = 'INVALID_NAMEFOLDER' INVALID_DOWNLOAD = 'INVALID_DOWNLOAD' DUPLICATE_TAG_ERROR = 'DUPLICATE_TAG_ERROR' + # file metadata related error + LOCAL_METADATA_FILE_EXISTS = 'LOCAL_METADATA_FILE_EXISTS' + VERSION_NOT_EXIST = 'VERSION_NOT_EXIST' DATASET_NOT_EXIST = 'DATASET_NOT_EXIST' DATASET_PERMISSION = 'DATASET_PERMISSION' diff --git a/app/services/output_manager/message_handler.py b/app/services/output_manager/message_handler.py index e7fbfd24..990f0743 100644 --- a/app/services/output_manager/message_handler.py +++ b/app/services/output_manager/message_handler.py @@ -195,6 +195,14 @@ def start_uploading(filename): def cancel_upload(): logger.warning('Upload cancelled.') + @staticmethod + def cancel_metadata_download(): + logger.warning('Metadata download cancelled.') + + @staticmethod + def metadata_download_success(): + logger.succeed('Metadata download complete.') + @staticmethod def start_requests(): """e.g. start requests.""" From 3379ef654031dd4c9e2d97e92f071939fee5f752 Mon Sep 17 00:00:00 2001 From: zhiren Date: Mon, 27 Nov 2023 17:21:00 -0500 Subject: [PATCH 06/10] add test cases for metadata download --- .../file_metadata/file_metadata_client.py | 11 ++- attr | 3 + general | 8 ++ tag | 3 + tests/app/commands/test_entry_point.py | 2 + tests/app/commands/test_file.py | 95 +++++++++++++++++++ .../test_file_metadata_client.py | 31 ++++++ 7 files changed, 152 insertions(+), 1 deletion(-) create mode 100644 attr create mode 100644 general create mode 100644 tag create mode 100644 tests/app/services/file_manager/file_metadata/test_file_metadata_client.py diff --git a/app/services/file_manager/file_metadata/file_metadata_client.py b/app/services/file_manager/file_metadata/file_metadata_client.py index c7c9ced7..5dba76f5 100644 --- a/app/services/file_manager/file_metadata/file_metadata_client.py +++ b/app/services/file_manager/file_metadata/file_metadata_client.py @@ -3,6 +3,9 @@ # Contact Indoc Systems for any questions regarding the use of this source code. import json +from typing import Any +from typing import Dict +from typing import List from typing import Union from app.utils.aggregated import search_item @@ -52,10 +55,14 @@ def save_file_metadata(self, file_loc: str, metadata: Union[dict, list]) -> None with open(file_loc, 'w') as f: json.dump(metadata, f, indent=4) - def download_file_metadata(self) -> None: + def download_file_metadata(self) -> List[Dict[str, Any]]: """ Summary: Download file metadata from metadata service, including. + Returns: + item_res (Dict[str, Any]): general metadata of file. + attribute_detail (Dict[str, Any]): attribute metadata of file. + tags (List[str]): tags metadata of file. """ project_code, object_path = self.file_path.split('/', 1) @@ -70,3 +77,5 @@ def download_file_metadata(self) -> None: self.save_file_metadata(self.general_location, item_res) self.save_file_metadata(self.attribute_location, attribute_detail) self.save_file_metadata(self.tag_location, tags) + + return item_res, attribute_detail, tags diff --git a/attr b/attr new file mode 100644 index 00000000..b9fa3341 --- /dev/null +++ b/attr @@ -0,0 +1,3 @@ +{ + "attr_1": "value" +} diff --git a/general b/general new file mode 100644 index 00000000..94ff2bcc --- /dev/null +++ b/general @@ -0,0 +1,8 @@ +{ + "id": "test", + "parent_id": "test_parent", + "parent_path": "", + "name": "admin", + "zone": 0, + "status": "ACTIVE" +} diff --git a/tag b/tag new file mode 100644 index 00000000..c322c19f --- /dev/null +++ b/tag @@ -0,0 +1,3 @@ +[ + "test" +] diff --git a/tests/app/commands/test_entry_point.py b/tests/app/commands/test_entry_point.py index 2085de4b..21c50c11 100644 --- a/tests/app/commands/test_entry_point.py +++ b/tests/app/commands/test_entry_point.py @@ -11,6 +11,7 @@ from app.commands.file import file_download from app.commands.file import file_export_manifest from app.commands.file import file_list +from app.commands.file import file_metadata_download from app.commands.file import file_put from app.commands.file import file_resume from app.commands.project import project_list_all @@ -62,6 +63,7 @@ def test_file_commands(user_login_true): 'attribute-export': file_export_manifest, 'download': file_download, 'resume': file_resume, + 'metadata': file_metadata_download, } file_commands_object = entry_point.commands.get('file') file_commands_object.callback() diff --git a/tests/app/commands/test_file.py b/tests/app/commands/test_file.py index 0e01a49b..52fae3cb 100644 --- a/tests/app/commands/test_file.py +++ b/tests/app/commands/test_file.py @@ -6,6 +6,7 @@ import questionary from app.commands.file import file_list +from app.commands.file import file_metadata_download from app.commands.file import file_put from app.commands.file import file_resume from app.services.file_manager.file_upload.models import FileObject @@ -125,3 +126,97 @@ def test_empty_file_list_with_pagination(requests_mock, mocker, cli_runner): result = cli_runner.invoke(file_list, ['testproject/admin', '-z', 'greenroom']) outputs = result.output.split('\n') assert outputs[0] == ' ' + + +def test_download_file_metadata_file_duplicate_success(mocker, cli_runner): + mocker.patch( + 'app.services.user_authentication.token_manager.SrvTokenManager.decode_access_token', + return_value=decoded_token(), + ) + + donwload_metadata_mock = mocker.patch( + 'app.services.file_manager.file_metadata.file_metadata_client.FileMetaClient.download_file_metadata', + return_value=None, + ) + + general_loc = 'test_g.txt' + attribute_loc = 'test_a.txt' + tag_loc = 'test_t.txt' + # create a test file + runner = click.testing.CliRunner() + with runner.isolated_filesystem(): + # create all file to make duplicationn + with open(general_loc, 'w') as f: + f.write(general_loc) + with open(attribute_loc, 'w') as f: + f.write(attribute_loc) + with open(tag_loc, 'w') as f: + f.write(tag_loc) + + result = cli_runner.invoke( + file_metadata_download, + ['test08171/admin/test20231115_9/test.log', '-g', 'test_g.txt', '-a', 'test_a.txt', '-t', 'test_t.txt'], + input='y', + ) + + assert result.exit_code == 0 + + outputs = result.output + excepted_output = ( + customized_error_msg(ECustomizedError.LOCAL_METADATA_FILE_EXISTS) + + f'\n - general: {general_loc}' + + f'\n - attribute: {attribute_loc}' + + f'\n - tag: {tag_loc}\n' + + 'Do you want to overwrite the existing file? [y/N]: y\n' + + 'Metadata download complete.\n' + ) + assert outputs == excepted_output + + donwload_metadata_mock.assert_called_once() + + +def test_download_file_metadata_file_duplicate_abort(mocker, cli_runner): + mocker.patch( + 'app.services.user_authentication.token_manager.SrvTokenManager.decode_access_token', + return_value=decoded_token(), + ) + + donwload_metadata_mock = mocker.patch( + 'app.services.file_manager.file_metadata.file_metadata_client.FileMetaClient.download_file_metadata', + return_value=None, + ) + + general_loc = 'test_g.txt' + attribute_loc = 'test_a.txt' + tag_loc = 'test_t.txt' + # create a test file + runner = click.testing.CliRunner() + with runner.isolated_filesystem(): + # create all file to make duplicationn + with open(general_loc, 'w') as f: + f.write(general_loc) + with open(attribute_loc, 'w') as f: + f.write(attribute_loc) + with open(tag_loc, 'w') as f: + f.write(tag_loc) + + result = cli_runner.invoke( + file_metadata_download, + ['test08171/admin/test20231115_9/test.log', '-g', 'test_g.txt', '-a', 'test_a.txt', '-t', 'test_t.txt'], + input='n', + ) + + assert result.exit_code == 1 + + outputs = result.output + excepted_output = ( + customized_error_msg(ECustomizedError.LOCAL_METADATA_FILE_EXISTS) + + f'\n - general: {general_loc}' + + f'\n - attribute: {attribute_loc}' + + f'\n - tag: {tag_loc}\n' + + 'Do you want to overwrite the existing file? [y/N]: n\n' + + 'Metadata download cancelled.\n' + ) + assert outputs == excepted_output + + assert donwload_metadata_mock.call_count == 0 diff --git a/tests/app/services/file_manager/file_metadata/test_file_metadata_client.py b/tests/app/services/file_manager/file_metadata/test_file_metadata_client.py new file mode 100644 index 00000000..84dc6e70 --- /dev/null +++ b/tests/app/services/file_manager/file_metadata/test_file_metadata_client.py @@ -0,0 +1,31 @@ +# Copyright (C) 2022-2023 Indoc Systems +# +# Contact Indoc Systems for any questions regarding the use of this source code. + +from app.services.file_manager.file_metadata.file_metadata_client import FileMetaClient + + +def test_file_metadata_client_get_detail_success(mocker): + item_info = { + 'id': 'test', + 'parent_id': 'test_parent', + 'parent_path': '', + 'name': 'admin', + 'zone': 0, + 'status': 'ACTIVE', + } + tags = ['test'] + attributes = {'template_uid': {'attr_1': 'value'}} + mocker.patch( + 'app.services.file_manager.file_metadata.file_metadata_client.search_item', + return_value={'result': {**item_info, 'extended': {'extra': {'tags': tags, 'attributes': attributes}}}}, + ) + + file_meta_client = FileMetaClient('zone', 'project_code/object_path', 'general', 'attr', 'tag') + assert file_meta_client.project_code == 'project_code' + assert file_meta_client.object_path == 'object_path' + + item_info, attributes, tags = file_meta_client.download_file_metadata() + assert item_info == item_info + assert attributes == attributes + assert tags == tags From 0b61c30ad7be38d55eae03fa21c843cab5d63bf1 Mon Sep 17 00:00:00 2001 From: zhiren Date: Mon, 27 Nov 2023 17:23:44 -0500 Subject: [PATCH 07/10] remove the testing data --- attr | 3 --- general | 8 -------- tag | 3 --- .../file_metadata/test_file_metadata_client.py | 4 ++++ 4 files changed, 4 insertions(+), 14 deletions(-) delete mode 100644 attr delete mode 100644 general delete mode 100644 tag diff --git a/attr b/attr deleted file mode 100644 index b9fa3341..00000000 --- a/attr +++ /dev/null @@ -1,3 +0,0 @@ -{ - "attr_1": "value" -} diff --git a/general b/general deleted file mode 100644 index 94ff2bcc..00000000 --- a/general +++ /dev/null @@ -1,8 +0,0 @@ -{ - "id": "test", - "parent_id": "test_parent", - "parent_path": "", - "name": "admin", - "zone": 0, - "status": "ACTIVE" -} diff --git a/tag b/tag deleted file mode 100644 index c322c19f..00000000 --- a/tag +++ /dev/null @@ -1,3 +0,0 @@ -[ - "test" -] diff --git a/tests/app/services/file_manager/file_metadata/test_file_metadata_client.py b/tests/app/services/file_manager/file_metadata/test_file_metadata_client.py index 84dc6e70..38735f59 100644 --- a/tests/app/services/file_manager/file_metadata/test_file_metadata_client.py +++ b/tests/app/services/file_manager/file_metadata/test_file_metadata_client.py @@ -20,6 +20,10 @@ def test_file_metadata_client_get_detail_success(mocker): 'app.services.file_manager.file_metadata.file_metadata_client.search_item', return_value={'result': {**item_info, 'extended': {'extra': {'tags': tags, 'attributes': attributes}}}}, ) + mocker.patch( + 'app.services.file_manager.file_metadata.file_metadata_client.FileMetaClient.save_file_metadata', + return_value=None, + ) file_meta_client = FileMetaClient('zone', 'project_code/object_path', 'general', 'attr', 'tag') assert file_meta_client.project_code == 'project_code' From c3f91fd8b7061f87f37ee330b579431cc19ffd4d Mon Sep 17 00:00:00 2001 From: zhiren Date: Tue, 28 Nov 2023 11:22:35 -0500 Subject: [PATCH 08/10] rename the output file name --- app/commands/file.py | 25 ++------ .../file_metadata/file_metadata_client.py | 59 +++++++++++++++--- tests/app/commands/test_file.py | 62 +++++++++++-------- 3 files changed, 90 insertions(+), 56 deletions(-) diff --git a/app/commands/file.py b/app/commands/file.py index 85836fbd..3237e411 100644 --- a/app/commands/file.py +++ b/app/commands/file.py @@ -482,31 +482,14 @@ def file_metadata_download(**kwargs): file_path = kwargs.get('file_path') zone = kwargs.get('zone') - general_location = kwargs.get('general') - attribute_location = kwargs.get('attribute') - tag_location = kwargs.get('tag') + general_folder = kwargs.get('general').rstrip('/') + attribute_folder = kwargs.get('attribute').rstrip('/') + tag_folder = kwargs.get('tag').rstrip('/') # user = UserConfig() # Check zone and upload-message zone = get_zone(zone) if zone else AppConfig.Env.green_zone.lower() - - # check if the manifest file exists and ask user whether to overwrite - try: - duplicate_error = customized_error_msg(ECustomizedError.LOCAL_METADATA_FILE_EXISTS) - overwrite_check = False - for x in ['general', 'attribute', 'tag']: - if os.path.exists(kwargs.get(x)): - overwrite_check = True - duplicate_error = duplicate_error + f'\n - {x}: {kwargs.get(x)}' - - if overwrite_check: - duplicate_error = duplicate_error + '\nDo you want to overwrite the existing file?' - click.confirm(duplicate_error, abort=True) - except Abort: - message_handler.SrvOutPutHandler.cancel_metadata_download() - exit(1) - - file_meta_client = FileMetaClient(zone, file_path, general_location, attribute_location, tag_location) + file_meta_client = FileMetaClient(zone, file_path, general_folder, attribute_folder, tag_folder) file_meta_client.download_file_metadata() message_handler.SrvOutPutHandler.metadata_download_success() diff --git a/app/services/file_manager/file_metadata/file_metadata_client.py b/app/services/file_manager/file_metadata/file_metadata_client.py index 5dba76f5..b038322f 100644 --- a/app/services/file_manager/file_metadata/file_metadata_client.py +++ b/app/services/file_manager/file_metadata/file_metadata_client.py @@ -3,11 +3,23 @@ # Contact Indoc Systems for any questions regarding the use of this source code. import json +from os import makedirs +from os.path import basename +from os.path import dirname +from os.path import exists +from os.path import join +from sys import exit from typing import Any from typing import Dict from typing import List from typing import Union +import click +from click.exceptions import Abort + +import app.services.output_manager.message_handler as message_handler +from app.services.output_manager.error_handler import ECustomizedError +from app.services.output_manager.error_handler import customized_error_msg from app.utils.aggregated import search_item @@ -22,9 +34,9 @@ def __init__( self, zone: str, file_path: str, - general_location: str, - attribute_location: str, - tag_location: str, + general_folder: str, + attribute_folder: str, + tag_folder: str, ) -> None: """ Summary: @@ -32,19 +44,47 @@ def __init__( Parameters: zone (str): zone. file_path (str): file path. - general_location (str): local file location of general metadata eg. item_id, project_code. - attribute_location (str): local file location of attribute metadata. - tag_location (str): local file location of tag metadata. + general_folder (str): local folder of general metadata eg. item_id, project_code. + attribute_folder (str): local folder of attribute metadata. + tag_folder (str): local folder of tag metadata. """ self.zone = zone self.file_path = file_path self.project_code, self.object_path = self.file_path.split('/', 1) + # only get the name regardless of the extension + self.file_name = basename(self.object_path).rsplit('.', 1)[0] # location of metadata files - self.general_location = general_location - self.attribute_location = attribute_location - self.tag_location = tag_location + self.general_location = join(general_folder, f'{self.file_name}-general.json') + self.attribute_location = join(attribute_folder, f'{self.file_name}-attribute.json') + self.tag_location = join(tag_folder, f'{self.file_name}-tag.json') + self._check_duplication(self.general_location, self.attribute_location, self.tag_location) + + def _check_duplication(self, general_loc: str, attribute_loc: str, tag_loc: str) -> None: + """ + Summary: + Check if the metadata files already exist in location system + and ask user whether to overwrite. + """ + + file_dict = {'general': general_loc, 'attribute': attribute_loc, 'tag': tag_loc} + + # check if the manifest file exists and ask user whether to overwrite + try: + duplicate_error = customized_error_msg(ECustomizedError.LOCAL_METADATA_FILE_EXISTS) + overwrite_check = False + for metadata_name, location in file_dict.items(): + if exists(location): + overwrite_check = True + duplicate_error = duplicate_error + f'\n - {metadata_name}: {location}' + + if overwrite_check: + duplicate_error = duplicate_error + '\nDo you want to overwrite the existing file?' + click.confirm(duplicate_error, abort=True) + except Abort: + message_handler.SrvOutPutHandler.cancel_metadata_download() + exit(1) def save_file_metadata(self, file_loc: str, metadata: Union[dict, list]) -> None: """ @@ -52,6 +92,7 @@ def save_file_metadata(self, file_loc: str, metadata: Union[dict, list]) -> None Save file metadata to local file. """ + makedirs(dirname(file_loc), exist_ok=True) with open(file_loc, 'w') as f: json.dump(metadata, f, indent=4) diff --git a/tests/app/commands/test_file.py b/tests/app/commands/test_file.py index 52fae3cb..e270ee00 100644 --- a/tests/app/commands/test_file.py +++ b/tests/app/commands/test_file.py @@ -2,6 +2,9 @@ # # Contact Indoc Systems for any questions regarding the use of this source code. +from os import makedirs +from os.path import dirname + import click import questionary @@ -9,6 +12,7 @@ from app.commands.file import file_metadata_download from app.commands.file import file_put from app.commands.file import file_resume +from app.services.file_manager.file_metadata.file_metadata_client import FileMetaClient from app.services.file_manager.file_upload.models import FileObject from app.services.output_manager.error_handler import ECustomizedError from app.services.output_manager.error_handler import customized_error_msg @@ -139,23 +143,26 @@ def test_download_file_metadata_file_duplicate_success(mocker, cli_runner): return_value=None, ) - general_loc = 'test_g.txt' - attribute_loc = 'test_a.txt' - tag_loc = 'test_t.txt' + metadata_loc = './test' + file_path = 'project_code/admin/test.py' # create a test file runner = click.testing.CliRunner() with runner.isolated_filesystem(): + file_meta_client = FileMetaClient('zone', file_path, metadata_loc, metadata_loc, metadata_loc) # create all file to make duplicationn - with open(general_loc, 'w') as f: - f.write(general_loc) - with open(attribute_loc, 'w') as f: - f.write(attribute_loc) - with open(tag_loc, 'w') as f: - f.write(tag_loc) + makedirs(dirname(file_meta_client.general_location), exist_ok=True) + with open(file_meta_client.general_location, 'w') as f: + f.write(file_meta_client.general_location) + makedirs(dirname(file_meta_client.attribute_location), exist_ok=True) + with open(file_meta_client.attribute_location, 'w') as f: + f.write(file_meta_client.attribute_location) + makedirs(dirname(file_meta_client.tag_location), exist_ok=True) + with open(file_meta_client.tag_location, 'w') as f: + f.write(file_meta_client.tag_location) result = cli_runner.invoke( file_metadata_download, - ['test08171/admin/test20231115_9/test.log', '-g', 'test_g.txt', '-a', 'test_a.txt', '-t', 'test_t.txt'], + [file_path, '-g', metadata_loc, '-a', metadata_loc, '-t', metadata_loc], input='y', ) @@ -164,9 +171,9 @@ def test_download_file_metadata_file_duplicate_success(mocker, cli_runner): outputs = result.output excepted_output = ( customized_error_msg(ECustomizedError.LOCAL_METADATA_FILE_EXISTS) - + f'\n - general: {general_loc}' - + f'\n - attribute: {attribute_loc}' - + f'\n - tag: {tag_loc}\n' + + f'\n - general: {file_meta_client.general_location}' + + f'\n - attribute: {file_meta_client.attribute_location}' + + f'\n - tag: {file_meta_client.tag_location}\n' + 'Do you want to overwrite the existing file? [y/N]: y\n' + 'Metadata download complete.\n' ) @@ -186,23 +193,26 @@ def test_download_file_metadata_file_duplicate_abort(mocker, cli_runner): return_value=None, ) - general_loc = 'test_g.txt' - attribute_loc = 'test_a.txt' - tag_loc = 'test_t.txt' + metadata_loc = './test' + file_path = 'project_code/admin/test.py' # create a test file runner = click.testing.CliRunner() with runner.isolated_filesystem(): + file_meta_client = FileMetaClient('zone', file_path, metadata_loc, metadata_loc, metadata_loc) # create all file to make duplicationn - with open(general_loc, 'w') as f: - f.write(general_loc) - with open(attribute_loc, 'w') as f: - f.write(attribute_loc) - with open(tag_loc, 'w') as f: - f.write(tag_loc) + makedirs(dirname(file_meta_client.general_location), exist_ok=True) + with open(file_meta_client.general_location, 'w') as f: + f.write(file_meta_client.general_location) + makedirs(dirname(file_meta_client.attribute_location), exist_ok=True) + with open(file_meta_client.attribute_location, 'w') as f: + f.write(file_meta_client.attribute_location) + makedirs(dirname(file_meta_client.tag_location), exist_ok=True) + with open(file_meta_client.tag_location, 'w') as f: + f.write(file_meta_client.tag_location) result = cli_runner.invoke( file_metadata_download, - ['test08171/admin/test20231115_9/test.log', '-g', 'test_g.txt', '-a', 'test_a.txt', '-t', 'test_t.txt'], + [file_path, '-g', metadata_loc, '-a', metadata_loc, '-t', metadata_loc], input='n', ) @@ -211,9 +221,9 @@ def test_download_file_metadata_file_duplicate_abort(mocker, cli_runner): outputs = result.output excepted_output = ( customized_error_msg(ECustomizedError.LOCAL_METADATA_FILE_EXISTS) - + f'\n - general: {general_loc}' - + f'\n - attribute: {attribute_loc}' - + f'\n - tag: {tag_loc}\n' + + f'\n - general: {file_meta_client.general_location}' + + f'\n - attribute: {file_meta_client.attribute_location}' + + f'\n - tag: {file_meta_client.tag_location}\n' + 'Do you want to overwrite the existing file? [y/N]: n\n' + 'Metadata download cancelled.\n' ) From c3de687f39cee3c9fbb239a1022cb6b92a3f4aa5 Mon Sep 17 00:00:00 2001 From: zhiren Date: Wed, 29 Nov 2023 16:53:25 -0500 Subject: [PATCH 09/10] reformat the attribute file with the template name --- app/configs/app_config.py | 1 + app/configs/config.py | 4 ++++ .../file_metadata/file_metadata_client.py | 11 +++++---- app/utils/aggregated.py | 13 ++++++++++ .../test_file_metadata_client.py | 24 +++++++++++++++---- 5 files changed, 45 insertions(+), 8 deletions(-) diff --git a/app/configs/app_config.py b/app/configs/app_config.py index e3485e0c..79e4158f 100644 --- a/app/configs/app_config.py +++ b/app/configs/app_config.py @@ -45,3 +45,4 @@ class Connections: url_keycloak_token = f'{ConfigClass.url_keycloak}/token' url_bff = ConfigClass.url_bff url_base = ConfigClass.base_url + url_portal = ConfigClass.url_portal diff --git a/app/configs/config.py b/app/configs/config.py index 7b61c438..e4a36b4d 100644 --- a/app/configs/config.py +++ b/app/configs/config.py @@ -41,6 +41,10 @@ def base_url(self) -> str: def url_bff(self) -> str: return f'{self.base_url}/cli' + @computed_field + def url_portal(self) -> str: + return f'{self.base_url}/portal' + @computed_field def url_keycloak_realm(self) -> str: return f'https://iam.{self.domain}/realms/pilot' diff --git a/app/services/file_manager/file_metadata/file_metadata_client.py b/app/services/file_manager/file_metadata/file_metadata_client.py index b038322f..dff11954 100644 --- a/app/services/file_manager/file_metadata/file_metadata_client.py +++ b/app/services/file_manager/file_metadata/file_metadata_client.py @@ -20,6 +20,7 @@ import app.services.output_manager.message_handler as message_handler from app.services.output_manager.error_handler import ECustomizedError from app.services.output_manager.error_handler import customized_error_msg +from app.utils.aggregated import get_attribute_template_by_id from app.utils.aggregated import search_item @@ -111,12 +112,14 @@ def download_file_metadata(self) -> List[Dict[str, Any]]: extra_info = item_res.pop('extended', {}).get('extra') tags = extra_info.get('tags', []) attributes = extra_info.get('attributes', []) - # remove the uuid of attribute template - attribute_detail = attributes.get(next(iter(attributes))) + # use the uuid of attribute template to get template name + template_uuid = next(iter(attributes)) + attribute_name = get_attribute_template_by_id(template_uuid).get('name') + attribute_detail = attributes.get(template_uuid) # save metadata into files self.save_file_metadata(self.general_location, item_res) - self.save_file_metadata(self.attribute_location, attribute_detail) + self.save_file_metadata(self.attribute_location, {attribute_name: attribute_detail}) self.save_file_metadata(self.tag_location, tags) - return item_res, attribute_detail, tags + return item_res, {attribute_name: attribute_detail}, tags diff --git a/app/utils/aggregated.py b/app/utils/aggregated.py index a6d13cfe..a9052ece 100644 --- a/app/utils/aggregated.py +++ b/app/utils/aggregated.py @@ -6,6 +6,7 @@ import re import shutil from typing import Any +from typing import Dict from typing import List import httpx @@ -49,6 +50,18 @@ def search_item(project_code, zone, folder_relative_path, item_type, container_t return res.json() +@require_valid_token() +def get_attribute_template_by_id(template_id: str) -> Dict[str, Any]: + token = UserConfig().access_token + url = AppConfig.Connections.url_portal + f'/v1/data/manifest/{template_id}' + headers = {'Authorization': 'Bearer ' + token} + res = resilient_session().get(url, headers=headers) + if res.status_code != 200: + SrvErrorHandler.default_handle(res.text, True) + + return res.json().get('result', {}) + + @require_valid_token() def get_file_info_by_geid(geid: list): token = UserConfig().access_token diff --git a/tests/app/services/file_manager/file_metadata/test_file_metadata_client.py b/tests/app/services/file_manager/file_metadata/test_file_metadata_client.py index 38735f59..23ec570b 100644 --- a/tests/app/services/file_manager/file_metadata/test_file_metadata_client.py +++ b/tests/app/services/file_manager/file_metadata/test_file_metadata_client.py @@ -2,10 +2,12 @@ # # Contact Indoc Systems for any questions regarding the use of this source code. +from app.configs.app_config import AppConfig from app.services.file_manager.file_metadata.file_metadata_client import FileMetaClient +from tests.conftest import decoded_token -def test_file_metadata_client_get_detail_success(mocker): +def test_file_metadata_client_get_detail_success(mocker, httpx_mock): item_info = { 'id': 'test', 'parent_id': 'test_parent', @@ -15,11 +17,25 @@ def test_file_metadata_client_get_detail_success(mocker): 'status': 'ACTIVE', } tags = ['test'] - attributes = {'template_uid': {'attr_1': 'value'}} + attri_template_uid = 'template_uid' + attri_template_name = 'template_name' + attributes = {attri_template_uid: {'attr_1': 'value'}} + + mocker.patch( + 'app.services.user_authentication.token_manager.SrvTokenManager.decode_access_token', + return_value=decoded_token(), + ) + mocker.patch( 'app.services.file_manager.file_metadata.file_metadata_client.search_item', return_value={'result': {**item_info, 'extended': {'extra': {'tags': tags, 'attributes': attributes}}}}, ) + httpx_mock.add_response( + url=AppConfig.Connections.url_portal + f'/v1/data/manifest/{attri_template_uid}', + method='GET', + json={'result': {'id': attri_template_uid, 'name': attri_template_name}}, + ) + mocker.patch( 'app.services.file_manager.file_metadata.file_metadata_client.FileMetaClient.save_file_metadata', return_value=None, @@ -29,7 +45,7 @@ def test_file_metadata_client_get_detail_success(mocker): assert file_meta_client.project_code == 'project_code' assert file_meta_client.object_path == 'object_path' - item_info, attributes, tags = file_meta_client.download_file_metadata() + item_info, res_attributes, tags = file_meta_client.download_file_metadata() assert item_info == item_info - assert attributes == attributes + assert res_attributes == {attri_template_name: attributes.get(attri_template_uid)} assert tags == tags From 6c9a12dd09a13bfc8b024186ec577fac3c388130 Mon Sep 17 00:00:00 2001 From: zhiren Date: Mon, 4 Dec 2023 15:37:59 -0500 Subject: [PATCH 10/10] bumpup into next version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index eab325d4..ce7a52f1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "app" -version = "2.8.5" +version = "2.9.0" description = "This service is designed to support pilot platform" authors = ["Indoc Systems"]