diff --git a/app/commands/entry_point.py b/app/commands/entry_point.py index b7830398..34c6b511 100644 --- a/app/commands/entry_point.py +++ b/app/commands/entry_point.py @@ -20,6 +20,7 @@ from .file import file_download from .file import file_export_manifest from .file import file_list +from .file import file_metadata_download from .file import file_put from .file import file_resume @@ -72,6 +73,7 @@ def user_group(): file_group.add_command(file_list) file_group.add_command(file_download) file_group.add_command(file_resume) +file_group.add_command(file_metadata_download) project_group.add_command(project_list_all) user_group.add_command(login) user_group.add_command(logout) diff --git a/app/commands/file.py b/app/commands/file.py index c3bacfaf..3237e411 100644 --- a/app/commands/file.py +++ b/app/commands/file.py @@ -16,6 +16,7 @@ from app.services.file_manager.file_download.download_client import SrvFileDownload from app.services.file_manager.file_list import SrvFileList from app.services.file_manager.file_manifests import SrvFileManifests +from app.services.file_manager.file_metadata.file_metadata_client import FileMetaClient from app.services.file_manager.file_upload.file_upload import assemble_path from app.services.file_manager.file_upload.file_upload import resume_upload from app.services.file_manager.file_upload.file_upload import simple_upload @@ -435,3 +436,60 @@ def file_download(**kwargs): for item in item_res: srv_download = SrvFileDownload(zone, interactive) srv_download.simple_download_file(output_path, [item]) + + +@click.command(name='metadata') +@click.argument('file_path', type=click.STRING) +@click.option( + '-z', + '--zone', + default=AppConfig.Env.green_zone, + required=True, + help=file_help.file_help_page(file_help.FileHELP.FILE_META_Z), + show_default=False, +) +@click.option( + '-g', + '--general', + default=None, + required=True, + help=file_help.file_help_page(file_help.FileHELP.FILE_META_G), + show_default=True, +) +@click.option( + '-a', + '--attribute', + default=None, + required=True, + help=file_help.file_help_page(file_help.FileHELP.FILE_META_A), + show_default=True, +) +@click.option( + '-t', + '--tag', + default=None, + required=True, + help=file_help.file_help_page(file_help.FileHELP.FILE_META_T), + show_default=True, +) +@require_valid_token() +@doc(file_help.file_help_page(file_help.FileHELP.FILE_META)) +def file_metadata_download(**kwargs): + ''' + Summary: + Download metadata of a file including general, attribute and tag. + ''' + + file_path = kwargs.get('file_path') + zone = kwargs.get('zone') + general_folder = kwargs.get('general').rstrip('/') + attribute_folder = kwargs.get('attribute').rstrip('/') + tag_folder = kwargs.get('tag').rstrip('/') + + # user = UserConfig() + # Check zone and upload-message + zone = get_zone(zone) if zone else AppConfig.Env.green_zone.lower() + file_meta_client = FileMetaClient(zone, file_path, general_folder, attribute_folder, tag_folder) + file_meta_client.download_file_metadata() + + message_handler.SrvOutPutHandler.metadata_download_success() diff --git a/app/configs/app_config.py b/app/configs/app_config.py index e3485e0c..79e4158f 100644 --- a/app/configs/app_config.py +++ b/app/configs/app_config.py @@ -45,3 +45,4 @@ class Connections: url_keycloak_token = f'{ConfigClass.url_keycloak}/token' url_bff = ConfigClass.url_bff url_base = ConfigClass.base_url + url_portal = ConfigClass.url_portal diff --git a/app/configs/config.py b/app/configs/config.py index 7b61c438..e4a36b4d 100644 --- a/app/configs/config.py +++ b/app/configs/config.py @@ -41,6 +41,10 @@ def base_url(self) -> str: def url_bff(self) -> str: return f'{self.base_url}/cli' + @computed_field + def url_portal(self) -> str: + return f'{self.base_url}/portal' + @computed_field def url_keycloak_realm(self) -> str: return f'https://iam.{self.domain}/realms/pilot' diff --git a/app/resources/custom_error.py b/app/resources/custom_error.py index 16a1c4f4..c6233054 100644 --- a/app/resources/custom_error.py +++ b/app/resources/custom_error.py @@ -93,6 +93,8 @@ class Error: 'INVALID_FOLDER': 'Provided folder does not exist', 'INVALID_NAMEFOLDER': 'User name folder is missing or provided user name folder does not exist', 'INVALID_DOWNLOAD': 'Invalid download, file/folder not exist or folder is empty: %s', + # file metadata related error + 'LOCAL_METADATA_FILE_EXISTS': 'Following metadata file already exists in the local directory: ', 'VERSION_NOT_EXIST': 'Version not available: %s', 'DATASET_NOT_EXIST': 'Dataset not found in your dataset list', 'DATASET_PERMISSION': 'You do not have permission to access this dataset', diff --git a/app/resources/custom_help.py b/app/resources/custom_help.py index 65b6aec3..a3d3122f 100644 --- a/app/resources/custom_help.py +++ b/app/resources/custom_help.py @@ -54,6 +54,11 @@ class HelpPage: "The processed pipeline of your processed files. [only used with '--source' option]" ), 'FILE_UPLOAD_ZIP': 'Upload folder as a compressed zip file.', + 'FILE_META': 'Download metadata file of a given file in target zone.', + 'FILE_META_Z': 'Target Zone (i.e., core/greenroom)', + 'FILE_META_G': 'The location of general metadata file', + 'FILE_META_A': 'The location of attribute metadata file', + 'FILE_META_T': 'The location of tag metadata file', }, 'config': { 'SET_CONFIG': 'Chose config file and set for cli.', diff --git a/app/services/file_manager/file_metadata/__init__.py b/app/services/file_manager/file_metadata/__init__.py new file mode 100644 index 00000000..96b7c430 --- /dev/null +++ b/app/services/file_manager/file_metadata/__init__.py @@ -0,0 +1,3 @@ +# Copyright (C) 2022-2023 Indoc Systems +# +# Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/app/services/file_manager/file_metadata/file_metadata_client.py b/app/services/file_manager/file_metadata/file_metadata_client.py new file mode 100644 index 00000000..dff11954 --- /dev/null +++ b/app/services/file_manager/file_metadata/file_metadata_client.py @@ -0,0 +1,125 @@ +# Copyright (C) 2022-2023 Indoc Systems +# +# Contact Indoc Systems for any questions regarding the use of this source code. + +import json +from os import makedirs +from os.path import basename +from os.path import dirname +from os.path import exists +from os.path import join +from sys import exit +from typing import Any +from typing import Dict +from typing import List +from typing import Union + +import click +from click.exceptions import Abort + +import app.services.output_manager.message_handler as message_handler +from app.services.output_manager.error_handler import ECustomizedError +from app.services.output_manager.error_handler import customized_error_msg +from app.utils.aggregated import get_attribute_template_by_id +from app.utils.aggregated import search_item + + +class FileMetaClient: + """ + Summary: + A client for interacting with file metadata. currently support to download + file metadata from metadata service. + """ + + def __init__( + self, + zone: str, + file_path: str, + general_folder: str, + attribute_folder: str, + tag_folder: str, + ) -> None: + """ + Summary: + Initialize file metadata client. + Parameters: + zone (str): zone. + file_path (str): file path. + general_folder (str): local folder of general metadata eg. item_id, project_code. + attribute_folder (str): local folder of attribute metadata. + tag_folder (str): local folder of tag metadata. + """ + + self.zone = zone + self.file_path = file_path + self.project_code, self.object_path = self.file_path.split('/', 1) + # only get the name regardless of the extension + self.file_name = basename(self.object_path).rsplit('.', 1)[0] + + # location of metadata files + self.general_location = join(general_folder, f'{self.file_name}-general.json') + self.attribute_location = join(attribute_folder, f'{self.file_name}-attribute.json') + self.tag_location = join(tag_folder, f'{self.file_name}-tag.json') + self._check_duplication(self.general_location, self.attribute_location, self.tag_location) + + def _check_duplication(self, general_loc: str, attribute_loc: str, tag_loc: str) -> None: + """ + Summary: + Check if the metadata files already exist in location system + and ask user whether to overwrite. + """ + + file_dict = {'general': general_loc, 'attribute': attribute_loc, 'tag': tag_loc} + + # check if the manifest file exists and ask user whether to overwrite + try: + duplicate_error = customized_error_msg(ECustomizedError.LOCAL_METADATA_FILE_EXISTS) + overwrite_check = False + for metadata_name, location in file_dict.items(): + if exists(location): + overwrite_check = True + duplicate_error = duplicate_error + f'\n - {metadata_name}: {location}' + + if overwrite_check: + duplicate_error = duplicate_error + '\nDo you want to overwrite the existing file?' + click.confirm(duplicate_error, abort=True) + except Abort: + message_handler.SrvOutPutHandler.cancel_metadata_download() + exit(1) + + def save_file_metadata(self, file_loc: str, metadata: Union[dict, list]) -> None: + """ + Summary: + Save file metadata to local file. + """ + + makedirs(dirname(file_loc), exist_ok=True) + with open(file_loc, 'w') as f: + json.dump(metadata, f, indent=4) + + def download_file_metadata(self) -> List[Dict[str, Any]]: + """ + Summary: + Download file metadata from metadata service, including. + Returns: + item_res (Dict[str, Any]): general metadata of file. + attribute_detail (Dict[str, Any]): attribute metadata of file. + tags (List[str]): tags metadata of file. + """ + + project_code, object_path = self.file_path.split('/', 1) + item_res = search_item(project_code, self.zone, object_path, 'file').get('result', {}) + extra_info = item_res.pop('extended', {}).get('extra') + tags = extra_info.get('tags', []) + attributes = extra_info.get('attributes', []) + # use the uuid of attribute template to get template name + template_uuid = next(iter(attributes)) + attribute_name = get_attribute_template_by_id(template_uuid).get('name') + attribute_detail = attributes.get(template_uuid) + + # save metadata into files + self.save_file_metadata(self.general_location, item_res) + self.save_file_metadata(self.attribute_location, {attribute_name: attribute_detail}) + self.save_file_metadata(self.tag_location, tags) + + return item_res, {attribute_name: attribute_detail}, tags diff --git a/app/services/output_manager/error_handler.py b/app/services/output_manager/error_handler.py index d855013f..4fa9586c 100644 --- a/app/services/output_manager/error_handler.py +++ b/app/services/output_manager/error_handler.py @@ -63,6 +63,9 @@ class ECustomizedError(enum.Enum): INVALID_NAMEFOLDER = 'INVALID_NAMEFOLDER' INVALID_DOWNLOAD = 'INVALID_DOWNLOAD' DUPLICATE_TAG_ERROR = 'DUPLICATE_TAG_ERROR' + # file metadata related error + LOCAL_METADATA_FILE_EXISTS = 'LOCAL_METADATA_FILE_EXISTS' + VERSION_NOT_EXIST = 'VERSION_NOT_EXIST' DATASET_NOT_EXIST = 'DATASET_NOT_EXIST' DATASET_PERMISSION = 'DATASET_PERMISSION' diff --git a/app/services/output_manager/help_page.py b/app/services/output_manager/help_page.py index d48d0e2b..18b0702e 100644 --- a/app/services/output_manager/help_page.py +++ b/app/services/output_manager/help_page.py @@ -73,6 +73,12 @@ class FileHELP(enum.Enum): FILE_UPLOAD_PIPELINE = 'FILE_UPLOAD_PIPELINE' FILE_UPLOAD_ZIP = 'FILE_UPLOAD_ZIP' + FILE_META = 'FILE_META' + FILE_META_Z = 'FILE_META_Z' + FILE_META_G = 'FILE_META_G' + FILE_META_A = 'FILE_META_A' + FILE_META_T = 'FILE_META_T' + def file_help_page(FileHELP: FileHELP): helps = help_msg.get('file', 'default file help') diff --git a/app/services/output_manager/message_handler.py b/app/services/output_manager/message_handler.py index e7fbfd24..990f0743 100644 --- a/app/services/output_manager/message_handler.py +++ b/app/services/output_manager/message_handler.py @@ -195,6 +195,14 @@ def start_uploading(filename): def cancel_upload(): logger.warning('Upload cancelled.') + @staticmethod + def cancel_metadata_download(): + logger.warning('Metadata download cancelled.') + + @staticmethod + def metadata_download_success(): + logger.succeed('Metadata download complete.') + @staticmethod def start_requests(): """e.g. start requests.""" diff --git a/app/utils/aggregated.py b/app/utils/aggregated.py index a6d13cfe..a9052ece 100644 --- a/app/utils/aggregated.py +++ b/app/utils/aggregated.py @@ -6,6 +6,7 @@ import re import shutil from typing import Any +from typing import Dict from typing import List import httpx @@ -49,6 +50,18 @@ def search_item(project_code, zone, folder_relative_path, item_type, container_t return res.json() +@require_valid_token() +def get_attribute_template_by_id(template_id: str) -> Dict[str, Any]: + token = UserConfig().access_token + url = AppConfig.Connections.url_portal + f'/v1/data/manifest/{template_id}' + headers = {'Authorization': 'Bearer ' + token} + res = resilient_session().get(url, headers=headers) + if res.status_code != 200: + SrvErrorHandler.default_handle(res.text, True) + + return res.json().get('result', {}) + + @require_valid_token() def get_file_info_by_geid(geid: list): token = UserConfig().access_token diff --git a/pyproject.toml b/pyproject.toml index 47b1d022..ce7a52f1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "app" -version = "2.8.6" +version = "2.9.0" description = "This service is designed to support pilot platform" authors = ["Indoc Systems"] diff --git a/tests/app/commands/test_entry_point.py b/tests/app/commands/test_entry_point.py index 2085de4b..21c50c11 100644 --- a/tests/app/commands/test_entry_point.py +++ b/tests/app/commands/test_entry_point.py @@ -11,6 +11,7 @@ from app.commands.file import file_download from app.commands.file import file_export_manifest from app.commands.file import file_list +from app.commands.file import file_metadata_download from app.commands.file import file_put from app.commands.file import file_resume from app.commands.project import project_list_all @@ -62,6 +63,7 @@ def test_file_commands(user_login_true): 'attribute-export': file_export_manifest, 'download': file_download, 'resume': file_resume, + 'metadata': file_metadata_download, } file_commands_object = entry_point.commands.get('file') file_commands_object.callback() diff --git a/tests/app/commands/test_file.py b/tests/app/commands/test_file.py index 0e01a49b..e270ee00 100644 --- a/tests/app/commands/test_file.py +++ b/tests/app/commands/test_file.py @@ -2,12 +2,17 @@ # # Contact Indoc Systems for any questions regarding the use of this source code. +from os import makedirs +from os.path import dirname + import click import questionary from app.commands.file import file_list +from app.commands.file import file_metadata_download from app.commands.file import file_put from app.commands.file import file_resume +from app.services.file_manager.file_metadata.file_metadata_client import FileMetaClient from app.services.file_manager.file_upload.models import FileObject from app.services.output_manager.error_handler import ECustomizedError from app.services.output_manager.error_handler import customized_error_msg @@ -125,3 +130,103 @@ def test_empty_file_list_with_pagination(requests_mock, mocker, cli_runner): result = cli_runner.invoke(file_list, ['testproject/admin', '-z', 'greenroom']) outputs = result.output.split('\n') assert outputs[0] == ' ' + + +def test_download_file_metadata_file_duplicate_success(mocker, cli_runner): + mocker.patch( + 'app.services.user_authentication.token_manager.SrvTokenManager.decode_access_token', + return_value=decoded_token(), + ) + + donwload_metadata_mock = mocker.patch( + 'app.services.file_manager.file_metadata.file_metadata_client.FileMetaClient.download_file_metadata', + return_value=None, + ) + + metadata_loc = './test' + file_path = 'project_code/admin/test.py' + # create a test file + runner = click.testing.CliRunner() + with runner.isolated_filesystem(): + file_meta_client = FileMetaClient('zone', file_path, metadata_loc, metadata_loc, metadata_loc) + # create all file to make duplicationn + makedirs(dirname(file_meta_client.general_location), exist_ok=True) + with open(file_meta_client.general_location, 'w') as f: + f.write(file_meta_client.general_location) + makedirs(dirname(file_meta_client.attribute_location), exist_ok=True) + with open(file_meta_client.attribute_location, 'w') as f: + f.write(file_meta_client.attribute_location) + makedirs(dirname(file_meta_client.tag_location), exist_ok=True) + with open(file_meta_client.tag_location, 'w') as f: + f.write(file_meta_client.tag_location) + + result = cli_runner.invoke( + file_metadata_download, + [file_path, '-g', metadata_loc, '-a', metadata_loc, '-t', metadata_loc], + input='y', + ) + + assert result.exit_code == 0 + + outputs = result.output + excepted_output = ( + customized_error_msg(ECustomizedError.LOCAL_METADATA_FILE_EXISTS) + + f'\n - general: {file_meta_client.general_location}' + + f'\n - attribute: {file_meta_client.attribute_location}' + + f'\n - tag: {file_meta_client.tag_location}\n' + + 'Do you want to overwrite the existing file? [y/N]: y\n' + + 'Metadata download complete.\n' + ) + assert outputs == excepted_output + + donwload_metadata_mock.assert_called_once() + + +def test_download_file_metadata_file_duplicate_abort(mocker, cli_runner): + mocker.patch( + 'app.services.user_authentication.token_manager.SrvTokenManager.decode_access_token', + return_value=decoded_token(), + ) + + donwload_metadata_mock = mocker.patch( + 'app.services.file_manager.file_metadata.file_metadata_client.FileMetaClient.download_file_metadata', + return_value=None, + ) + + metadata_loc = './test' + file_path = 'project_code/admin/test.py' + # create a test file + runner = click.testing.CliRunner() + with runner.isolated_filesystem(): + file_meta_client = FileMetaClient('zone', file_path, metadata_loc, metadata_loc, metadata_loc) + # create all file to make duplicationn + makedirs(dirname(file_meta_client.general_location), exist_ok=True) + with open(file_meta_client.general_location, 'w') as f: + f.write(file_meta_client.general_location) + makedirs(dirname(file_meta_client.attribute_location), exist_ok=True) + with open(file_meta_client.attribute_location, 'w') as f: + f.write(file_meta_client.attribute_location) + makedirs(dirname(file_meta_client.tag_location), exist_ok=True) + with open(file_meta_client.tag_location, 'w') as f: + f.write(file_meta_client.tag_location) + + result = cli_runner.invoke( + file_metadata_download, + [file_path, '-g', metadata_loc, '-a', metadata_loc, '-t', metadata_loc], + input='n', + ) + + assert result.exit_code == 1 + + outputs = result.output + excepted_output = ( + customized_error_msg(ECustomizedError.LOCAL_METADATA_FILE_EXISTS) + + f'\n - general: {file_meta_client.general_location}' + + f'\n - attribute: {file_meta_client.attribute_location}' + + f'\n - tag: {file_meta_client.tag_location}\n' + + 'Do you want to overwrite the existing file? [y/N]: n\n' + + 'Metadata download cancelled.\n' + ) + assert outputs == excepted_output + + assert donwload_metadata_mock.call_count == 0 diff --git a/tests/app/services/file_manager/file_metadata/test_file_metadata_client.py b/tests/app/services/file_manager/file_metadata/test_file_metadata_client.py new file mode 100644 index 00000000..23ec570b --- /dev/null +++ b/tests/app/services/file_manager/file_metadata/test_file_metadata_client.py @@ -0,0 +1,51 @@ +# Copyright (C) 2022-2023 Indoc Systems +# +# Contact Indoc Systems for any questions regarding the use of this source code. + +from app.configs.app_config import AppConfig +from app.services.file_manager.file_metadata.file_metadata_client import FileMetaClient +from tests.conftest import decoded_token + + +def test_file_metadata_client_get_detail_success(mocker, httpx_mock): + item_info = { + 'id': 'test', + 'parent_id': 'test_parent', + 'parent_path': '', + 'name': 'admin', + 'zone': 0, + 'status': 'ACTIVE', + } + tags = ['test'] + attri_template_uid = 'template_uid' + attri_template_name = 'template_name' + attributes = {attri_template_uid: {'attr_1': 'value'}} + + mocker.patch( + 'app.services.user_authentication.token_manager.SrvTokenManager.decode_access_token', + return_value=decoded_token(), + ) + + mocker.patch( + 'app.services.file_manager.file_metadata.file_metadata_client.search_item', + return_value={'result': {**item_info, 'extended': {'extra': {'tags': tags, 'attributes': attributes}}}}, + ) + httpx_mock.add_response( + url=AppConfig.Connections.url_portal + f'/v1/data/manifest/{attri_template_uid}', + method='GET', + json={'result': {'id': attri_template_uid, 'name': attri_template_name}}, + ) + + mocker.patch( + 'app.services.file_manager.file_metadata.file_metadata_client.FileMetaClient.save_file_metadata', + return_value=None, + ) + + file_meta_client = FileMetaClient('zone', 'project_code/object_path', 'general', 'attr', 'tag') + assert file_meta_client.project_code == 'project_code' + assert file_meta_client.object_path == 'object_path' + + item_info, res_attributes, tags = file_meta_client.download_file_metadata() + assert item_info == item_info + assert res_attributes == {attri_template_name: attributes.get(attri_template_uid)} + assert tags == tags