From 12a43d53294ea9ae3de9dc5c153a9afcf29b6f1d Mon Sep 17 00:00:00 2001 From: zhiren Date: Wed, 13 Dec 2023 17:30:40 -0500 Subject: [PATCH 1/5] add the basic logic of file move/renaming command --- app/commands/entry_point.py | 2 + app/commands/file.py | 26 ++++++++ app/configs/config.py | 3 +- app/resources/custom_help.py | 2 + .../file_manager/file_move/__init__.py | 3 + .../file_move/file_move_client.py | 59 +++++++++++++++++++ app/services/output_manager/help_page.py | 3 + 7 files changed, 97 insertions(+), 1 deletion(-) create mode 100644 app/services/file_manager/file_move/__init__.py create mode 100644 app/services/file_manager/file_move/file_move_client.py diff --git a/app/commands/entry_point.py b/app/commands/entry_point.py index 34c6b511..1770ac0a 100644 --- a/app/commands/entry_point.py +++ b/app/commands/entry_point.py @@ -21,6 +21,7 @@ from .file import file_export_manifest from .file import file_list from .file import file_metadata_download +from .file import file_move from .file import file_put from .file import file_resume @@ -74,6 +75,7 @@ def user_group(): file_group.add_command(file_download) file_group.add_command(file_resume) file_group.add_command(file_metadata_download) +file_group.add_command(file_move) project_group.add_command(project_list_all) user_group.add_command(login) user_group.add_command(logout) diff --git a/app/commands/file.py b/app/commands/file.py index 2e0e2263..2ff23d97 100644 --- a/app/commands/file.py +++ b/app/commands/file.py @@ -16,6 +16,7 @@ from app.services.file_manager.file_list import SrvFileList from app.services.file_manager.file_manifests import SrvFileManifests from app.services.file_manager.file_metadata.file_metadata_client import FileMetaClient +from app.services.file_manager.file_move.file_move_client import FileMoveClient from app.services.file_manager.file_upload.file_upload import assemble_path from app.services.file_manager.file_upload.file_upload import resume_upload from app.services.file_manager.file_upload.file_upload import simple_upload @@ -501,3 +502,28 @@ def file_metadata_download(**kwargs): file_meta_client.download_file_metadata() message_handler.SrvOutPutHandler.metadata_download_success() + + +@click.command(name='move') +@click.argument('project_code', type=click.STRING) +@click.argument('src_item_path', type=click.STRING) +@click.argument('dest_item_path', type=click.STRING) +@click.option( + '-z', + '--zone', + default=AppConfig.Env.green_zone, + required=True, + help=file_help.file_help_page(file_help.FileHELP.FILE_MOVE_Z), + show_default=False, +) +@require_valid_token() +@doc(file_help.file_help_page(file_help.FileHELP.FILE_MOVE)) +def file_move(**kwargs): + project_code = kwargs.get('project_code') + src_item_path = kwargs.get('src_item_path') + dest_item_path = kwargs.get('dest_item_path') + zone = kwargs.get('zone') + + zone = get_zone(zone) if zone else AppConfig.Env.green_zone.lower() + file_meta_client = FileMoveClient(zone, project_code, src_item_path, dest_item_path) + file_meta_client.move_file() diff --git a/app/configs/config.py b/app/configs/config.py index 04bbf98c..065a10bc 100644 --- a/app/configs/config.py +++ b/app/configs/config.py @@ -36,7 +36,8 @@ def base_url(self) -> str: @computed_field def url_bff(self) -> str: - return f'{self.base_url}/cli' + # return f'{self.base_url}/cli' + return 'http://localhost:5080' @computed_field def url_portal(self) -> str: diff --git a/app/resources/custom_help.py b/app/resources/custom_help.py index 03407b8f..28413446 100644 --- a/app/resources/custom_help.py +++ b/app/resources/custom_help.py @@ -60,6 +60,8 @@ class HelpPage: 'FILE_META_G': 'The location of general metadata file', 'FILE_META_A': 'The location of attribute metadata file', 'FILE_META_T': 'The location of tag metadata file', + 'FILE_MOVE': 'Move/Rename files/folders to a given Project path.', + 'FILE_MOVE_Z': 'Target Zone (i.e., core/greenroom).', }, 'config': { 'SET_CONFIG': 'Chose config file and set for cli.', diff --git a/app/services/file_manager/file_move/__init__.py b/app/services/file_manager/file_move/__init__.py new file mode 100644 index 00000000..96b7c430 --- /dev/null +++ b/app/services/file_manager/file_move/__init__.py @@ -0,0 +1,3 @@ +# Copyright (C) 2022-2023 Indoc Systems +# +# Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/app/services/file_manager/file_move/file_move_client.py b/app/services/file_manager/file_move/file_move_client.py new file mode 100644 index 00000000..f13f7ab8 --- /dev/null +++ b/app/services/file_manager/file_move/file_move_client.py @@ -0,0 +1,59 @@ +# Copyright (C) 2022-2023 Indoc Systems +# +# Contact Indoc Systems for any questions regarding the use of this source code. + +from app.configs.app_config import AppConfig +from app.configs.user_config import UserConfig +from app.utils.aggregated import resilient_session + + +class FileMoveClient: + """ + Summary: + A client for interacting with file metadata. currently support to download + file metadata from metadata service. + """ + + def __init__( + self, + zone: str, + project_code: str, + src_item_path: str, + dest_item_path: str, + ) -> None: + """ + Summary: + Initialize file move client. + Parameters: + zone (str): zone. + project_code (str): project code. + src_item_path (str): source item path. + dest_item_path (str): destination item path. + """ + + self.zone = zone + self.project_code = project_code + self.src_item_path = src_item_path + self.dest_item_path = dest_item_path + + self.user = UserConfig() + + def move_file(self) -> None: + """ + Summary: + Move file. + """ + + try: + url = AppConfig.Connections.url_bff + f'/v1/{self.project_code}/files' + payload = { + 'src_item_path': self.src_item_path, + 'dest_item_path': self.dest_item_path, + 'zone': self.zone, + } + headers = {'Authorization': 'Bearer ' + self.user.access_token, 'Session-ID': self.user.session_id} + + response = resilient_session().patch(url, json=payload, headers=headers, timeout=None) + response.raise_for_status() + except Exception as e: + raise e diff --git a/app/services/output_manager/help_page.py b/app/services/output_manager/help_page.py index 18b0702e..9ce631f0 100644 --- a/app/services/output_manager/help_page.py +++ b/app/services/output_manager/help_page.py @@ -79,6 +79,9 @@ class FileHELP(enum.Enum): FILE_META_A = 'FILE_META_A' FILE_META_T = 'FILE_META_T' + FILE_MOVE = 'FILE_MOVE' + FILE_MOVE_Z = 'FILE_MOVE_Z' + def file_help_page(FileHELP: FileHELP): helps = help_msg.get('file', 'default file help') From 31fd9ddc29bcc49fa423aebb8c560c7892695f4c Mon Sep 17 00:00:00 2001 From: zhiren Date: Thu, 14 Dec 2023 16:03:36 -0500 Subject: [PATCH 2/5] add new command for file move --- app/configs/config.py | 3 +-- .../file_manager/file_move/file_move_client.py | 15 +++++++++++++-- app/services/output_manager/message_handler.py | 10 ++++++++++ 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/app/configs/config.py b/app/configs/config.py index 065a10bc..04bbf98c 100644 --- a/app/configs/config.py +++ b/app/configs/config.py @@ -36,8 +36,7 @@ def base_url(self) -> str: @computed_field def url_bff(self) -> str: - # return f'{self.base_url}/cli' - return 'http://localhost:5080' + return f'{self.base_url}/cli' @computed_field def url_portal(self) -> str: diff --git a/app/services/file_manager/file_move/file_move_client.py b/app/services/file_manager/file_move/file_move_client.py index f13f7ab8..cffe3334 100644 --- a/app/services/file_manager/file_move/file_move_client.py +++ b/app/services/file_manager/file_move/file_move_client.py @@ -2,6 +2,7 @@ # # Contact Indoc Systems for any questions regarding the use of this source code. +import app.services.output_manager.message_handler as message_handler from app.configs.app_config import AppConfig from app.configs.user_config import UserConfig from app.utils.aggregated import resilient_session @@ -55,5 +56,15 @@ def move_file(self) -> None: response = resilient_session().patch(url, json=payload, headers=headers, timeout=None) response.raise_for_status() - except Exception as e: - raise e + except Exception: + if response.status_code == 422: + error_message = '' + for x in response.json().get('detail'): + error_message += '\n' + x.get('msg') + else: + error_message = response.json().get('error_msg') + message_handler.SrvOutPutHandler.move_action_failed(self.src_item_path, self.dest_item_path, error_message) + exit(1) + + message_handler.SrvOutPutHandler.move_action_success(self.src_item_path, self.dest_item_path) + return response.json() diff --git a/app/services/output_manager/message_handler.py b/app/services/output_manager/message_handler.py index 990f0743..3b2e11bf 100644 --- a/app/services/output_manager/message_handler.py +++ b/app/services/output_manager/message_handler.py @@ -203,6 +203,16 @@ def cancel_metadata_download(): def metadata_download_success(): logger.succeed('Metadata download complete.') + @staticmethod + def move_action_success(src, dest): + """e.g. Move action succeed.""" + return logger.succeed(f'Successfully moved {src} to {dest}') + + @staticmethod + def move_action_failed(src, dest, error): + """e.g. Move action failed.""" + return logger.error(f'Failed to move {src} to {dest}: {error}') + @staticmethod def start_requests(): """e.g. start requests.""" From 6ff1dccf8bcb556a7ccb0ab7e40939e4c95b36fd Mon Sep 17 00:00:00 2001 From: zhiren Date: Thu, 14 Dec 2023 17:57:21 -0500 Subject: [PATCH 3/5] add the test cases for file move --- app/commands/file.py | 2 + .../file_move/file_move_client.py | 3 +- tests/app/commands/test_entry_point.py | 2 + tests/app/commands/test_file.py | 18 +++++ .../file_move/test_file_move_client.py | 73 +++++++++++++++++++ 5 files changed, 96 insertions(+), 2 deletions(-) create mode 100644 tests/app/services/file_manager/file_move/test_file_move_client.py diff --git a/app/commands/file.py b/app/commands/file.py index 2ff23d97..76ffcd2a 100644 --- a/app/commands/file.py +++ b/app/commands/file.py @@ -527,3 +527,5 @@ def file_move(**kwargs): zone = get_zone(zone) if zone else AppConfig.Env.green_zone.lower() file_meta_client = FileMoveClient(zone, project_code, src_item_path, dest_item_path) file_meta_client.move_file() + + message_handler.SrvOutPutHandler.move_action_success(src_item_path, dest_item_path) diff --git a/app/services/file_manager/file_move/file_move_client.py b/app/services/file_manager/file_move/file_move_client.py index cffe3334..06a860d8 100644 --- a/app/services/file_manager/file_move/file_move_client.py +++ b/app/services/file_manager/file_move/file_move_client.py @@ -66,5 +66,4 @@ def move_file(self) -> None: message_handler.SrvOutPutHandler.move_action_failed(self.src_item_path, self.dest_item_path, error_message) exit(1) - message_handler.SrvOutPutHandler.move_action_success(self.src_item_path, self.dest_item_path) - return response.json() + return response.json().get('result') diff --git a/tests/app/commands/test_entry_point.py b/tests/app/commands/test_entry_point.py index 21c50c11..37adc20d 100644 --- a/tests/app/commands/test_entry_point.py +++ b/tests/app/commands/test_entry_point.py @@ -12,6 +12,7 @@ from app.commands.file import file_export_manifest from app.commands.file import file_list from app.commands.file import file_metadata_download +from app.commands.file import file_move from app.commands.file import file_put from app.commands.file import file_resume from app.commands.project import project_list_all @@ -64,6 +65,7 @@ def test_file_commands(user_login_true): 'download': file_download, 'resume': file_resume, 'metadata': file_metadata_download, + 'move': file_move, } file_commands_object = entry_point.commands.get('file') file_commands_object.callback() diff --git a/tests/app/commands/test_file.py b/tests/app/commands/test_file.py index 172f0bc2..9ac78b45 100644 --- a/tests/app/commands/test_file.py +++ b/tests/app/commands/test_file.py @@ -13,6 +13,7 @@ from app.commands.file import file_download from app.commands.file import file_list from app.commands.file import file_metadata_download +from app.commands.file import file_move from app.commands.file import file_put from app.commands.file import file_resume from app.services.file_manager.file_metadata.file_metadata_client import FileMetaClient @@ -332,3 +333,20 @@ def test_download_file_metadata_file_duplicate_abort(mocker, cli_runner): assert outputs == excepted_output assert donwload_metadata_mock.call_count == 0 + + +def test_file_move_success(mocker, cli_runner): + mocker.patch( + 'app.services.user_authentication.token_manager.SrvTokenManager.decode_access_token', + return_value=decoded_token(), + ) + + file_move_mock = mocker.patch( + 'app.services.file_manager.file_move.file_move_client.FileMoveClient.move_file', + return_value=None, + ) + + result = cli_runner.invoke(file_move, ['test_project', 'src_item_path', 'dest_item_path']) + outputs = result.output.split('\n') + assert outputs[0] == 'Successfully moved src_item_path to dest_item_path' + file_move_mock.assert_called_once() diff --git a/tests/app/services/file_manager/file_move/test_file_move_client.py b/tests/app/services/file_manager/file_move/test_file_move_client.py new file mode 100644 index 00000000..678ad01f --- /dev/null +++ b/tests/app/services/file_manager/file_move/test_file_move_client.py @@ -0,0 +1,73 @@ +# Copyright (C) 2022-2023 Indoc Systems +# +# Contact Indoc Systems for any questions regarding the use of this source code. + +from app.configs.app_config import AppConfig +from app.services.file_manager.file_move.file_move_client import FileMoveClient +from tests.conftest import decoded_token + + +def test_file_move_success(mocker, httpx_mock): + project_code = 'test_code' + item_info = {'result': {'id': 'test_id', 'name': 'test_name'}} + + mocker.patch( + 'app.services.user_authentication.token_manager.SrvTokenManager.decode_access_token', + return_value=decoded_token(), + ) + + httpx_mock.add_response( + url=AppConfig.Connections.url_bff + f'/v1/{project_code}/files', + method='PATCH', + json={'result': item_info}, + ) + + file_move_client = FileMoveClient('zone', project_code, 'src_item_path', 'dest_item_path') + res = file_move_client.move_file() + assert res == item_info + + +def test_file_move_error_with_permission_denied_403(mocker, httpx_mock, capfd): + project_code = 'test_code' + + mocker.patch( + 'app.services.user_authentication.token_manager.SrvTokenManager.decode_access_token', + return_value=decoded_token(), + ) + + httpx_mock.add_response( + url=AppConfig.Connections.url_bff + f'/v1/{project_code}/files', + method='PATCH', + json={'result': {}, 'error_msg': 'error_msg'}, + status_code=403, + ) + + file_move_client = FileMoveClient('zone', project_code, 'src_item_path', 'dest_item_path') + try: + file_move_client.move_file() + except SystemExit: + out, _ = capfd.readouterr() + assert out == 'Failed to move src_item_path to dest_item_path: error_msg\n' + + +def test_file_move_error_with_wrong_input_422(mocker, httpx_mock, capfd): + project_code = 'test_code' + + mocker.patch( + 'app.services.user_authentication.token_manager.SrvTokenManager.decode_access_token', + return_value=decoded_token(), + ) + + httpx_mock.add_response( + url=AppConfig.Connections.url_bff + f'/v1/{project_code}/files', + method='PATCH', + json={'detail': [{'loc': ['body', 'src_item_path'], 'msg': 'error_msg', 'type': 'value_error'}]}, + status_code=422, + ) + + file_move_client = FileMoveClient('zone', project_code, 'src_item_path', 'dest_item_path') + try: + file_move_client.move_file() + except SystemExit: + out, _ = capfd.readouterr() + assert out == 'Failed to move src_item_path to dest_item_path: \nerror_msg\n' From 6092b4bb2051962d0b1fdebbe9cbc001e727a723 Mon Sep 17 00:00:00 2001 From: zhiren Date: Fri, 15 Dec 2023 16:44:49 -0500 Subject: [PATCH 4/5] add the response in try exception --- app/services/file_manager/file_move/file_move_client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/services/file_manager/file_move/file_move_client.py b/app/services/file_manager/file_move/file_move_client.py index 06a860d8..706d32c0 100644 --- a/app/services/file_manager/file_move/file_move_client.py +++ b/app/services/file_manager/file_move/file_move_client.py @@ -56,6 +56,8 @@ def move_file(self) -> None: response = resilient_session().patch(url, json=payload, headers=headers, timeout=None) response.raise_for_status() + + return response.json().get('result') except Exception: if response.status_code == 422: error_message = '' @@ -65,5 +67,3 @@ def move_file(self) -> None: error_message = response.json().get('error_msg') message_handler.SrvOutPutHandler.move_action_failed(self.src_item_path, self.dest_item_path, error_message) exit(1) - - return response.json().get('result') From dfe7d1161b7c0cbcc69e24ff5f210026b21bc25d Mon Sep 17 00:00:00 2001 From: zhiren Date: Thu, 21 Dec 2023 09:09:31 -0500 Subject: [PATCH 5/5] bumpup versions --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 5ec41f62..ca5c4d3b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "app" -version = "2.9.4" +version = "2.9.5" description = "This service is designed to support pilot platform" authors = ["Indoc Systems"]