From 0b9a178369d0bf069be5e655585b7e0fda378f5c Mon Sep 17 00:00:00 2001 From: Hugo Seixas Antunes Date: Thu, 30 Nov 2023 16:55:40 +0100 Subject: [PATCH 01/22] PILOT-4152: Update cli logic to fit dynamic chunk size when uploading (#112) - update GET /v1/files/chunks/presigned api to pass the chunk_size as parameter. - set chunk_size to upload_chunk in stream_upload --- .../file_manager/file_upload/models.py | 5 +++-- .../file_manager/file_upload/upload_client.py | 19 ++++++++++++------- .../file_upload/test_upload_client.py | 2 +- 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/app/services/file_manager/file_upload/models.py b/app/services/file_manager/file_upload/models.py index 8d0b481b..aaf36ffb 100644 --- a/app/services/file_manager/file_upload/models.py +++ b/app/services/file_manager/file_upload/models.py @@ -7,7 +7,8 @@ from os.path import basename from os.path import dirname from os.path import getsize -from typing import List +from typing import Any +from typing import Dict from typing import Tuple from tqdm import tqdm @@ -62,7 +63,7 @@ class FileObject: total_chunks: int # resumable info - uploaded_chunks: List[dict] + uploaded_chunks: Dict[str, Dict[str, Any]] # progress bar object progress_bar = None diff --git a/app/services/file_manager/file_upload/upload_client.py b/app/services/file_manager/file_upload/upload_client.py index bea34f29..0640f3e5 100644 --- a/app/services/file_manager/file_upload/upload_client.py +++ b/app/services/file_manager/file_upload/upload_client.py @@ -61,7 +61,7 @@ def __init__( self.user = UserConfig() self.operator = self.user.username self.upload_message = upload_message - self.chunk_size = AppConfig.Env.chunk_size # remove + self.chunk_size = AppConfig.Env.chunk_size self.base_url = { AppConfig.Env.green_zone: AppConfig.Connections.url_upload_greenroom, AppConfig.Env.core_zone: AppConfig.Connections.url_upload_core, @@ -111,7 +111,7 @@ def resume_upload(self, unfinished_file_objects: List[FileObject]) -> List[FileO Parameter: - unfinished_file_objects(List[FileObject]): the unfinished items that need to be resumed. return: - - list of FileObject: the infomation retrieved from backend. + - list of FileObject: the information retrieved from backend. - resumable_id(str): the unique identifier for multipart upload. - object_path(str): the path in the object storage. - local_path(str): the local path of file. @@ -121,6 +121,7 @@ def resume_upload(self, unfinished_file_objects: List[FileObject]) -> List[FileO headers = {'Authorization': 'Bearer ' + self.user.access_token, 'Session-ID': self.user.session_id} url = AppConfig.Connections.url_bff + f'/v1/project/{self.project_code}/files/resumable' rid_file_object_map = {x.resumable_id: x for x in unfinished_file_objects} + payload = { 'bucket': self.bucket, 'zone': self.zone, @@ -300,8 +301,11 @@ def stream_upload(self, file_object: FileObject, pool: ThreadPool) -> List[Apply # after all the chunks have been uploaded. chunk_result = [] while True: - chunk = f.read(self.chunk_size) - chunk_etag = file_object.uploaded_chunks.get(str(count + 1)) + chunk = file_object.uploaded_chunks.get(str(count + 1), {}) + chunk_etag = chunk.get('etag') + chunk_size = chunk.get('chunk_size', self.chunk_size) + + chunk = f.read(chunk_size) local_chunk_etag = hashlib.md5(chunk).hexdigest() if not chunk: break @@ -312,11 +316,11 @@ def stream_upload(self, file_object: FileObject, pool: ThreadPool) -> List[Apply if chunk_etag != local_chunk_etag: SrvErrorHandler.customized_handle(ECustomizedError.INVALID_CHUNK_UPLOAD, value=count + 1) raise INVALID_CHUNK_ETAG(count + 1) - file_object.update_progress(self.chunk_size) + file_object.update_progress(chunk_size) else: res = pool.apply_async( self.upload_chunk, - args=(file_object, count + 1, chunk, local_chunk_etag), + args=(file_object, count + 1, chunk, local_chunk_etag, chunk_size), ) chunk_result.append(res) @@ -326,7 +330,7 @@ def stream_upload(self, file_object: FileObject, pool: ThreadPool) -> List[Apply return chunk_result - def upload_chunk(self, file_object: FileObject, chunk_number: int, chunk: str, etag: str) -> None: + def upload_chunk(self, file_object: FileObject, chunk_number: int, chunk: str, etag: str, chunk_size: int) -> None: """ Summary: The function is to upload a chunk directly into minio storage. @@ -353,6 +357,7 @@ def upload_chunk(self, file_object: FileObject, chunk_number: int, chunk: str, e 'key': file_object.item_id, 'upload_id': file_object.resumable_id, 'chunk_number': chunk_number, + 'chunk_size': chunk_size, } headers = { 'Authorization': 'Bearer ' + self.user.access_token, diff --git a/tests/app/services/file_manager/file_upload/test_upload_client.py b/tests/app/services/file_manager/file_upload/test_upload_client.py index d1809d0b..53c3cd76 100644 --- a/tests/app/services/file_manager/file_upload/test_upload_client.py +++ b/tests/app/services/file_manager/file_upload/test_upload_client.py @@ -73,7 +73,7 @@ def test_chunk_upload(httpx_mock, mocker): mocker.patch('app.services.file_manager.file_upload.models.FileObject.generate_meta', return_value=(1, 1)) test_obj = FileObject('test', 'test', 'test', 'test', 'test') - res = upload_client.upload_chunk(test_obj, 0, b'1', 'test_etag') + res = upload_client.upload_chunk(test_obj, 0, b'1', 'test_etag', 10) assert test_obj.progress_bar.n == 1 assert res.status_code == 200 From f53e30a2e590e0178c074f8d3669475c4ef32d03 Mon Sep 17 00:00:00 2001 From: Color Zhan Date: Fri, 1 Dec 2023 16:00:36 -0500 Subject: [PATCH 02/22] bumpup version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index eab325d4..47b1d022 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "app" -version = "2.8.5" +version = "2.8.6" description = "This service is designed to support pilot platform" authors = ["Indoc Systems"] From 94da05f2fc42af3feb032e7980db5b5b6313ab8e Mon Sep 17 00:00:00 2001 From: Color Zhan Date: Mon, 4 Dec 2023 17:12:24 -0500 Subject: [PATCH 03/22] Pilot 4090: add new command to download file metadata to local (#113) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * PILOT-3962: Port over changes from 2.7.3 to 2.7.4 * prepare the release branch * use correct version * add new command metadata to download file metadata/tags/attributes * add new command for downloading file metadata * add test cases for metadata download * remove the testing data * rename the output file name * reformat the attribute file with the template name * bumpup into next version --------- Co-authored-by: Daniel Co-authored-by: zhiren Co-authored-by: Dušan Andrić --- app/commands/entry_point.py | 2 + app/commands/file.py | 58 ++++++++ app/configs/app_config.py | 1 + app/configs/config.py | 4 + app/resources/custom_error.py | 2 + app/resources/custom_help.py | 5 + .../file_manager/file_metadata/__init__.py | 3 + .../file_metadata/file_metadata_client.py | 125 ++++++++++++++++++ app/services/output_manager/error_handler.py | 3 + app/services/output_manager/help_page.py | 6 + .../output_manager/message_handler.py | 8 ++ app/utils/aggregated.py | 13 ++ pyproject.toml | 2 +- tests/app/commands/test_entry_point.py | 2 + tests/app/commands/test_file.py | 105 +++++++++++++++ .../test_file_metadata_client.py | 51 +++++++ 16 files changed, 389 insertions(+), 1 deletion(-) create mode 100644 app/services/file_manager/file_metadata/__init__.py create mode 100644 app/services/file_manager/file_metadata/file_metadata_client.py create mode 100644 tests/app/services/file_manager/file_metadata/test_file_metadata_client.py diff --git a/app/commands/entry_point.py b/app/commands/entry_point.py index b7830398..34c6b511 100644 --- a/app/commands/entry_point.py +++ b/app/commands/entry_point.py @@ -20,6 +20,7 @@ from .file import file_download from .file import file_export_manifest from .file import file_list +from .file import file_metadata_download from .file import file_put from .file import file_resume @@ -72,6 +73,7 @@ def user_group(): file_group.add_command(file_list) file_group.add_command(file_download) file_group.add_command(file_resume) +file_group.add_command(file_metadata_download) project_group.add_command(project_list_all) user_group.add_command(login) user_group.add_command(logout) diff --git a/app/commands/file.py b/app/commands/file.py index c3bacfaf..3237e411 100644 --- a/app/commands/file.py +++ b/app/commands/file.py @@ -16,6 +16,7 @@ from app.services.file_manager.file_download.download_client import SrvFileDownload from app.services.file_manager.file_list import SrvFileList from app.services.file_manager.file_manifests import SrvFileManifests +from app.services.file_manager.file_metadata.file_metadata_client import FileMetaClient from app.services.file_manager.file_upload.file_upload import assemble_path from app.services.file_manager.file_upload.file_upload import resume_upload from app.services.file_manager.file_upload.file_upload import simple_upload @@ -435,3 +436,60 @@ def file_download(**kwargs): for item in item_res: srv_download = SrvFileDownload(zone, interactive) srv_download.simple_download_file(output_path, [item]) + + +@click.command(name='metadata') +@click.argument('file_path', type=click.STRING) +@click.option( + '-z', + '--zone', + default=AppConfig.Env.green_zone, + required=True, + help=file_help.file_help_page(file_help.FileHELP.FILE_META_Z), + show_default=False, +) +@click.option( + '-g', + '--general', + default=None, + required=True, + help=file_help.file_help_page(file_help.FileHELP.FILE_META_G), + show_default=True, +) +@click.option( + '-a', + '--attribute', + default=None, + required=True, + help=file_help.file_help_page(file_help.FileHELP.FILE_META_A), + show_default=True, +) +@click.option( + '-t', + '--tag', + default=None, + required=True, + help=file_help.file_help_page(file_help.FileHELP.FILE_META_T), + show_default=True, +) +@require_valid_token() +@doc(file_help.file_help_page(file_help.FileHELP.FILE_META)) +def file_metadata_download(**kwargs): + ''' + Summary: + Download metadata of a file including general, attribute and tag. + ''' + + file_path = kwargs.get('file_path') + zone = kwargs.get('zone') + general_folder = kwargs.get('general').rstrip('/') + attribute_folder = kwargs.get('attribute').rstrip('/') + tag_folder = kwargs.get('tag').rstrip('/') + + # user = UserConfig() + # Check zone and upload-message + zone = get_zone(zone) if zone else AppConfig.Env.green_zone.lower() + file_meta_client = FileMetaClient(zone, file_path, general_folder, attribute_folder, tag_folder) + file_meta_client.download_file_metadata() + + message_handler.SrvOutPutHandler.metadata_download_success() diff --git a/app/configs/app_config.py b/app/configs/app_config.py index e3485e0c..79e4158f 100644 --- a/app/configs/app_config.py +++ b/app/configs/app_config.py @@ -45,3 +45,4 @@ class Connections: url_keycloak_token = f'{ConfigClass.url_keycloak}/token' url_bff = ConfigClass.url_bff url_base = ConfigClass.base_url + url_portal = ConfigClass.url_portal diff --git a/app/configs/config.py b/app/configs/config.py index 7b61c438..e4a36b4d 100644 --- a/app/configs/config.py +++ b/app/configs/config.py @@ -41,6 +41,10 @@ def base_url(self) -> str: def url_bff(self) -> str: return f'{self.base_url}/cli' + @computed_field + def url_portal(self) -> str: + return f'{self.base_url}/portal' + @computed_field def url_keycloak_realm(self) -> str: return f'https://iam.{self.domain}/realms/pilot' diff --git a/app/resources/custom_error.py b/app/resources/custom_error.py index 16a1c4f4..c6233054 100644 --- a/app/resources/custom_error.py +++ b/app/resources/custom_error.py @@ -93,6 +93,8 @@ class Error: 'INVALID_FOLDER': 'Provided folder does not exist', 'INVALID_NAMEFOLDER': 'User name folder is missing or provided user name folder does not exist', 'INVALID_DOWNLOAD': 'Invalid download, file/folder not exist or folder is empty: %s', + # file metadata related error + 'LOCAL_METADATA_FILE_EXISTS': 'Following metadata file already exists in the local directory: ', 'VERSION_NOT_EXIST': 'Version not available: %s', 'DATASET_NOT_EXIST': 'Dataset not found in your dataset list', 'DATASET_PERMISSION': 'You do not have permission to access this dataset', diff --git a/app/resources/custom_help.py b/app/resources/custom_help.py index 65b6aec3..a3d3122f 100644 --- a/app/resources/custom_help.py +++ b/app/resources/custom_help.py @@ -54,6 +54,11 @@ class HelpPage: "The processed pipeline of your processed files. [only used with '--source' option]" ), 'FILE_UPLOAD_ZIP': 'Upload folder as a compressed zip file.', + 'FILE_META': 'Download metadata file of a given file in target zone.', + 'FILE_META_Z': 'Target Zone (i.e., core/greenroom)', + 'FILE_META_G': 'The location of general metadata file', + 'FILE_META_A': 'The location of attribute metadata file', + 'FILE_META_T': 'The location of tag metadata file', }, 'config': { 'SET_CONFIG': 'Chose config file and set for cli.', diff --git a/app/services/file_manager/file_metadata/__init__.py b/app/services/file_manager/file_metadata/__init__.py new file mode 100644 index 00000000..96b7c430 --- /dev/null +++ b/app/services/file_manager/file_metadata/__init__.py @@ -0,0 +1,3 @@ +# Copyright (C) 2022-2023 Indoc Systems +# +# Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/app/services/file_manager/file_metadata/file_metadata_client.py b/app/services/file_manager/file_metadata/file_metadata_client.py new file mode 100644 index 00000000..dff11954 --- /dev/null +++ b/app/services/file_manager/file_metadata/file_metadata_client.py @@ -0,0 +1,125 @@ +# Copyright (C) 2022-2023 Indoc Systems +# +# Contact Indoc Systems for any questions regarding the use of this source code. + +import json +from os import makedirs +from os.path import basename +from os.path import dirname +from os.path import exists +from os.path import join +from sys import exit +from typing import Any +from typing import Dict +from typing import List +from typing import Union + +import click +from click.exceptions import Abort + +import app.services.output_manager.message_handler as message_handler +from app.services.output_manager.error_handler import ECustomizedError +from app.services.output_manager.error_handler import customized_error_msg +from app.utils.aggregated import get_attribute_template_by_id +from app.utils.aggregated import search_item + + +class FileMetaClient: + """ + Summary: + A client for interacting with file metadata. currently support to download + file metadata from metadata service. + """ + + def __init__( + self, + zone: str, + file_path: str, + general_folder: str, + attribute_folder: str, + tag_folder: str, + ) -> None: + """ + Summary: + Initialize file metadata client. + Parameters: + zone (str): zone. + file_path (str): file path. + general_folder (str): local folder of general metadata eg. item_id, project_code. + attribute_folder (str): local folder of attribute metadata. + tag_folder (str): local folder of tag metadata. + """ + + self.zone = zone + self.file_path = file_path + self.project_code, self.object_path = self.file_path.split('/', 1) + # only get the name regardless of the extension + self.file_name = basename(self.object_path).rsplit('.', 1)[0] + + # location of metadata files + self.general_location = join(general_folder, f'{self.file_name}-general.json') + self.attribute_location = join(attribute_folder, f'{self.file_name}-attribute.json') + self.tag_location = join(tag_folder, f'{self.file_name}-tag.json') + self._check_duplication(self.general_location, self.attribute_location, self.tag_location) + + def _check_duplication(self, general_loc: str, attribute_loc: str, tag_loc: str) -> None: + """ + Summary: + Check if the metadata files already exist in location system + and ask user whether to overwrite. + """ + + file_dict = {'general': general_loc, 'attribute': attribute_loc, 'tag': tag_loc} + + # check if the manifest file exists and ask user whether to overwrite + try: + duplicate_error = customized_error_msg(ECustomizedError.LOCAL_METADATA_FILE_EXISTS) + overwrite_check = False + for metadata_name, location in file_dict.items(): + if exists(location): + overwrite_check = True + duplicate_error = duplicate_error + f'\n - {metadata_name}: {location}' + + if overwrite_check: + duplicate_error = duplicate_error + '\nDo you want to overwrite the existing file?' + click.confirm(duplicate_error, abort=True) + except Abort: + message_handler.SrvOutPutHandler.cancel_metadata_download() + exit(1) + + def save_file_metadata(self, file_loc: str, metadata: Union[dict, list]) -> None: + """ + Summary: + Save file metadata to local file. + """ + + makedirs(dirname(file_loc), exist_ok=True) + with open(file_loc, 'w') as f: + json.dump(metadata, f, indent=4) + + def download_file_metadata(self) -> List[Dict[str, Any]]: + """ + Summary: + Download file metadata from metadata service, including. + Returns: + item_res (Dict[str, Any]): general metadata of file. + attribute_detail (Dict[str, Any]): attribute metadata of file. + tags (List[str]): tags metadata of file. + """ + + project_code, object_path = self.file_path.split('/', 1) + item_res = search_item(project_code, self.zone, object_path, 'file').get('result', {}) + extra_info = item_res.pop('extended', {}).get('extra') + tags = extra_info.get('tags', []) + attributes = extra_info.get('attributes', []) + # use the uuid of attribute template to get template name + template_uuid = next(iter(attributes)) + attribute_name = get_attribute_template_by_id(template_uuid).get('name') + attribute_detail = attributes.get(template_uuid) + + # save metadata into files + self.save_file_metadata(self.general_location, item_res) + self.save_file_metadata(self.attribute_location, {attribute_name: attribute_detail}) + self.save_file_metadata(self.tag_location, tags) + + return item_res, {attribute_name: attribute_detail}, tags diff --git a/app/services/output_manager/error_handler.py b/app/services/output_manager/error_handler.py index d855013f..4fa9586c 100644 --- a/app/services/output_manager/error_handler.py +++ b/app/services/output_manager/error_handler.py @@ -63,6 +63,9 @@ class ECustomizedError(enum.Enum): INVALID_NAMEFOLDER = 'INVALID_NAMEFOLDER' INVALID_DOWNLOAD = 'INVALID_DOWNLOAD' DUPLICATE_TAG_ERROR = 'DUPLICATE_TAG_ERROR' + # file metadata related error + LOCAL_METADATA_FILE_EXISTS = 'LOCAL_METADATA_FILE_EXISTS' + VERSION_NOT_EXIST = 'VERSION_NOT_EXIST' DATASET_NOT_EXIST = 'DATASET_NOT_EXIST' DATASET_PERMISSION = 'DATASET_PERMISSION' diff --git a/app/services/output_manager/help_page.py b/app/services/output_manager/help_page.py index d48d0e2b..18b0702e 100644 --- a/app/services/output_manager/help_page.py +++ b/app/services/output_manager/help_page.py @@ -73,6 +73,12 @@ class FileHELP(enum.Enum): FILE_UPLOAD_PIPELINE = 'FILE_UPLOAD_PIPELINE' FILE_UPLOAD_ZIP = 'FILE_UPLOAD_ZIP' + FILE_META = 'FILE_META' + FILE_META_Z = 'FILE_META_Z' + FILE_META_G = 'FILE_META_G' + FILE_META_A = 'FILE_META_A' + FILE_META_T = 'FILE_META_T' + def file_help_page(FileHELP: FileHELP): helps = help_msg.get('file', 'default file help') diff --git a/app/services/output_manager/message_handler.py b/app/services/output_manager/message_handler.py index e7fbfd24..990f0743 100644 --- a/app/services/output_manager/message_handler.py +++ b/app/services/output_manager/message_handler.py @@ -195,6 +195,14 @@ def start_uploading(filename): def cancel_upload(): logger.warning('Upload cancelled.') + @staticmethod + def cancel_metadata_download(): + logger.warning('Metadata download cancelled.') + + @staticmethod + def metadata_download_success(): + logger.succeed('Metadata download complete.') + @staticmethod def start_requests(): """e.g. start requests.""" diff --git a/app/utils/aggregated.py b/app/utils/aggregated.py index a6d13cfe..a9052ece 100644 --- a/app/utils/aggregated.py +++ b/app/utils/aggregated.py @@ -6,6 +6,7 @@ import re import shutil from typing import Any +from typing import Dict from typing import List import httpx @@ -49,6 +50,18 @@ def search_item(project_code, zone, folder_relative_path, item_type, container_t return res.json() +@require_valid_token() +def get_attribute_template_by_id(template_id: str) -> Dict[str, Any]: + token = UserConfig().access_token + url = AppConfig.Connections.url_portal + f'/v1/data/manifest/{template_id}' + headers = {'Authorization': 'Bearer ' + token} + res = resilient_session().get(url, headers=headers) + if res.status_code != 200: + SrvErrorHandler.default_handle(res.text, True) + + return res.json().get('result', {}) + + @require_valid_token() def get_file_info_by_geid(geid: list): token = UserConfig().access_token diff --git a/pyproject.toml b/pyproject.toml index 47b1d022..ce7a52f1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "app" -version = "2.8.6" +version = "2.9.0" description = "This service is designed to support pilot platform" authors = ["Indoc Systems"] diff --git a/tests/app/commands/test_entry_point.py b/tests/app/commands/test_entry_point.py index 2085de4b..21c50c11 100644 --- a/tests/app/commands/test_entry_point.py +++ b/tests/app/commands/test_entry_point.py @@ -11,6 +11,7 @@ from app.commands.file import file_download from app.commands.file import file_export_manifest from app.commands.file import file_list +from app.commands.file import file_metadata_download from app.commands.file import file_put from app.commands.file import file_resume from app.commands.project import project_list_all @@ -62,6 +63,7 @@ def test_file_commands(user_login_true): 'attribute-export': file_export_manifest, 'download': file_download, 'resume': file_resume, + 'metadata': file_metadata_download, } file_commands_object = entry_point.commands.get('file') file_commands_object.callback() diff --git a/tests/app/commands/test_file.py b/tests/app/commands/test_file.py index 0e01a49b..e270ee00 100644 --- a/tests/app/commands/test_file.py +++ b/tests/app/commands/test_file.py @@ -2,12 +2,17 @@ # # Contact Indoc Systems for any questions regarding the use of this source code. +from os import makedirs +from os.path import dirname + import click import questionary from app.commands.file import file_list +from app.commands.file import file_metadata_download from app.commands.file import file_put from app.commands.file import file_resume +from app.services.file_manager.file_metadata.file_metadata_client import FileMetaClient from app.services.file_manager.file_upload.models import FileObject from app.services.output_manager.error_handler import ECustomizedError from app.services.output_manager.error_handler import customized_error_msg @@ -125,3 +130,103 @@ def test_empty_file_list_with_pagination(requests_mock, mocker, cli_runner): result = cli_runner.invoke(file_list, ['testproject/admin', '-z', 'greenroom']) outputs = result.output.split('\n') assert outputs[0] == ' ' + + +def test_download_file_metadata_file_duplicate_success(mocker, cli_runner): + mocker.patch( + 'app.services.user_authentication.token_manager.SrvTokenManager.decode_access_token', + return_value=decoded_token(), + ) + + donwload_metadata_mock = mocker.patch( + 'app.services.file_manager.file_metadata.file_metadata_client.FileMetaClient.download_file_metadata', + return_value=None, + ) + + metadata_loc = './test' + file_path = 'project_code/admin/test.py' + # create a test file + runner = click.testing.CliRunner() + with runner.isolated_filesystem(): + file_meta_client = FileMetaClient('zone', file_path, metadata_loc, metadata_loc, metadata_loc) + # create all file to make duplicationn + makedirs(dirname(file_meta_client.general_location), exist_ok=True) + with open(file_meta_client.general_location, 'w') as f: + f.write(file_meta_client.general_location) + makedirs(dirname(file_meta_client.attribute_location), exist_ok=True) + with open(file_meta_client.attribute_location, 'w') as f: + f.write(file_meta_client.attribute_location) + makedirs(dirname(file_meta_client.tag_location), exist_ok=True) + with open(file_meta_client.tag_location, 'w') as f: + f.write(file_meta_client.tag_location) + + result = cli_runner.invoke( + file_metadata_download, + [file_path, '-g', metadata_loc, '-a', metadata_loc, '-t', metadata_loc], + input='y', + ) + + assert result.exit_code == 0 + + outputs = result.output + excepted_output = ( + customized_error_msg(ECustomizedError.LOCAL_METADATA_FILE_EXISTS) + + f'\n - general: {file_meta_client.general_location}' + + f'\n - attribute: {file_meta_client.attribute_location}' + + f'\n - tag: {file_meta_client.tag_location}\n' + + 'Do you want to overwrite the existing file? [y/N]: y\n' + + 'Metadata download complete.\n' + ) + assert outputs == excepted_output + + donwload_metadata_mock.assert_called_once() + + +def test_download_file_metadata_file_duplicate_abort(mocker, cli_runner): + mocker.patch( + 'app.services.user_authentication.token_manager.SrvTokenManager.decode_access_token', + return_value=decoded_token(), + ) + + donwload_metadata_mock = mocker.patch( + 'app.services.file_manager.file_metadata.file_metadata_client.FileMetaClient.download_file_metadata', + return_value=None, + ) + + metadata_loc = './test' + file_path = 'project_code/admin/test.py' + # create a test file + runner = click.testing.CliRunner() + with runner.isolated_filesystem(): + file_meta_client = FileMetaClient('zone', file_path, metadata_loc, metadata_loc, metadata_loc) + # create all file to make duplicationn + makedirs(dirname(file_meta_client.general_location), exist_ok=True) + with open(file_meta_client.general_location, 'w') as f: + f.write(file_meta_client.general_location) + makedirs(dirname(file_meta_client.attribute_location), exist_ok=True) + with open(file_meta_client.attribute_location, 'w') as f: + f.write(file_meta_client.attribute_location) + makedirs(dirname(file_meta_client.tag_location), exist_ok=True) + with open(file_meta_client.tag_location, 'w') as f: + f.write(file_meta_client.tag_location) + + result = cli_runner.invoke( + file_metadata_download, + [file_path, '-g', metadata_loc, '-a', metadata_loc, '-t', metadata_loc], + input='n', + ) + + assert result.exit_code == 1 + + outputs = result.output + excepted_output = ( + customized_error_msg(ECustomizedError.LOCAL_METADATA_FILE_EXISTS) + + f'\n - general: {file_meta_client.general_location}' + + f'\n - attribute: {file_meta_client.attribute_location}' + + f'\n - tag: {file_meta_client.tag_location}\n' + + 'Do you want to overwrite the existing file? [y/N]: n\n' + + 'Metadata download cancelled.\n' + ) + assert outputs == excepted_output + + assert donwload_metadata_mock.call_count == 0 diff --git a/tests/app/services/file_manager/file_metadata/test_file_metadata_client.py b/tests/app/services/file_manager/file_metadata/test_file_metadata_client.py new file mode 100644 index 00000000..23ec570b --- /dev/null +++ b/tests/app/services/file_manager/file_metadata/test_file_metadata_client.py @@ -0,0 +1,51 @@ +# Copyright (C) 2022-2023 Indoc Systems +# +# Contact Indoc Systems for any questions regarding the use of this source code. + +from app.configs.app_config import AppConfig +from app.services.file_manager.file_metadata.file_metadata_client import FileMetaClient +from tests.conftest import decoded_token + + +def test_file_metadata_client_get_detail_success(mocker, httpx_mock): + item_info = { + 'id': 'test', + 'parent_id': 'test_parent', + 'parent_path': '', + 'name': 'admin', + 'zone': 0, + 'status': 'ACTIVE', + } + tags = ['test'] + attri_template_uid = 'template_uid' + attri_template_name = 'template_name' + attributes = {attri_template_uid: {'attr_1': 'value'}} + + mocker.patch( + 'app.services.user_authentication.token_manager.SrvTokenManager.decode_access_token', + return_value=decoded_token(), + ) + + mocker.patch( + 'app.services.file_manager.file_metadata.file_metadata_client.search_item', + return_value={'result': {**item_info, 'extended': {'extra': {'tags': tags, 'attributes': attributes}}}}, + ) + httpx_mock.add_response( + url=AppConfig.Connections.url_portal + f'/v1/data/manifest/{attri_template_uid}', + method='GET', + json={'result': {'id': attri_template_uid, 'name': attri_template_name}}, + ) + + mocker.patch( + 'app.services.file_manager.file_metadata.file_metadata_client.FileMetaClient.save_file_metadata', + return_value=None, + ) + + file_meta_client = FileMetaClient('zone', 'project_code/object_path', 'general', 'attr', 'tag') + assert file_meta_client.project_code == 'project_code' + assert file_meta_client.object_path == 'object_path' + + item_info, res_attributes, tags = file_meta_client.download_file_metadata() + assert item_info == item_info + assert res_attributes == {attri_template_name: attributes.get(attri_template_uid)} + assert tags == tags From 01723168fd7555fd5803e44f8ecdb1c1b339a086 Mon Sep 17 00:00:00 2001 From: Color Zhan Date: Tue, 5 Dec 2023 16:30:31 -0500 Subject: [PATCH 04/22] Pilot 4102: update `file upload` command to read tag/attribute file (#115) * add new command metadata to download file metadata/tags/attributes * update the upload command to read tag/attribute file when upload * fixup test cases * bumpup to next version --------- Co-authored-by: zhiren --- app/commands/file.py | 39 +++++++++++-------- .../file_upload/upload_validator.py | 14 +++---- pyproject.toml | 2 +- tests/app/commands/test_file.py | 5 ++- 4 files changed, 35 insertions(+), 25 deletions(-) diff --git a/app/commands/file.py b/app/commands/file.py index 3237e411..83338b2b 100644 --- a/app/commands/file.py +++ b/app/commands/file.py @@ -12,7 +12,6 @@ import app.services.output_manager.help_page as file_help import app.services.output_manager.message_handler as message_handler from app.configs.app_config import AppConfig -from app.configs.user_config import UserConfig from app.services.file_manager.file_download.download_client import SrvFileDownload from app.services.file_manager.file_list import SrvFileList from app.services.file_manager.file_manifests import SrvFileManifests @@ -40,15 +39,21 @@ def cli(): @click.command(name='upload') -@click.argument('paths', type=click.Path(exists=True), nargs=-1) -@click.option('-p', '--project-path', required=True, help=file_help.file_help_page(file_help.FileHELP.FILE_UPLOAD_P)) +@click.argument('files', type=click.Path(exists=True), nargs=-1) +@click.option( + '-p', + '--project-path', + required=True, + type=click.Path(), + help=file_help.file_help_page(file_help.FileHELP.FILE_UPLOAD_P), +) @click.option( '-a', '--attribute', default=None, required=False, help=file_help.file_help_page(file_help.FileHELP.FILE_UPLOAD_A), - # type=click.Path(exists=True), + type=click.File('rb'), show_default=True, ) @click.option( @@ -58,6 +63,7 @@ def cli(): required=False, multiple=True, help=file_help.file_help_page(file_help.FileHELP.FILE_UPLOAD_T), + type=click.File('rb'), show_default=True, ) @click.option( @@ -112,21 +118,25 @@ def cli(): def file_put(**kwargs): # noqa: C901 """""" - paths = kwargs.get('paths') + files = kwargs.get('files') project_path = kwargs.get('project_path') - tag = kwargs.get('tag') + tag_files = kwargs.get('tag') zone = kwargs.get('zone') upload_message = kwargs.get('upload_message') source_file = kwargs.get('source_file') zipping = kwargs.get('zip') - attribute = kwargs.get('attribute') + attribute_file = kwargs.get('attribute') thread = kwargs.get('thread') output_path = kwargs.get('output_path') - user = UserConfig() + # load tag json file to list, and attribute file to dict + tag = [] + for t_f in tag_files: + tag.extend(json.load(t_f)) + attribute = json.load(attribute_file) if attribute_file else None + # Check zone and upload-message zone = get_zone(zone) if zone else AppConfig.Env.green_zone.lower() - toc = customized_error_msg(ECustomizedError.TOU_CONTENT).replace(' ', '...') try: if zone.lower() == AppConfig.Env.core_zone.lower() and click.confirm(fit_terminal_width(toc), abort=True): @@ -136,7 +146,7 @@ def file_put(**kwargs): # noqa: C901 exit(1) # check if user input at least one file/folder - if len(paths) == 0: + if len(files) == 0: SrvErrorHandler.customized_handle(ECustomizedError.INVALID_PATHS, True) # check if the manifest file exists @@ -157,7 +167,6 @@ def file_put(**kwargs): # noqa: C901 'upload_message': upload_message, 'source': source_file, 'project_code': project_code, - 'token': user.access_token, 'attribute': attribute, 'tag': tag, } @@ -181,10 +190,10 @@ def file_put(**kwargs): # noqa: C901 # be the parent folder node + the shortest non-exist folder. (like one level down). # Unique Paths - paths = set(paths) + files = set(files) # the loop will read all input path(folder or files) # and process them one by one - for f in paths: + for f in files: # so this function will always return the furthest folder node as current_folder_node+parent_folder_id current_folder_node, parent_folder, create_folder_flag, result_file = assemble_path( f, @@ -275,10 +284,9 @@ def validate_upload_event(event): upload_message = event.get('upload_message') source = event.get('source') project_code = event.get('project_code') - token = event.get('token') attribute = event.get('attribute') tag = event.get('tag') - validator = UploadEventValidator(project_code, zone, upload_message, source, token, attribute, tag) + validator = UploadEventValidator(project_code, zone, upload_message, source, attribute, tag) converted_content = validator.validate_upload_event() return converted_content @@ -486,7 +494,6 @@ def file_metadata_download(**kwargs): attribute_folder = kwargs.get('attribute').rstrip('/') tag_folder = kwargs.get('tag').rstrip('/') - # user = UserConfig() # Check zone and upload-message zone = get_zone(zone) if zone else AppConfig.Env.green_zone.lower() file_meta_client = FileMetaClient(zone, file_path, general_folder, attribute_folder, tag_folder) diff --git a/app/services/file_manager/file_upload/upload_validator.py b/app/services/file_manager/file_upload/upload_validator.py index 3e48dcde..51500b40 100644 --- a/app/services/file_manager/file_upload/upload_validator.py +++ b/app/services/file_manager/file_upload/upload_validator.py @@ -2,7 +2,9 @@ # # Contact Indoc Systems for any questions regarding the use of this source code. -import os +from typing import Any +from typing import Dict +from typing import List from app.configs.app_config import AppConfig from app.services.file_manager.file_manifests import SrvFileManifests @@ -13,12 +15,13 @@ class UploadEventValidator: - def __init__(self, project_code, zone, upload_message, source, token, attribute, tag): + def __init__( + self, project_code: str, zone: str, upload_message: str, source: str, attribute: Dict[str, Any], tag: List[str] + ): self.project_code = project_code self.zone = zone self.upload_message = upload_message self.source = source - self.token = token self.attribute = attribute self.tag = tag @@ -37,11 +40,8 @@ def validate_zone(self): def validate_attribute(self): srv_manifest = SrvFileManifests() - if not os.path.isfile(self.attribute): - raise Exception('Attribute not exist in the given path') try: - attribute = srv_manifest.read_manifest_template(self.attribute) - attribute = srv_manifest.convert_import(attribute, self.project_code) + attribute = srv_manifest.convert_import(self.attribute, self.project_code) srv_manifest.validate_manifest(attribute) return attribute except Exception: diff --git a/pyproject.toml b/pyproject.toml index ce7a52f1..08678608 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "app" -version = "2.9.0" +version = "2.9.1" description = "This service is designed to support pilot platform" authors = ["Indoc Systems"] diff --git a/tests/app/commands/test_file.py b/tests/app/commands/test_file.py index e270ee00..b2a185fc 100644 --- a/tests/app/commands/test_file.py +++ b/tests/app/commands/test_file.py @@ -2,6 +2,7 @@ # # Contact Indoc Systems for any questions regarding the use of this source code. +import json from os import makedirs from os.path import dirname @@ -40,9 +41,11 @@ def test_file_upload_command_success_with_attribute(mocker, cli_runner): with runner.isolated_filesystem(): with open('test.txt', 'w') as f: f.write('test.txt') + with open('template.json', 'w') as f: + json.dump({'template': {'attr1': 'value'}}, f) result = cli_runner.invoke( - file_put, ['--project-path', 'test', '--thread', 1, '--attribute', 'test.json', 'test.txt'] + file_put, ['--project-path', 'test', '--thread', 1, '--attribute', 'template.json', 'test.txt'] ) assert result.exit_code == 0 simple_upload_mock.assert_called_once() From 052e6bbfc8f7f5d35317eb630f5efc722693a1e4 Mon Sep 17 00:00:00 2001 From: Color Zhan Date: Fri, 8 Dec 2023 09:09:42 -0500 Subject: [PATCH 05/22] Pilot 3723: add proper handler when `invalid token` return from KONG gateway (#117) * add the new error handler for 401 when portal has shared login session with cli * add the test cases for token refresh * update version to next patch --------- Co-authored-by: zhiren --- app/services/dataset_manager/dataset_list.py | 2 ++ app/services/project_manager/project.py | 2 ++ .../user_authentication/token_manager.py | 3 +++ app/utils/aggregated.py | 2 ++ pyproject.toml | 2 +- .../user_authentication/test_token_manager.py | 20 +++++++++++++++++++ tests/app/utils/test_aggregated.py | 2 +- 7 files changed, 31 insertions(+), 2 deletions(-) diff --git a/app/services/dataset_manager/dataset_list.py b/app/services/dataset_manager/dataset_list.py index 75b50a2f..0f128c05 100644 --- a/app/services/dataset_manager/dataset_list.py +++ b/app/services/dataset_manager/dataset_list.py @@ -42,6 +42,8 @@ def list_datasets(self, page, page_size): return res_to_dict elif response.status_code == 404: SrvErrorHandler.customized_handle(ECustomizedError.USER_DISABLED, True) + elif response.status_code == 401: + SrvErrorHandler.customized_handle(ECustomizedError.INVALID_TOKEN, if_exit=True) else: SrvErrorHandler.default_handle(response.content, True) except Exception as e: diff --git a/app/services/project_manager/project.py b/app/services/project_manager/project.py index 8a8c930b..79c08dca 100644 --- a/app/services/project_manager/project.py +++ b/app/services/project_manager/project.py @@ -43,6 +43,8 @@ def list_projects(self, page, page_size, order, order_by): return res_to_dict elif response.status_code == 404: SrvErrorHandler.customized_handle(ECustomizedError.USER_DISABLED, True) + elif response.status_code == 401: + SrvErrorHandler.customized_handle(ECustomizedError.INVALID_TOKEN, if_exit=True) else: SrvErrorHandler.default_handle(response.content, True) except Exception: diff --git a/app/services/user_authentication/token_manager.py b/app/services/user_authentication/token_manager.py index ebc574d0..314adbe8 100644 --- a/app/services/user_authentication/token_manager.py +++ b/app/services/user_authentication/token_manager.py @@ -12,6 +12,7 @@ from app.configs.user_config import UserConfig from app.models.enums import LoginMethod from app.models.service_meta_class import MetaService +from app.services.output_manager.error_handler import ECustomizedError from app.services.output_manager.error_handler import SrvErrorHandler from app.services.user_authentication.user_login_logout import exchange_api_key @@ -93,6 +94,8 @@ def refresh(self, azp: str) -> None: response = requests.post(url, data=payload, headers=headers) if response.status_code == 200: self.update_token(response.json()['access_token'], response.json()['refresh_token']) + elif response.status_code == 401: + SrvErrorHandler.customized_handle(ECustomizedError.INVALID_TOKEN, if_exit=True) else: SrvErrorHandler.default_handle(response.content) diff --git a/app/utils/aggregated.py b/app/utils/aggregated.py index a9052ece..a888fd14 100644 --- a/app/utils/aggregated.py +++ b/app/utils/aggregated.py @@ -44,6 +44,8 @@ def search_item(project_code, zone, folder_relative_path, item_type, container_t SrvErrorHandler.customized_handle(ECustomizedError.PERMISSION_DENIED, project_code) elif res.status_code == 404: pass + elif res.status_code == 401: + SrvErrorHandler.customized_handle(ECustomizedError.INVALID_TOKEN, if_exit=True) elif res.status_code != 200: SrvErrorHandler.default_handle(res.text, True) diff --git a/pyproject.toml b/pyproject.toml index 08678608..85770675 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "app" -version = "2.9.1" +version = "2.9.2" description = "This service is designed to support pilot platform" authors = ["Indoc Systems"] diff --git a/tests/app/services/user_authentication/test_token_manager.py b/tests/app/services/user_authentication/test_token_manager.py index 50379c80..6846ae20 100644 --- a/tests/app/services/user_authentication/test_token_manager.py +++ b/tests/app/services/user_authentication/test_token_manager.py @@ -3,9 +3,12 @@ # Contact Indoc Systems for any questions regarding the use of this source code. import jwt +import pytest +from app.configs.app_config import AppConfig from app.configs.user_config import UserConfig from app.services.user_authentication.token_manager import SrvTokenManager +from tests.conftest import decoded_token class TestSrvTokenManager: @@ -38,3 +41,20 @@ def test_refresh_api_key_calls_keycloak_and_stores_access_token_in_config(self, assert manager.config.access_token == access_token assert manager.config.refresh_token == '' + + def test_refresh_failed_with_invalid_token(self, requests_mock, mocker, settings, capsys): + manager = SrvTokenManager() + mocker.patch( + 'app.services.user_authentication.token_manager.SrvTokenManager.decode_access_token', + return_value=decoded_token(), + ) + + requests_mock.post( + AppConfig.Connections.url_keycloak_token, + status_code=401, + ) + + with pytest.raises(SystemExit): + manager.refresh('test_azp') + out, _ = capsys.readouterr() + assert out.rstrip() == 'Your login session has expired. Please try again or log in again.' diff --git a/tests/app/utils/test_aggregated.py b/tests/app/utils/test_aggregated.py index 9ddadd2f..08932873 100644 --- a/tests/app/utils/test_aggregated.py +++ b/tests/app/utils/test_aggregated.py @@ -99,4 +99,4 @@ def test_search_file_error_handling_with_401(requests_mock, mocker, capsys): with pytest.raises(SystemExit): search_item(test_project_code, 'zone', 'folder_relative_path', 'file', 'project') out, _ = capsys.readouterr() - assert out.rstrip() == 'Authentication failed.' + assert out.rstrip() == 'Your login session has expired. Please try again or log in again.' From 74942c26f4bb5c9ce011b6453282e800893993d5 Mon Sep 17 00:00:00 2001 From: zhiren Date: Fri, 8 Dec 2023 15:23:52 -0500 Subject: [PATCH 06/22] update config.py that config_path is configurable --- app/configs/config.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/app/configs/config.py b/app/configs/config.py index e4a36b4d..04bbf98c 100644 --- a/app/configs/config.py +++ b/app/configs/config.py @@ -17,10 +17,7 @@ class Settings(BaseSettings): project: str = 'pilot' app_name: str = 'pilotcli' - @computed_field - def config_path(self) -> str: - return str(Path.home() / f'.{self.app_name}') - + config_path: str = str(Path.home() / f'.{app_name}') config_file: str = 'config.ini' keycloak_device_client_id: str = 'cli' From ae74b14fa8dac1ee9dc169370f1c06f766822d15 Mon Sep 17 00:00:00 2001 From: Color Zhan Date: Mon, 11 Dec 2023 17:04:31 -0500 Subject: [PATCH 07/22] Pilot 3970: update cli logic with project folder implementation (#118) * remove the unnecessary item type in search function * testing * update logic for project folder * add the test case for project folder in list/upload api * update download logic for project folder * add the more test case for file download * use correct default config * bumup version --------- Co-authored-by: zhiren --- app/commands/file.py | 15 +-- app/services/file_manager/file_list.py | 15 ++- .../file_manager/file_upload/file_upload.py | 19 ++-- .../file_upload/upload_validator.py | 2 +- app/utils/aggregated.py | 3 +- pyproject.toml | 2 +- tests/app/commands/test_file.py | 107 +++++++++++++++++- .../file_upload/test_file_upload.py | 34 ++++++ tests/app/utils/test_aggregated.py | 8 +- 9 files changed, 173 insertions(+), 32 deletions(-) diff --git a/app/commands/file.py b/app/commands/file.py index 83338b2b..55323b7f 100644 --- a/app/commands/file.py +++ b/app/commands/file.py @@ -195,7 +195,7 @@ def file_put(**kwargs): # noqa: C901 # and process them one by one for f in files: # so this function will always return the furthest folder node as current_folder_node+parent_folder_id - current_folder_node, parent_folder, create_folder_flag, result_file = assemble_path( + current_folder_node, parent_folder, create_folder_flag, target_folder = assemble_path( f, target_folder, project_code, @@ -418,17 +418,18 @@ def file_download(**kwargs): else: item_res = [] for path in paths: - project_code = path.strip('/').split('/')[0] + project_code, root_folder = path.strip('/').split('/')[:2] target_path = '/'.join(path.split('/')[1::]) - item = search_item(project_code, zone, target_path, '') + # search the root to check for name folder or project folder + root_item = search_item(project_code, zone, root_folder).get('result', {}) + target_path = 'shared/' + target_path if root_item.get('type') == 'project_folder' else target_path + + # search the target item and download to local + item = search_item(project_code, zone, target_path) if item.get('code') == 200 and item.get('result'): item_status = 'success' item_result = item.get('result') item_geid = item.get('result').get('id') - elif item.get('code') == 403 and item.get('error_msg'): - item_status = item.get('error_msg') - item_result = {} - item_geid = path else: item_status = 'File Not Exist' item_result = {} diff --git a/app/services/file_manager/file_list.py b/app/services/file_manager/file_list.py index 7c178e5b..705bcd1d 100644 --- a/app/services/file_manager/file_list.py +++ b/app/services/file_manager/file_list.py @@ -29,7 +29,13 @@ def list_files(self, paths, zone, page, page_size): source_type = 'project' else: source_type = 'project' - res = search_item(project_code, zone, folder_rel_path, 'folder') + res = search_item(project_code, zone, folder_rel_path) + parent_folder = res.get('result') + # if the target folder is project folder add the default path + if parent_folder.get('type') == 'project_folder': + folder_rel_path = 'shared/' + folder_rel_path + + # now query the backend to get the file list get_url = AppConfig.Connections.url_bff + f'/v1/{project_code}/files/query' headers = { 'Authorization': 'Bearer ' + self.user.access_token, @@ -49,12 +55,13 @@ def list_files(self, paths, zone, page, page_size): elif res_json.get('error_msg') == 'Folder not exist': SrvErrorHandler.customized_handle(ECustomizedError.INVALID_FOLDER, True) res = res_json.get('result') - files = '' - folders = '' + + # then format the console output + files, folders = '', '' for f in res: if 'file' == f.get('type'): files = files + f.get('name') + ' ...' - elif f.get('type') in ['folder', 'name_folder']: + elif f.get('type') in ['folder', 'name_folder', 'project_folder']: folders = folders + f"\033[34m{f.get('name')}\033[0m ..." f_string = folders + files return f_string diff --git a/app/services/file_manager/file_upload/file_upload.py b/app/services/file_manager/file_upload/file_upload.py index 64c600e4..ad8ea08c 100644 --- a/app/services/file_manager/file_upload/file_upload.py +++ b/app/services/file_manager/file_upload/file_upload.py @@ -66,29 +66,30 @@ def assemble_path( - current_file_path: the format file path on platform - parent_folder: the item information of longest parent folder - create_folder_flag: the flag to indicate if need to create new folder - - result_file: the result file if zipping + - target_folder: result object path on platform ''' current_file_path = target_folder + '/' + f.rstrip('/').split('/')[-1] - result_file = current_file_path - if zipping: - result_file = result_file + '.zip' - # set name folder as first parent folder name_folder = target_folder.split('/')[0] - parent_folder = search_item(project_code, zone, name_folder, 'name_folder') - parent_folder = parent_folder.get('result') + parent_folder = search_item(project_code, zone, name_folder).get('result', {}) # if f input is a file then current_folder_node is target_folder # otherwise it is target_folder + f input name current_folder_node = target_folder if os.path.isfile(f) else current_file_path create_folder_flag = False + # always add `shared/` as prefix to folder/file if + # they directly under the project root folder + if parent_folder.get('type') == 'project_folder': + current_folder_node = 'shared/' + current_folder_node + target_folder = 'shared/' + target_folder + if len(current_file_path.split('/')) > 2: sub_path = target_folder.split('/') for index in range(len(sub_path) - 1): folder_path = '/'.join(sub_path[0 : 2 + index]) - res = search_item(project_code, zone, folder_path, 'folder') + res = search_item(project_code, zone, folder_path) # find the longest existing folder as parent folder # if user input a path that need to create some folders @@ -111,7 +112,7 @@ def assemble_path( if not parent_folder: SrvErrorHandler.customized_handle(ECustomizedError.PERMISSION_DENIED, True) - return current_folder_node, parent_folder, create_folder_flag, result_file + return current_folder_node, parent_folder, create_folder_flag, target_folder def simple_upload( # noqa: C901 diff --git a/app/services/file_manager/file_upload/upload_validator.py b/app/services/file_manager/file_upload/upload_validator.py index 51500b40..9dd40531 100644 --- a/app/services/file_manager/file_upload/upload_validator.py +++ b/app/services/file_manager/file_upload/upload_validator.py @@ -32,7 +32,7 @@ def validate_zone(self): ECustomizedError.INVALID_UPLOAD_REQUEST, True, value='upload-message is required' ) if self.source: - source_file_info = search_item(self.project_code, AppConfig.Env.core_zone.lower(), self.source, 'file') + source_file_info = search_item(self.project_code, AppConfig.Env.core_zone.lower(), self.source) source_file_info = source_file_info['result'] if not source_file_info: SrvErrorHandler.customized_handle(ECustomizedError.INVALID_SOURCE_FILE, True, value=self.source) diff --git a/app/utils/aggregated.py b/app/utils/aggregated.py index a888fd14..912b99c0 100644 --- a/app/utils/aggregated.py +++ b/app/utils/aggregated.py @@ -28,14 +28,13 @@ def resilient_session(): @require_valid_token() -def search_item(project_code, zone, folder_relative_path, item_type, container_type='project'): +def search_item(project_code, zone, folder_relative_path, container_type='project'): token = UserConfig().access_token url = AppConfig.Connections.url_bff + '/v1/project/{}/search'.format(project_code) params = { 'zone': zone, 'project_code': project_code, 'path': folder_relative_path, - 'item_type': item_type, 'container_type': container_type, } headers = {'Authorization': 'Bearer ' + token} diff --git a/pyproject.toml b/pyproject.toml index 85770675..9154869f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "app" -version = "2.9.2" +version = "2.9.3" description = "This service is designed to support pilot platform" authors = ["Indoc Systems"] diff --git a/tests/app/commands/test_file.py b/tests/app/commands/test_file.py index b2a185fc..172f0bc2 100644 --- a/tests/app/commands/test_file.py +++ b/tests/app/commands/test_file.py @@ -7,8 +7,10 @@ from os.path import dirname import click +import pytest import questionary +from app.commands.file import file_download from app.commands.file import file_list from app.commands.file import file_metadata_download from app.commands.file import file_put @@ -95,19 +97,30 @@ def test_resumable_upload_command_failed_with_file_not_exists(mocker, cli_runner assert result.output == customized_error_msg(ECustomizedError.INVALID_RESUMABLE) + '\n' -def test_file_list_with_pagination(requests_mock, mocker, cli_runner): +def test_file_list_with_pagination_with_folder_success(requests_mock, mocker, cli_runner): mocker.patch( 'app.services.user_authentication.token_manager.SrvTokenManager.decode_access_token', return_value=decoded_token(), ) - mocker.patch('app.services.file_manager.file_list.search_item', return_value=None) + mocker.patch( + 'app.services.file_manager.file_list.search_item', + return_value={ + 'result': { + 'type': 'folder', + 'id': 'id', + } + }, + ) requests_mock.get( 'http://bff_cli' + '/v1/testproject/files/query', json={ 'code': 200, 'error_msg': '', - 'result': [{'type': 'file', 'name': 'file1'}, {'type': 'file', 'name': 'file2'}], + 'result': [ + {'type': 'file', 'name': 'file1'}, + {'type': 'file', 'name': 'file2'}, + ], }, ) mocker.patch.object(questionary, 'select') @@ -117,13 +130,56 @@ def test_file_list_with_pagination(requests_mock, mocker, cli_runner): assert outputs[0] == 'file1 file2 ' +@pytest.mark.parametrize('parent_folder_type', ['name_folder', 'project_folder']) +def test_file_list_with_pagination_with_name_project_folder(requests_mock, mocker, cli_runner, parent_folder_type): + mocker.patch( + 'app.services.user_authentication.token_manager.SrvTokenManager.decode_access_token', + return_value=decoded_token(), + ) + + mocker.patch( + 'app.services.file_manager.file_list.search_item', + return_value={ + 'result': { + 'type': parent_folder_type, + 'id': 'id', + } + }, + ) + requests_mock.get( + 'http://bff_cli' + '/v1/testproject/files/query', + json={ + 'code': 200, + 'error_msg': '', + 'result': [ + {'type': 'folder', 'name': 'folder1'}, + {'type': 'name_folder', 'name': 'name_folder1'}, + {'type': 'project_folder', 'name': 'project_folder1'}, + ], + }, + ) + mocker.patch.object(questionary, 'select') + questionary.select.return_value.ask.return_value = 'exit' + result = cli_runner.invoke(file_list, ['testproject/admin', '-z', 'greenroom']) + outputs = result.output.split('\n') + assert outputs[0] == 'folder1 name_folder1 project_folder1 ' + + def test_empty_file_list_with_pagination(requests_mock, mocker, cli_runner): mocker.patch( 'app.services.user_authentication.token_manager.SrvTokenManager.decode_access_token', return_value=decoded_token(), ) - mocker.patch('app.services.file_manager.file_list.search_item', return_value=None) + mocker.patch( + 'app.services.file_manager.file_list.search_item', + return_value={ + 'result': { + 'type': 'folder', + 'id': 'id', + } + }, + ) requests_mock.get( 'http://bff_cli' + '/v1/testproject/files/query', json={'code': 200, 'error_msg': '', 'result': []}, @@ -135,6 +191,49 @@ def test_empty_file_list_with_pagination(requests_mock, mocker, cli_runner): assert outputs[0] == ' ' +@pytest.mark.parametrize('parent_folder_type', ['name_folder', 'project_folder']) +def test_file_download_success(requests_mock, mocker, cli_runner, parent_folder_type): + mocker.patch( + 'app.services.user_authentication.token_manager.SrvTokenManager.decode_access_token', + return_value=decoded_token(), + ) + + search_mock = mocker.patch( + 'app.commands.file.search_item', + side_effect=[ + { + 'code': 200, + 'result': { + 'type': parent_folder_type, + 'name': 'test', + 'id': 'id', + }, + }, + { + 'code': 200, + 'result': { + 'type': 'file', + 'id': 'id', + }, + }, + ], + ) + + download_mock = mocker.patch( + 'app.services.file_manager.file_download.download_client.SrvFileDownload.simple_download_file', + return_value=None, + ) + + project_code, target_folder = 'testproject', 'test/test.txt' + result = cli_runner.invoke(file_download, [f'{project_code}/{target_folder}', './']) + outputs = result.output.split('\n') + assert outputs[0] == '' + + except_target_folder = 'test/test.txt' if parent_folder_type == 'name_folder' else 'shared/test/test.txt' + search_mock.assert_called_with(project_code, 'greenroom', except_target_folder) + download_mock.assert_called_once() + + def test_download_file_metadata_file_duplicate_success(mocker, cli_runner): mocker.patch( 'app.services.user_authentication.token_manager.SrvTokenManager.decode_access_token', diff --git a/tests/app/services/file_manager/file_upload/test_file_upload.py b/tests/app/services/file_manager/file_upload/test_file_upload.py index c8eddaa4..cff579fa 100644 --- a/tests/app/services/file_manager/file_upload/test_file_upload.py +++ b/tests/app/services/file_manager/file_upload/test_file_upload.py @@ -28,6 +28,7 @@ def test_assemble_path_at_name_folder(mocker): 'parent_path': '', 'name': 'admin', 'zone': 0, + 'type': 'name_folder', } }, ) @@ -55,6 +56,7 @@ def test_assemble_path_at_exsting_folder(mocker): 'parent_path': '', 'name': 'admin', 'zone': 0, + 'type': 'folder', } }, { @@ -64,6 +66,7 @@ def test_assemble_path_at_exsting_folder(mocker): 'parent_path': 'admin', 'name': 'test_folder_exist', 'zone': 0, + 'type': 'folder', } }, ] @@ -93,6 +96,7 @@ def test_assemble_path_at_non_existing_folder(mocker): 'parent_path': '', 'name': 'admin', 'zone': 0, + 'type': 'folder', } }, {'result': {}}, @@ -109,6 +113,36 @@ def test_assemble_path_at_non_existing_folder(mocker): assert create_folder_flag is True +def test_assemble_path_at_project_folder(mocker): + local_file_path = './test/file.txt' + target_folder = 'project_folder' + project_code = 'test_project' + zone = 0 + resumable_id = None + + mocker.patch( + 'app.services.file_manager.file_upload.file_upload.search_item', + return_value={ + 'result': { + 'id': 'test', + 'parent_id': 'test_parent', + 'parent_path': '', + 'name': 'project_folder', + 'zone': 0, + 'type': 'project_folder', + } + }, + ) + + current_file_path, parent_folder, create_folder_flag, target_folder = assemble_path( + local_file_path, target_folder, project_code, zone, resumable_id + ) + assert current_file_path == 'shared/project_folder/file.txt' + assert parent_folder.get('name') == 'project_folder' + assert target_folder == 'shared/project_folder' + assert create_folder_flag is False + + def test_file_upload_skip_empty_file(mocker, tmp_path, capfd): file_name = 'test' upload_event = { diff --git a/tests/app/utils/test_aggregated.py b/tests/app/utils/test_aggregated.py index 08932873..d449a5f4 100644 --- a/tests/app/utils/test_aggregated.py +++ b/tests/app/utils/test_aggregated.py @@ -55,7 +55,7 @@ def test_search_file_should_return_200(requests_mock, mocker): 'storage': {'id': 'storage-id', 'location_uri': 'minio-path', 'version': 'version-id'}, 'extended': {'id': 'extended-id', 'extra': {'tags': [], 'system_tags': [], 'attributes': {}}}, } - res = search_item(test_project_code, 'zone', 'folder_relative_path', 'file', 'project') + res = search_item(test_project_code, 'zone', 'folder_relative_path', 'project') assert res['result'] == expected_result @@ -68,7 +68,7 @@ def test_search_item_returns_response_when_status_code_is_404(requests_mock, moc status_code=404, ) - response = search_item(test_project_code, 'zone', 'folder_relative_path', 'file', 'project') + response = search_item(test_project_code, 'zone', 'folder_relative_path', 'project') assert response == expected_response @@ -81,7 +81,7 @@ def test_search_file_error_handling_with_403(requests_mock, mocker, capsys): status_code=403, ) with pytest.raises(SystemExit): - search_item(test_project_code, 'zone', 'folder_relative_path', 'file', 'project') + search_item(test_project_code, 'zone', 'folder_relative_path', 'project') out, _ = capsys.readouterr() assert ( out.rstrip() @@ -97,6 +97,6 @@ def test_search_file_error_handling_with_401(requests_mock, mocker, capsys): status_code=401, ) with pytest.raises(SystemExit): - search_item(test_project_code, 'zone', 'folder_relative_path', 'file', 'project') + search_item(test_project_code, 'zone', 'folder_relative_path', 'project') out, _ = capsys.readouterr() assert out.rstrip() == 'Your login session has expired. Please try again or log in again.' From 1e742bffae94c6640c8651b6616cec000e00d15d Mon Sep 17 00:00:00 2001 From: zhiren Date: Tue, 12 Dec 2023 10:01:32 -0500 Subject: [PATCH 08/22] update the uploading helper message --- app/commands/file.py | 6 +++--- app/resources/custom_help.py | 19 ++++++++++--------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/app/commands/file.py b/app/commands/file.py index 55323b7f..2e0e2263 100644 --- a/app/commands/file.py +++ b/app/commands/file.py @@ -77,7 +77,7 @@ def cli(): @click.option( '-m', '--upload-message', - default='', + default=None, required=False, help=file_help.file_help_page(file_help.FileHELP.FILE_UPLOAD_M), show_default=True, @@ -103,7 +103,7 @@ def cli(): '-td', default=1, required=False, - help='The number of thread for upload a file', + help='The number of threads for uploading a file.', show_default=True, ) @click.option( @@ -111,7 +111,7 @@ def cli(): '-o', default='./manifest.json', required=False, - help='The output path for the manifest file of resumable upload', + help='The output path for the manifest file of resumable upload.', show_default=True, ) @doc(file_help.file_help_page(file_help.FileHELP.FILE_UPLOAD)) diff --git a/app/resources/custom_help.py b/app/resources/custom_help.py index a3d3122f..03407b8f 100644 --- a/app/resources/custom_help.py +++ b/app/resources/custom_help.py @@ -37,25 +37,26 @@ class HelpPage: 'FILE_SYNC': 'Download files/folders from a given Project/folder/file in core zone.', 'FILE_UPLOAD': 'Upload files/folders to a given Project path.', 'FILE_RESUME': 'Resume the upload process with given manifest file.', - 'FILE_Z': 'Target Zone (i.e., core/greenroom) [default: greenroom]', + 'FILE_Z': 'Target Zone (i.e., core/greenroom).', 'FILE_ATTRIBUTE_P': 'Project Code', 'FILE_ATTRIBUTE_N': 'Attribute Template Name', 'FILE_SYNC_ZIP': 'Download files as a zip.', 'FILE_SYNC_I': 'Enable downloading by geid.', - 'FILE_SYNC_Z': 'Target Zone (i.e., core/greenroom)', - 'FILE_UPLOAD_P': 'Project folder path starting from Project code. (i.e., indoctestproject/user/folder)', - 'FILE_UPLOAD_A': 'File Attribute Template used for annotating files during upload.', - 'FILE_UPLOAD_T': ( - 'Add a tag to the file. This option could be used multiple times for adding multiple tags.' + 'FILE_SYNC_Z': 'Target Zone (i.e., core/greenroom).', + 'FILE_UPLOAD_P': 'Project folder path starting from Project Code. (i.e., indoctestproject/user/folder)', + 'FILE_UPLOAD_A': 'Add attributes to the file using a File Attribute Template.', + 'FILE_UPLOAD_T': 'Add tags to the file using a Tag file.', + 'FILE_UPLOAD_M': 'The message used to comment on the purpose of uploading your processed file.', + 'FILE_UPLOAD_S': ( + 'Project file path for identifying a source file when creating an upstream ' + 'file lineage node. Source files must exist in the Core zone.' ), - 'FILE_UPLOAD_M': 'The message used to comment on the purpose of uploading your processed file', - 'FILE_UPLOAD_S': 'The Project path of the source file of your processed files.', 'FILE_UPLOAD_PIPELINE': ( "The processed pipeline of your processed files. [only used with '--source' option]" ), 'FILE_UPLOAD_ZIP': 'Upload folder as a compressed zip file.', 'FILE_META': 'Download metadata file of a given file in target zone.', - 'FILE_META_Z': 'Target Zone (i.e., core/greenroom)', + 'FILE_META_Z': 'Target Zone (i.e., core/greenroom).', 'FILE_META_G': 'The location of general metadata file', 'FILE_META_A': 'The location of attribute metadata file', 'FILE_META_T': 'The location of tag metadata file', From 079bbce327ff5b5868ccd82d66ec0b7201b6f1dc Mon Sep 17 00:00:00 2001 From: zhiren Date: Thu, 14 Dec 2023 12:07:50 -0500 Subject: [PATCH 09/22] hotfix the metadata use wrong input as container code --- .../file_metadata/file_metadata_client.py | 20 +++++++---- .../test_file_metadata_client.py | 35 +++++++++++++++++++ 2 files changed, 48 insertions(+), 7 deletions(-) diff --git a/app/services/file_manager/file_metadata/file_metadata_client.py b/app/services/file_manager/file_metadata/file_metadata_client.py index dff11954..56379d87 100644 --- a/app/services/file_manager/file_metadata/file_metadata_client.py +++ b/app/services/file_manager/file_metadata/file_metadata_client.py @@ -17,6 +17,7 @@ import click from click.exceptions import Abort +import app.services.logger_services.log_functions as logger import app.services.output_manager.message_handler as message_handler from app.services.output_manager.error_handler import ECustomizedError from app.services.output_manager.error_handler import customized_error_msg @@ -108,18 +109,23 @@ def download_file_metadata(self) -> List[Dict[str, Any]]: """ project_code, object_path = self.file_path.split('/', 1) - item_res = search_item(project_code, self.zone, object_path, 'file').get('result', {}) + item_res = search_item(project_code, self.zone, object_path).get('result', {}) extra_info = item_res.pop('extended', {}).get('extra') tags = extra_info.get('tags', []) - attributes = extra_info.get('attributes', []) + attributes = extra_info.get('attributes', {}) # use the uuid of attribute template to get template name - template_uuid = next(iter(attributes)) - attribute_name = get_attribute_template_by_id(template_uuid).get('name') - attribute_detail = attributes.get(template_uuid) + attribute_info = {} + if len(attributes): + template_uuid = next(iter(attributes)) + attribute_name = get_attribute_template_by_id(template_uuid).get('name') + attribute_detail = attributes.get(template_uuid) + attribute_info = {attribute_name: attribute_detail} + self.save_file_metadata(self.attribute_location, attribute_info) + else: + logger.warning('No attribute metadata found.') # save metadata into files self.save_file_metadata(self.general_location, item_res) - self.save_file_metadata(self.attribute_location, {attribute_name: attribute_detail}) self.save_file_metadata(self.tag_location, tags) - return item_res, {attribute_name: attribute_detail}, tags + return item_res, attribute_info, tags diff --git a/tests/app/services/file_manager/file_metadata/test_file_metadata_client.py b/tests/app/services/file_manager/file_metadata/test_file_metadata_client.py index 23ec570b..669cd06a 100644 --- a/tests/app/services/file_manager/file_metadata/test_file_metadata_client.py +++ b/tests/app/services/file_manager/file_metadata/test_file_metadata_client.py @@ -49,3 +49,38 @@ def test_file_metadata_client_get_detail_success(mocker, httpx_mock): assert item_info == item_info assert res_attributes == {attri_template_name: attributes.get(attri_template_uid)} assert tags == tags + + +def test_file_metadata_client_get_detail_success_with_no_tag_attributes(mocker, httpx_mock): + item_info = { + 'id': 'test', + 'parent_id': 'test_parent', + 'parent_path': '', + 'name': 'admin', + 'zone': 0, + 'status': 'ACTIVE', + } + + mocker.patch( + 'app.services.user_authentication.token_manager.SrvTokenManager.decode_access_token', + return_value=decoded_token(), + ) + + mocker.patch( + 'app.services.file_manager.file_metadata.file_metadata_client.search_item', + return_value={'result': {**item_info, 'extended': {'extra': {'tags': [], 'attributes': {}}}}}, + ) + + mocker.patch( + 'app.services.file_manager.file_metadata.file_metadata_client.FileMetaClient.save_file_metadata', + return_value=None, + ) + + file_meta_client = FileMetaClient('zone', 'project_code/object_path', 'general', 'attr', 'tag') + assert file_meta_client.project_code == 'project_code' + assert file_meta_client.object_path == 'object_path' + + item_info, res_attributes, tags = file_meta_client.download_file_metadata() + assert item_info == item_info + assert res_attributes == {} + assert tags == tags From 9c4b3bff3ea998a57729ea611cf00e9fe37d9e7c Mon Sep 17 00:00:00 2001 From: zhiren Date: Tue, 19 Dec 2023 16:53:15 -0500 Subject: [PATCH 10/22] update config file to allow user customize the api_url and keycloak_url instead of domain only --- README.md | 26 +++++++++++++++++-------- app/configs/app_config.py | 6 +++--- app/configs/config.py | 41 ++++++++++++++------------------------- pyproject.toml | 2 +- 4 files changed, 37 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index 0fb3f2ef..a43e3926 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,24 @@ Command line tool that allows the user to execute data operations on the platfor - Python - [Click](https://click.palletsprojects.com/en/8.0.x/) +## Getting Started + +### Prerequisites +- Python 3.7+ +- [Poetry](https://python-poetry.org/docs/#installation) + +#### Run with Python +1. Install dependencies (optional: run in edit mode). + ``` + poetry install + poetry run python app/pilotcli.py --help + ``` +2. Add environment variables if needed. + + 1. Create a `.env` file in the root directory of the project. + 2. Sdd following two environmental varibles to the `.env` file. + - `api_url`: the url that the api server is hosted on. default is `https://api.pilot.indocresearch.com/pilot` + - `keycloak_url`: thr url that the keycloak server is hosted on. default is `https://iam.pilot.indocresearch.com/realms/pilot/protocol/openid-connect` #### Run from bundled application 1. Navigate to the appropriate directory for your system. @@ -17,14 +35,6 @@ Command line tool that allows the user to execute data operations on the platfor ./app/bundled_app/mac/ ./app/bundled_app/mac_arm/ -#### Run with Python -1. Install dependencies (optional: run in edit mode). - - poetry install - poetry run pilotcli - -2. Add environment variables if needed. - ## Usage ./app/bundled_app/linux/pilotcli --help diff --git a/app/configs/app_config.py b/app/configs/app_config.py index 79e4158f..d1c77ef9 100644 --- a/app/configs/app_config.py +++ b/app/configs/app_config.py @@ -41,8 +41,8 @@ class Connections: url_dataset_v2download = ConfigClass.url_dataset_v2download url_dataset = ConfigClass.url_dataset url_validation = ConfigClass.url_validation - url_keycloak = ConfigClass.url_keycloak - url_keycloak_token = f'{ConfigClass.url_keycloak}/token' + url_keycloak = ConfigClass.keycloak_url + url_keycloak_token = f'{ConfigClass.keycloak_url}/token' url_bff = ConfigClass.url_bff - url_base = ConfigClass.base_url + url_base = ConfigClass.api_url url_portal = ConfigClass.url_portal diff --git a/app/configs/config.py b/app/configs/config.py index 04bbf98c..2d478319 100644 --- a/app/configs/config.py +++ b/app/configs/config.py @@ -28,59 +28,48 @@ class Settings(BaseSettings): harbor_client_secret: str = '' url_harbor: str = '' - domain: str = 'pilot.indocresearch.com' - - @computed_field - def base_url(self) -> str: - return f'https://api.{self.domain}/pilot' + api_url: str = 'https://api.pilot.indocresearch.com/pilot' + keycloak_url: str = 'https://iam.pilot.indocresearch.com/realms/pilot/protocol/openid-connect' @computed_field def url_bff(self) -> str: - return f'{self.base_url}/cli' + return f'{self.api_url}/cli' @computed_field def url_portal(self) -> str: - return f'{self.base_url}/portal' - - @computed_field - def url_keycloak_realm(self) -> str: - return f'https://iam.{self.domain}/realms/pilot' - - @computed_field - def url_keycloak(self) -> str: - return f'{self.url_keycloak_realm}/protocol/openid-connect' + return f'{self.api_url}/portal' @computed_field def url_authn(self) -> str: - return f'{self.base_url}/portal/users/auth' + return f'{self.api_url}/portal/users/auth' @computed_field def url_refresh_token(self) -> str: - return f'{self.base_url}/portal/users/refresh' + return f'{self.api_url}/portal/users/refresh' @computed_field def url_file_tag(self) -> str: - return f'{self.base_url}/portal/v2/%s/tags' + return f'{self.api_url}/portal/v2/%s/tags' @computed_field def url_upload_greenroom(self) -> str: - return f'{self.base_url}/upload/gr' + return f'{self.api_url}/upload/gr' @computed_field def url_upload_core(self) -> str: - return f'{self.base_url}/upload/core' + return f'{self.api_url}/upload/core' @computed_field def url_status(self) -> str: - return f'{self.base_url}/portal/v1/files/actions/tasks' + return f'{self.api_url}/portal/v1/files/actions/tasks' @computed_field def url_download_greenroom(self) -> str: - return f'{self.base_url}/portal/download/gr/' + return f'{self.api_url}/portal/download/gr/' @computed_field def url_download_core(self) -> str: - return f'{self.base_url}/portal/download/core/' + return f'{self.api_url}/portal/download/core/' @computed_field def url_v2_download_pre(self) -> str: @@ -88,15 +77,15 @@ def url_v2_download_pre(self) -> str: @computed_field def url_dataset_v2download(self) -> str: - return f'{self.base_url}/portal/download/core/v2/dataset' + return f'{self.api_url}/portal/download/core/v2/dataset' @computed_field def url_dataset(self) -> str: - return f'{self.base_url}/portal/v1/dataset' + return f'{self.api_url}/portal/v1/dataset' @computed_field def url_validation(self) -> str: - return f'{self.base_url}/v1/files/validation' + return f'{self.api_url}/v1/files/validation' @lru_cache(1) diff --git a/pyproject.toml b/pyproject.toml index 9154869f..5ec41f62 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "app" -version = "2.9.3" +version = "2.9.4" description = "This service is designed to support pilot platform" authors = ["Indoc Systems"] From c8c738101552da94c4fb582322d853b77e84ac4c Mon Sep 17 00:00:00 2001 From: zhiren Date: Tue, 19 Dec 2023 17:06:55 -0500 Subject: [PATCH 11/22] fixup the test cases --- app/services/user_authentication/user_login_logout.py | 2 +- tests/app/commands/test_user.py | 2 +- tests/app/services/user_authentication/test_token_manager.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/services/user_authentication/user_login_logout.py b/app/services/user_authentication/user_login_logout.py index 33d71838..fd3ce5e3 100644 --- a/app/services/user_authentication/user_login_logout.py +++ b/app/services/user_authentication/user_login_logout.py @@ -23,7 +23,7 @@ def exchange_api_key(api_key: str) -> Union[str, None]: """Exchange API Key with JWT token using Keycloak.""" - url = f'{ConfigClass.url_keycloak_realm}/api-key/{api_key}' + url = f'{ConfigClass.keycloak_url}/api-key/{api_key}' try: response = requests.get(url, timeout=5) response.raise_for_status() diff --git a/tests/app/commands/test_user.py b/tests/app/commands/test_user.py index 3d2ff335..bc06e552 100644 --- a/tests/app/commands/test_user.py +++ b/tests/app/commands/test_user.py @@ -14,7 +14,7 @@ def test_login_command_with_api_key_option_calls_keycloak_and_stores_response_in username = fake.user_name() api_key = fake.pystr(20) access_token = jwt.encode({'preferred_username': username}, key='').decode() - requests_mock.get(f'{settings.url_keycloak_realm}/api-key/{api_key}', json={'access_token': access_token}) + requests_mock.get(f'{settings.keycloak_url}/api-key/{api_key}', json={'access_token': access_token}) result = cli_runner.invoke(login, ['--api-key', api_key]) diff --git a/tests/app/services/user_authentication/test_token_manager.py b/tests/app/services/user_authentication/test_token_manager.py index 6846ae20..e0e5fc10 100644 --- a/tests/app/services/user_authentication/test_token_manager.py +++ b/tests/app/services/user_authentication/test_token_manager.py @@ -33,7 +33,7 @@ def test_refresh_api_key_calls_keycloak_and_stores_access_token_in_config(self, manager = SrvTokenManager() access_token = jwt.encode({}, key='').decode() requests_mock.get( - f'{settings.url_keycloak_realm}/api-key/{manager.config.api_key}', + f'{settings.keycloak_url}/api-key/{manager.config.api_key}', json={'access_token': access_token}, ) From 4487ad5315a90262eec1e9e7ea10c721123da0a6 Mon Sep 17 00:00:00 2001 From: zhiren Date: Tue, 19 Dec 2023 17:16:42 -0500 Subject: [PATCH 12/22] use the keycloak realm url instead of keycloak token in config --- app/configs/app_config.py | 5 +++-- app/configs/config.py | 6 +++++- app/services/user_authentication/user_login_logout.py | 2 +- tests/app/commands/test_user.py | 2 +- .../app/services/user_authentication/test_token_manager.py | 2 +- 5 files changed, 11 insertions(+), 6 deletions(-) diff --git a/app/configs/app_config.py b/app/configs/app_config.py index d1c77ef9..85e59928 100644 --- a/app/configs/app_config.py +++ b/app/configs/app_config.py @@ -41,8 +41,9 @@ class Connections: url_dataset_v2download = ConfigClass.url_dataset_v2download url_dataset = ConfigClass.url_dataset url_validation = ConfigClass.url_validation - url_keycloak = ConfigClass.keycloak_url - url_keycloak_token = f'{ConfigClass.keycloak_url}/token' + url_keycloak = ConfigClass.url_keycloak + url_keycloak_token = f'{ConfigClass.url_keycloak}/token' + url_keycloak_realm = ConfigClass.keycloak_realm_url url_bff = ConfigClass.url_bff url_base = ConfigClass.api_url url_portal = ConfigClass.url_portal diff --git a/app/configs/config.py b/app/configs/config.py index 2d478319..9b65018d 100644 --- a/app/configs/config.py +++ b/app/configs/config.py @@ -29,7 +29,7 @@ class Settings(BaseSettings): url_harbor: str = '' api_url: str = 'https://api.pilot.indocresearch.com/pilot' - keycloak_url: str = 'https://iam.pilot.indocresearch.com/realms/pilot/protocol/openid-connect' + keycloak_realm_url: str = 'https://iam.pilot.indocresearch.com/realms/pilot' @computed_field def url_bff(self) -> str: @@ -39,6 +39,10 @@ def url_bff(self) -> str: def url_portal(self) -> str: return f'{self.api_url}/portal' + @computed_field + def url_keycloak(self) -> str: + return f'{self.keycloak_realm_url}/protocol/openid-connect' + @computed_field def url_authn(self) -> str: return f'{self.api_url}/portal/users/auth' diff --git a/app/services/user_authentication/user_login_logout.py b/app/services/user_authentication/user_login_logout.py index fd3ce5e3..4b27f5bc 100644 --- a/app/services/user_authentication/user_login_logout.py +++ b/app/services/user_authentication/user_login_logout.py @@ -23,7 +23,7 @@ def exchange_api_key(api_key: str) -> Union[str, None]: """Exchange API Key with JWT token using Keycloak.""" - url = f'{ConfigClass.keycloak_url}/api-key/{api_key}' + url = f'{ConfigClass.keycloak_realm_url}/api-key/{api_key}' try: response = requests.get(url, timeout=5) response.raise_for_status() diff --git a/tests/app/commands/test_user.py b/tests/app/commands/test_user.py index bc06e552..3d2ff335 100644 --- a/tests/app/commands/test_user.py +++ b/tests/app/commands/test_user.py @@ -14,7 +14,7 @@ def test_login_command_with_api_key_option_calls_keycloak_and_stores_response_in username = fake.user_name() api_key = fake.pystr(20) access_token = jwt.encode({'preferred_username': username}, key='').decode() - requests_mock.get(f'{settings.keycloak_url}/api-key/{api_key}', json={'access_token': access_token}) + requests_mock.get(f'{settings.url_keycloak_realm}/api-key/{api_key}', json={'access_token': access_token}) result = cli_runner.invoke(login, ['--api-key', api_key]) diff --git a/tests/app/services/user_authentication/test_token_manager.py b/tests/app/services/user_authentication/test_token_manager.py index e0e5fc10..6846ae20 100644 --- a/tests/app/services/user_authentication/test_token_manager.py +++ b/tests/app/services/user_authentication/test_token_manager.py @@ -33,7 +33,7 @@ def test_refresh_api_key_calls_keycloak_and_stores_access_token_in_config(self, manager = SrvTokenManager() access_token = jwt.encode({}, key='').decode() requests_mock.get( - f'{settings.keycloak_url}/api-key/{manager.config.api_key}', + f'{settings.url_keycloak_realm}/api-key/{manager.config.api_key}', json={'access_token': access_token}, ) From 4726cf052471df7eb438bba4d449cf89e79dfe11 Mon Sep 17 00:00:00 2001 From: zhiren Date: Tue, 19 Dec 2023 17:17:20 -0500 Subject: [PATCH 13/22] update README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a43e3926..b469ea7e 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ Command line tool that allows the user to execute data operations on the platfor 1. Create a `.env` file in the root directory of the project. 2. Sdd following two environmental varibles to the `.env` file. - `api_url`: the url that the api server is hosted on. default is `https://api.pilot.indocresearch.com/pilot` - - `keycloak_url`: thr url that the keycloak server is hosted on. default is `https://iam.pilot.indocresearch.com/realms/pilot/protocol/openid-connect` + - `keycloak_realm_url`: thr url that the keycloak server is hosted on. default is `https://iam.pilot.indocresearch.com/realms/pilot` #### Run from bundled application 1. Navigate to the appropriate directory for your system. From e788074412f79d828f00a6a2cee52b364f93e989 Mon Sep 17 00:00:00 2001 From: zhiren Date: Tue, 19 Dec 2023 17:22:57 -0500 Subject: [PATCH 14/22] unify the config usage --- app/services/user_authentication/user_login_logout.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/services/user_authentication/user_login_logout.py b/app/services/user_authentication/user_login_logout.py index 4b27f5bc..b4ca2bef 100644 --- a/app/services/user_authentication/user_login_logout.py +++ b/app/services/user_authentication/user_login_logout.py @@ -23,7 +23,7 @@ def exchange_api_key(api_key: str) -> Union[str, None]: """Exchange API Key with JWT token using Keycloak.""" - url = f'{ConfigClass.keycloak_realm_url}/api-key/{api_key}' + url = f'{AppConfig.Connections.url_keycloak_realm}/api-key/{api_key}' try: response = requests.get(url, timeout=5) response.raise_for_status() From 2d532c43c881474c6e3409501c54c0e6ab22a2d1 Mon Sep 17 00:00:00 2001 From: zhiren Date: Tue, 19 Dec 2023 17:29:44 -0500 Subject: [PATCH 15/22] fixup test cases --- tests/app/commands/test_user.py | 5 ++++- tests/app/services/user_authentication/test_token_manager.py | 2 +- tests/conftest.py | 1 + 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/app/commands/test_user.py b/tests/app/commands/test_user.py index 3d2ff335..77b4b76d 100644 --- a/tests/app/commands/test_user.py +++ b/tests/app/commands/test_user.py @@ -5,6 +5,7 @@ import jwt from app.commands.user import login +from app.configs.app_config import AppConfig from app.configs.user_config import UserConfig @@ -14,7 +15,9 @@ def test_login_command_with_api_key_option_calls_keycloak_and_stores_response_in username = fake.user_name() api_key = fake.pystr(20) access_token = jwt.encode({'preferred_username': username}, key='').decode() - requests_mock.get(f'{settings.url_keycloak_realm}/api-key/{api_key}', json={'access_token': access_token}) + requests_mock.get( + f'{AppConfig.Connections.url_keycloak_realm}/api-key/{api_key}', json={'access_token': access_token} + ) result = cli_runner.invoke(login, ['--api-key', api_key]) diff --git a/tests/app/services/user_authentication/test_token_manager.py b/tests/app/services/user_authentication/test_token_manager.py index 6846ae20..049a26d5 100644 --- a/tests/app/services/user_authentication/test_token_manager.py +++ b/tests/app/services/user_authentication/test_token_manager.py @@ -33,7 +33,7 @@ def test_refresh_api_key_calls_keycloak_and_stores_access_token_in_config(self, manager = SrvTokenManager() access_token = jwt.encode({}, key='').decode() requests_mock.get( - f'{settings.url_keycloak_realm}/api-key/{manager.config.api_key}', + f'{AppConfig.Connections.url_keycloak_realm}/api-key/{manager.config.api_key}', json={'access_token': access_token}, ) diff --git a/tests/conftest.py b/tests/conftest.py index 8c74fa46..6755b434 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -27,6 +27,7 @@ def mock_settings(monkeypatch, mocker): monkeypatch.setattr(AppConfig.Connections, 'url_download_core', 'http://url_dataset_download_core') monkeypatch.setattr(AppConfig.Connections, 'url_upload_greenroom', 'http://upload_gr') monkeypatch.setattr(AppConfig.Connections, 'url_upload_core', 'http://upload_core') + monkeypatch.setattr(AppConfig.Connections, 'url_keycloak_realm', 'http://url_keycloak_realm') monkeypatch.setattr(UserConfig, 'username', 'test-user') monkeypatch.setattr(UserConfig, 'password', 'test-password') monkeypatch.setattr(UserConfig, 'api_key', 'test-api-key') From eab988e53d11dec38f563dd3ef2051e7ac8ccf36 Mon Sep 17 00:00:00 2001 From: Color Zhan Date: Thu, 21 Dec 2023 09:36:49 -0500 Subject: [PATCH 16/22] Pilot 4264: add new command to move/rename files (#120) * add the basic logic of file move/renaming command * add new command for file move * add the test cases for file move * add the response in try exception * bumpup versions --------- Co-authored-by: zhiren --- app/commands/entry_point.py | 2 + app/commands/file.py | 28 +++++++ app/resources/custom_help.py | 2 + .../file_manager/file_move/__init__.py | 3 + .../file_move/file_move_client.py | 69 ++++++++++++++++++ app/services/output_manager/help_page.py | 3 + .../output_manager/message_handler.py | 10 +++ pyproject.toml | 2 +- tests/app/commands/test_entry_point.py | 2 + tests/app/commands/test_file.py | 18 +++++ .../file_move/test_file_move_client.py | 73 +++++++++++++++++++ 11 files changed, 211 insertions(+), 1 deletion(-) create mode 100644 app/services/file_manager/file_move/__init__.py create mode 100644 app/services/file_manager/file_move/file_move_client.py create mode 100644 tests/app/services/file_manager/file_move/test_file_move_client.py diff --git a/app/commands/entry_point.py b/app/commands/entry_point.py index 34c6b511..1770ac0a 100644 --- a/app/commands/entry_point.py +++ b/app/commands/entry_point.py @@ -21,6 +21,7 @@ from .file import file_export_manifest from .file import file_list from .file import file_metadata_download +from .file import file_move from .file import file_put from .file import file_resume @@ -74,6 +75,7 @@ def user_group(): file_group.add_command(file_download) file_group.add_command(file_resume) file_group.add_command(file_metadata_download) +file_group.add_command(file_move) project_group.add_command(project_list_all) user_group.add_command(login) user_group.add_command(logout) diff --git a/app/commands/file.py b/app/commands/file.py index 2e0e2263..76ffcd2a 100644 --- a/app/commands/file.py +++ b/app/commands/file.py @@ -16,6 +16,7 @@ from app.services.file_manager.file_list import SrvFileList from app.services.file_manager.file_manifests import SrvFileManifests from app.services.file_manager.file_metadata.file_metadata_client import FileMetaClient +from app.services.file_manager.file_move.file_move_client import FileMoveClient from app.services.file_manager.file_upload.file_upload import assemble_path from app.services.file_manager.file_upload.file_upload import resume_upload from app.services.file_manager.file_upload.file_upload import simple_upload @@ -501,3 +502,30 @@ def file_metadata_download(**kwargs): file_meta_client.download_file_metadata() message_handler.SrvOutPutHandler.metadata_download_success() + + +@click.command(name='move') +@click.argument('project_code', type=click.STRING) +@click.argument('src_item_path', type=click.STRING) +@click.argument('dest_item_path', type=click.STRING) +@click.option( + '-z', + '--zone', + default=AppConfig.Env.green_zone, + required=True, + help=file_help.file_help_page(file_help.FileHELP.FILE_MOVE_Z), + show_default=False, +) +@require_valid_token() +@doc(file_help.file_help_page(file_help.FileHELP.FILE_MOVE)) +def file_move(**kwargs): + project_code = kwargs.get('project_code') + src_item_path = kwargs.get('src_item_path') + dest_item_path = kwargs.get('dest_item_path') + zone = kwargs.get('zone') + + zone = get_zone(zone) if zone else AppConfig.Env.green_zone.lower() + file_meta_client = FileMoveClient(zone, project_code, src_item_path, dest_item_path) + file_meta_client.move_file() + + message_handler.SrvOutPutHandler.move_action_success(src_item_path, dest_item_path) diff --git a/app/resources/custom_help.py b/app/resources/custom_help.py index 03407b8f..28413446 100644 --- a/app/resources/custom_help.py +++ b/app/resources/custom_help.py @@ -60,6 +60,8 @@ class HelpPage: 'FILE_META_G': 'The location of general metadata file', 'FILE_META_A': 'The location of attribute metadata file', 'FILE_META_T': 'The location of tag metadata file', + 'FILE_MOVE': 'Move/Rename files/folders to a given Project path.', + 'FILE_MOVE_Z': 'Target Zone (i.e., core/greenroom).', }, 'config': { 'SET_CONFIG': 'Chose config file and set for cli.', diff --git a/app/services/file_manager/file_move/__init__.py b/app/services/file_manager/file_move/__init__.py new file mode 100644 index 00000000..96b7c430 --- /dev/null +++ b/app/services/file_manager/file_move/__init__.py @@ -0,0 +1,3 @@ +# Copyright (C) 2022-2023 Indoc Systems +# +# Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/app/services/file_manager/file_move/file_move_client.py b/app/services/file_manager/file_move/file_move_client.py new file mode 100644 index 00000000..706d32c0 --- /dev/null +++ b/app/services/file_manager/file_move/file_move_client.py @@ -0,0 +1,69 @@ +# Copyright (C) 2022-2023 Indoc Systems +# +# Contact Indoc Systems for any questions regarding the use of this source code. + +import app.services.output_manager.message_handler as message_handler +from app.configs.app_config import AppConfig +from app.configs.user_config import UserConfig +from app.utils.aggregated import resilient_session + + +class FileMoveClient: + """ + Summary: + A client for interacting with file metadata. currently support to download + file metadata from metadata service. + """ + + def __init__( + self, + zone: str, + project_code: str, + src_item_path: str, + dest_item_path: str, + ) -> None: + """ + Summary: + Initialize file move client. + Parameters: + zone (str): zone. + project_code (str): project code. + src_item_path (str): source item path. + dest_item_path (str): destination item path. + """ + + self.zone = zone + self.project_code = project_code + self.src_item_path = src_item_path + self.dest_item_path = dest_item_path + + self.user = UserConfig() + + def move_file(self) -> None: + """ + Summary: + Move file. + """ + + try: + url = AppConfig.Connections.url_bff + f'/v1/{self.project_code}/files' + payload = { + 'src_item_path': self.src_item_path, + 'dest_item_path': self.dest_item_path, + 'zone': self.zone, + } + headers = {'Authorization': 'Bearer ' + self.user.access_token, 'Session-ID': self.user.session_id} + + response = resilient_session().patch(url, json=payload, headers=headers, timeout=None) + response.raise_for_status() + + return response.json().get('result') + except Exception: + if response.status_code == 422: + error_message = '' + for x in response.json().get('detail'): + error_message += '\n' + x.get('msg') + else: + error_message = response.json().get('error_msg') + message_handler.SrvOutPutHandler.move_action_failed(self.src_item_path, self.dest_item_path, error_message) + exit(1) diff --git a/app/services/output_manager/help_page.py b/app/services/output_manager/help_page.py index 18b0702e..9ce631f0 100644 --- a/app/services/output_manager/help_page.py +++ b/app/services/output_manager/help_page.py @@ -79,6 +79,9 @@ class FileHELP(enum.Enum): FILE_META_A = 'FILE_META_A' FILE_META_T = 'FILE_META_T' + FILE_MOVE = 'FILE_MOVE' + FILE_MOVE_Z = 'FILE_MOVE_Z' + def file_help_page(FileHELP: FileHELP): helps = help_msg.get('file', 'default file help') diff --git a/app/services/output_manager/message_handler.py b/app/services/output_manager/message_handler.py index 990f0743..3b2e11bf 100644 --- a/app/services/output_manager/message_handler.py +++ b/app/services/output_manager/message_handler.py @@ -203,6 +203,16 @@ def cancel_metadata_download(): def metadata_download_success(): logger.succeed('Metadata download complete.') + @staticmethod + def move_action_success(src, dest): + """e.g. Move action succeed.""" + return logger.succeed(f'Successfully moved {src} to {dest}') + + @staticmethod + def move_action_failed(src, dest, error): + """e.g. Move action failed.""" + return logger.error(f'Failed to move {src} to {dest}: {error}') + @staticmethod def start_requests(): """e.g. start requests.""" diff --git a/pyproject.toml b/pyproject.toml index 5ec41f62..ca5c4d3b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "app" -version = "2.9.4" +version = "2.9.5" description = "This service is designed to support pilot platform" authors = ["Indoc Systems"] diff --git a/tests/app/commands/test_entry_point.py b/tests/app/commands/test_entry_point.py index 21c50c11..37adc20d 100644 --- a/tests/app/commands/test_entry_point.py +++ b/tests/app/commands/test_entry_point.py @@ -12,6 +12,7 @@ from app.commands.file import file_export_manifest from app.commands.file import file_list from app.commands.file import file_metadata_download +from app.commands.file import file_move from app.commands.file import file_put from app.commands.file import file_resume from app.commands.project import project_list_all @@ -64,6 +65,7 @@ def test_file_commands(user_login_true): 'download': file_download, 'resume': file_resume, 'metadata': file_metadata_download, + 'move': file_move, } file_commands_object = entry_point.commands.get('file') file_commands_object.callback() diff --git a/tests/app/commands/test_file.py b/tests/app/commands/test_file.py index 172f0bc2..9ac78b45 100644 --- a/tests/app/commands/test_file.py +++ b/tests/app/commands/test_file.py @@ -13,6 +13,7 @@ from app.commands.file import file_download from app.commands.file import file_list from app.commands.file import file_metadata_download +from app.commands.file import file_move from app.commands.file import file_put from app.commands.file import file_resume from app.services.file_manager.file_metadata.file_metadata_client import FileMetaClient @@ -332,3 +333,20 @@ def test_download_file_metadata_file_duplicate_abort(mocker, cli_runner): assert outputs == excepted_output assert donwload_metadata_mock.call_count == 0 + + +def test_file_move_success(mocker, cli_runner): + mocker.patch( + 'app.services.user_authentication.token_manager.SrvTokenManager.decode_access_token', + return_value=decoded_token(), + ) + + file_move_mock = mocker.patch( + 'app.services.file_manager.file_move.file_move_client.FileMoveClient.move_file', + return_value=None, + ) + + result = cli_runner.invoke(file_move, ['test_project', 'src_item_path', 'dest_item_path']) + outputs = result.output.split('\n') + assert outputs[0] == 'Successfully moved src_item_path to dest_item_path' + file_move_mock.assert_called_once() diff --git a/tests/app/services/file_manager/file_move/test_file_move_client.py b/tests/app/services/file_manager/file_move/test_file_move_client.py new file mode 100644 index 00000000..678ad01f --- /dev/null +++ b/tests/app/services/file_manager/file_move/test_file_move_client.py @@ -0,0 +1,73 @@ +# Copyright (C) 2022-2023 Indoc Systems +# +# Contact Indoc Systems for any questions regarding the use of this source code. + +from app.configs.app_config import AppConfig +from app.services.file_manager.file_move.file_move_client import FileMoveClient +from tests.conftest import decoded_token + + +def test_file_move_success(mocker, httpx_mock): + project_code = 'test_code' + item_info = {'result': {'id': 'test_id', 'name': 'test_name'}} + + mocker.patch( + 'app.services.user_authentication.token_manager.SrvTokenManager.decode_access_token', + return_value=decoded_token(), + ) + + httpx_mock.add_response( + url=AppConfig.Connections.url_bff + f'/v1/{project_code}/files', + method='PATCH', + json={'result': item_info}, + ) + + file_move_client = FileMoveClient('zone', project_code, 'src_item_path', 'dest_item_path') + res = file_move_client.move_file() + assert res == item_info + + +def test_file_move_error_with_permission_denied_403(mocker, httpx_mock, capfd): + project_code = 'test_code' + + mocker.patch( + 'app.services.user_authentication.token_manager.SrvTokenManager.decode_access_token', + return_value=decoded_token(), + ) + + httpx_mock.add_response( + url=AppConfig.Connections.url_bff + f'/v1/{project_code}/files', + method='PATCH', + json={'result': {}, 'error_msg': 'error_msg'}, + status_code=403, + ) + + file_move_client = FileMoveClient('zone', project_code, 'src_item_path', 'dest_item_path') + try: + file_move_client.move_file() + except SystemExit: + out, _ = capfd.readouterr() + assert out == 'Failed to move src_item_path to dest_item_path: error_msg\n' + + +def test_file_move_error_with_wrong_input_422(mocker, httpx_mock, capfd): + project_code = 'test_code' + + mocker.patch( + 'app.services.user_authentication.token_manager.SrvTokenManager.decode_access_token', + return_value=decoded_token(), + ) + + httpx_mock.add_response( + url=AppConfig.Connections.url_bff + f'/v1/{project_code}/files', + method='PATCH', + json={'detail': [{'loc': ['body', 'src_item_path'], 'msg': 'error_msg', 'type': 'value_error'}]}, + status_code=422, + ) + + file_move_client = FileMoveClient('zone', project_code, 'src_item_path', 'dest_item_path') + try: + file_move_client.move_file() + except SystemExit: + out, _ = capfd.readouterr() + assert out == 'Failed to move src_item_path to dest_item_path: \nerror_msg\n' From 9693632e73ea04201757d858b1ebbc3f0aa203b6 Mon Sep 17 00:00:00 2001 From: zhiren Date: Tue, 9 Jan 2024 11:51:02 -0500 Subject: [PATCH 17/22] update copyright to 2024 --- app/__init__.py | 2 +- app/commands/__init__.py | 2 +- app/commands/container_registry.py | 2 +- app/commands/dataset.py | 2 +- app/commands/entry_point.py | 2 +- app/commands/file.py | 2 +- app/commands/project.py | 2 +- app/commands/user.py | 2 +- app/configs/__init__.py | 2 +- app/configs/app_config.py | 2 +- app/configs/config.py | 2 +- app/configs/user_config.py | 2 +- app/models/__init__.py | 2 +- app/models/enums.py | 2 +- app/models/service_meta_class.py | 2 +- app/models/singleton.py | 2 +- app/models/upload_form.py | 2 +- app/pilotcli.py | 2 +- app/resources/custom_error.py | 2 +- app/resources/custom_help.py | 2 +- app/services/__init__.py | 2 +- .../container_registry_manager/container_registry_manager.py | 2 +- app/services/crypto/__init__.py | 2 +- app/services/crypto/crypto.py | 2 +- app/services/dataset_manager/dataset_detail.py | 2 +- app/services/dataset_manager/dataset_download.py | 2 +- app/services/dataset_manager/dataset_list.py | 2 +- app/services/dataset_manager/model.py | 2 +- app/services/file_manager/__init__.py | 2 +- app/services/file_manager/file_download/__init__.py | 2 +- app/services/file_manager/file_download/download_client.py | 2 +- app/services/file_manager/file_download/model.py | 2 +- app/services/file_manager/file_list.py | 2 +- app/services/file_manager/file_manifests.py | 2 +- app/services/file_manager/file_metadata/__init__.py | 2 +- app/services/file_manager/file_metadata/file_metadata_client.py | 2 +- app/services/file_manager/file_move/__init__.py | 2 +- app/services/file_manager/file_move/file_move_client.py | 2 +- app/services/file_manager/file_tag.py | 2 +- app/services/file_manager/file_upload/__init__.py | 2 +- app/services/file_manager/file_upload/exception.py | 2 +- app/services/file_manager/file_upload/file_upload.py | 2 +- app/services/file_manager/file_upload/models.py | 2 +- app/services/file_manager/file_upload/upload_client.py | 2 +- app/services/file_manager/file_upload/upload_validator.py | 2 +- app/services/logger_services/__init__.py | 2 +- app/services/logger_services/log_functions.py | 2 +- app/services/output_manager/__init__.py | 2 +- app/services/output_manager/error_handler.py | 2 +- app/services/output_manager/help_page.py | 2 +- app/services/output_manager/message_handler.py | 2 +- app/services/project_manager/__init__.py | 2 +- app/services/project_manager/project.py | 2 +- app/services/user_authentication/__init__.py | 2 +- app/services/user_authentication/decorator.py | 2 +- app/services/user_authentication/token_manager.py | 2 +- app/services/user_authentication/user_login_logout.py | 2 +- app/utils/__init__.py | 2 +- app/utils/aggregated.py | 2 +- tests/__init__.py | 2 +- tests/app/commands/test_dataset.py | 2 +- tests/app/commands/test_entry_point.py | 2 +- tests/app/commands/test_file.py | 2 +- tests/app/commands/test_project_command.py | 2 +- tests/app/commands/test_user.py | 2 +- tests/app/configs/__init__.py | 2 +- tests/app/configs/test_user_config.py | 2 +- tests/app/services/dataset_manager/test_dataset_detail.py | 2 +- tests/app/services/dataset_manager/test_dataset_download.py | 2 +- tests/app/services/dataset_manager/test_dataset_list.py | 2 +- .../file_manager/file_metadata/test_file_metadata_client.py | 2 +- .../services/file_manager/file_move/test_file_move_client.py | 2 +- tests/app/services/file_manager/file_upload/test_file_upload.py | 2 +- tests/app/services/file_manager/file_upload/test_model.py | 2 +- .../app/services/file_manager/file_upload/test_upload_client.py | 2 +- tests/app/services/project_manager/test_project.py | 2 +- tests/app/services/user_authentication/test_token_manager.py | 2 +- .../app/services/user_authentication/test_user_login_logout.py | 2 +- tests/app/utils/test_aggregated.py | 2 +- tests/conftest.py | 2 +- tests/fixtures/__init__.py | 2 +- tests/fixtures/fake.py | 2 +- 82 files changed, 82 insertions(+), 82 deletions(-) diff --git a/app/__init__.py b/app/__init__.py index 96b7c430..346c7325 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -1,3 +1,3 @@ -# Copyright (C) 2022-2023 Indoc Systems +# Copyright (C) 2022-2024 Indoc Systems # # Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/app/commands/__init__.py b/app/commands/__init__.py index 96b7c430..346c7325 100644 --- a/app/commands/__init__.py +++ b/app/commands/__init__.py @@ -1,3 +1,3 @@ -# Copyright (C) 2022-2023 Indoc Systems +# Copyright (C) 2022-2024 Indoc Systems # # Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/app/commands/container_registry.py b/app/commands/container_registry.py index c93296fb..db2cc38c 100644 --- a/app/commands/container_registry.py +++ b/app/commands/container_registry.py @@ -1,4 +1,4 @@ -# Copyright (C) 2022-2023 Indoc Systems +# Copyright (C) 2022-2024 Indoc Systems # # Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/app/commands/dataset.py b/app/commands/dataset.py index 141e599d..8d03caa6 100644 --- a/app/commands/dataset.py +++ b/app/commands/dataset.py @@ -1,4 +1,4 @@ -# Copyright (C) 2022-2023 Indoc Systems +# Copyright (C) 2022-2024 Indoc Systems # # Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/app/commands/entry_point.py b/app/commands/entry_point.py index 1770ac0a..57e7b0f4 100644 --- a/app/commands/entry_point.py +++ b/app/commands/entry_point.py @@ -1,4 +1,4 @@ -# Copyright (C) 2022-2023 Indoc Systems +# Copyright (C) 2022-2024 Indoc Systems # # Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/app/commands/file.py b/app/commands/file.py index 76ffcd2a..9dae8f01 100644 --- a/app/commands/file.py +++ b/app/commands/file.py @@ -1,4 +1,4 @@ -# Copyright (C) 2022-2023 Indoc Systems +# Copyright (C) 2022-2024 Indoc Systems # # Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/app/commands/project.py b/app/commands/project.py index 47b3441c..998e9f20 100644 --- a/app/commands/project.py +++ b/app/commands/project.py @@ -1,4 +1,4 @@ -# Copyright (C) 2022-2023 Indoc Systems +# Copyright (C) 2022-2024 Indoc Systems # # Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/app/commands/user.py b/app/commands/user.py index 94888abe..b97af237 100644 --- a/app/commands/user.py +++ b/app/commands/user.py @@ -1,4 +1,4 @@ -# Copyright (C) 2022-2023 Indoc Systems +# Copyright (C) 2022-2024 Indoc Systems # # Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/app/configs/__init__.py b/app/configs/__init__.py index 96b7c430..346c7325 100644 --- a/app/configs/__init__.py +++ b/app/configs/__init__.py @@ -1,3 +1,3 @@ -# Copyright (C) 2022-2023 Indoc Systems +# Copyright (C) 2022-2024 Indoc Systems # # Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/app/configs/app_config.py b/app/configs/app_config.py index 85e59928..0fa2b15d 100644 --- a/app/configs/app_config.py +++ b/app/configs/app_config.py @@ -1,4 +1,4 @@ -# Copyright (C) 2022-2023 Indoc Systems +# Copyright (C) 2022-2024 Indoc Systems # # Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/app/configs/config.py b/app/configs/config.py index 9b65018d..0a846c37 100644 --- a/app/configs/config.py +++ b/app/configs/config.py @@ -1,4 +1,4 @@ -# Copyright (C) 2022-2023 Indoc Systems +# Copyright (C) 2022-2024 Indoc Systems # # Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/app/configs/user_config.py b/app/configs/user_config.py index 7deb3672..f66e4272 100644 --- a/app/configs/user_config.py +++ b/app/configs/user_config.py @@ -1,4 +1,4 @@ -# Copyright (C) 2022-2023 Indoc Systems +# Copyright (C) 2022-2024 Indoc Systems # # Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/app/models/__init__.py b/app/models/__init__.py index 96b7c430..346c7325 100644 --- a/app/models/__init__.py +++ b/app/models/__init__.py @@ -1,3 +1,3 @@ -# Copyright (C) 2022-2023 Indoc Systems +# Copyright (C) 2022-2024 Indoc Systems # # Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/app/models/enums.py b/app/models/enums.py index d4f626e4..4f368e0a 100644 --- a/app/models/enums.py +++ b/app/models/enums.py @@ -1,4 +1,4 @@ -# Copyright (C) 2023 Indoc Systems +# Copyright (C) 2023-2024 Indoc Systems # # Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/app/models/service_meta_class.py b/app/models/service_meta_class.py index e449847a..5914fff2 100644 --- a/app/models/service_meta_class.py +++ b/app/models/service_meta_class.py @@ -1,4 +1,4 @@ -# Copyright (C) 2022-2023 Indoc Systems +# Copyright (C) 2022-2024 Indoc Systems # # Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/app/models/singleton.py b/app/models/singleton.py index c78719e8..fdad9a59 100644 --- a/app/models/singleton.py +++ b/app/models/singleton.py @@ -1,4 +1,4 @@ -# Copyright (C) 2022-2023 Indoc Systems +# Copyright (C) 2022-2024 Indoc Systems # # Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/app/models/upload_form.py b/app/models/upload_form.py index 2cea1b9f..bf898c27 100644 --- a/app/models/upload_form.py +++ b/app/models/upload_form.py @@ -1,4 +1,4 @@ -# Copyright (C) 2022-2023 Indoc Systems +# Copyright (C) 2022-2024 Indoc Systems # # Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/app/pilotcli.py b/app/pilotcli.py index a2a25f12..c567f469 100644 --- a/app/pilotcli.py +++ b/app/pilotcli.py @@ -1,4 +1,4 @@ -# Copyright (C) 2022-2023 Indoc Systems +# Copyright (C) 2022-2024 Indoc Systems # # Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/app/resources/custom_error.py b/app/resources/custom_error.py index c6233054..ec566355 100644 --- a/app/resources/custom_error.py +++ b/app/resources/custom_error.py @@ -1,4 +1,4 @@ -# Copyright (C) 2022-2023 Indoc Systems +# Copyright (C) 2022-2024 Indoc Systems # # Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/app/resources/custom_help.py b/app/resources/custom_help.py index 28413446..47ed6f6c 100644 --- a/app/resources/custom_help.py +++ b/app/resources/custom_help.py @@ -1,4 +1,4 @@ -# Copyright (C) 2022-2023 Indoc Systems +# Copyright (C) 2022-2024 Indoc Systems # # Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/app/services/__init__.py b/app/services/__init__.py index 96b7c430..346c7325 100644 --- a/app/services/__init__.py +++ b/app/services/__init__.py @@ -1,3 +1,3 @@ -# Copyright (C) 2022-2023 Indoc Systems +# Copyright (C) 2022-2024 Indoc Systems # # Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/app/services/container_registry_manager/container_registry_manager.py b/app/services/container_registry_manager/container_registry_manager.py index fff0f492..37247ef7 100644 --- a/app/services/container_registry_manager/container_registry_manager.py +++ b/app/services/container_registry_manager/container_registry_manager.py @@ -1,4 +1,4 @@ -# Copyright (C) 2022-2023 Indoc Systems +# Copyright (C) 2022-2024 Indoc Systems # # Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/app/services/crypto/__init__.py b/app/services/crypto/__init__.py index 96b7c430..346c7325 100644 --- a/app/services/crypto/__init__.py +++ b/app/services/crypto/__init__.py @@ -1,3 +1,3 @@ -# Copyright (C) 2022-2023 Indoc Systems +# Copyright (C) 2022-2024 Indoc Systems # # Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/app/services/crypto/crypto.py b/app/services/crypto/crypto.py index 40b01b9c..fb0253f3 100644 --- a/app/services/crypto/crypto.py +++ b/app/services/crypto/crypto.py @@ -1,4 +1,4 @@ -# Copyright (C) 2022-2023 Indoc Systems +# Copyright (C) 2022-2024 Indoc Systems # # Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/app/services/dataset_manager/dataset_detail.py b/app/services/dataset_manager/dataset_detail.py index daeb72c8..4fd0a2a6 100644 --- a/app/services/dataset_manager/dataset_detail.py +++ b/app/services/dataset_manager/dataset_detail.py @@ -1,4 +1,4 @@ -# Copyright (C) 2022-2023 Indoc Systems +# Copyright (C) 2022-2024 Indoc Systems # # Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/app/services/dataset_manager/dataset_download.py b/app/services/dataset_manager/dataset_download.py index fdf6ca52..6fb127d7 100644 --- a/app/services/dataset_manager/dataset_download.py +++ b/app/services/dataset_manager/dataset_download.py @@ -1,4 +1,4 @@ -# Copyright (C) 2022-2023 Indoc Systems +# Copyright (C) 2022-2024 Indoc Systems # # Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/app/services/dataset_manager/dataset_list.py b/app/services/dataset_manager/dataset_list.py index 0f128c05..29334467 100644 --- a/app/services/dataset_manager/dataset_list.py +++ b/app/services/dataset_manager/dataset_list.py @@ -1,4 +1,4 @@ -# Copyright (C) 2022-2023 Indoc Systems +# Copyright (C) 2022-2024 Indoc Systems # # Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/app/services/dataset_manager/model.py b/app/services/dataset_manager/model.py index b2c3a8c6..b39745c2 100644 --- a/app/services/dataset_manager/model.py +++ b/app/services/dataset_manager/model.py @@ -1,4 +1,4 @@ -# Copyright (C) 2022-2023 Indoc Systems +# Copyright (C) 2022-2024 Indoc Systems # # Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/app/services/file_manager/__init__.py b/app/services/file_manager/__init__.py index 96b7c430..346c7325 100644 --- a/app/services/file_manager/__init__.py +++ b/app/services/file_manager/__init__.py @@ -1,3 +1,3 @@ -# Copyright (C) 2022-2023 Indoc Systems +# Copyright (C) 2022-2024 Indoc Systems # # Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/app/services/file_manager/file_download/__init__.py b/app/services/file_manager/file_download/__init__.py index 96b7c430..346c7325 100644 --- a/app/services/file_manager/file_download/__init__.py +++ b/app/services/file_manager/file_download/__init__.py @@ -1,3 +1,3 @@ -# Copyright (C) 2022-2023 Indoc Systems +# Copyright (C) 2022-2024 Indoc Systems # # Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/app/services/file_manager/file_download/download_client.py b/app/services/file_manager/file_download/download_client.py index 79ed1677..d9f0ca4c 100644 --- a/app/services/file_manager/file_download/download_client.py +++ b/app/services/file_manager/file_download/download_client.py @@ -1,4 +1,4 @@ -# Copyright (C) 2022-2023 Indoc Systems +# Copyright (C) 2022-2024 Indoc Systems # # Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/app/services/file_manager/file_download/model.py b/app/services/file_manager/file_download/model.py index b2c3a8c6..b39745c2 100644 --- a/app/services/file_manager/file_download/model.py +++ b/app/services/file_manager/file_download/model.py @@ -1,4 +1,4 @@ -# Copyright (C) 2022-2023 Indoc Systems +# Copyright (C) 2022-2024 Indoc Systems # # Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/app/services/file_manager/file_list.py b/app/services/file_manager/file_list.py index 705bcd1d..39982cd9 100644 --- a/app/services/file_manager/file_list.py +++ b/app/services/file_manager/file_list.py @@ -1,4 +1,4 @@ -# Copyright (C) 2022-2023 Indoc Systems +# Copyright (C) 2022-2024 Indoc Systems # # Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/app/services/file_manager/file_manifests.py b/app/services/file_manager/file_manifests.py index 72b8cc42..6aede86f 100644 --- a/app/services/file_manager/file_manifests.py +++ b/app/services/file_manager/file_manifests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2022-2023 Indoc Systems +# Copyright (C) 2022-2024 Indoc Systems # # Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/app/services/file_manager/file_metadata/__init__.py b/app/services/file_manager/file_metadata/__init__.py index 96b7c430..346c7325 100644 --- a/app/services/file_manager/file_metadata/__init__.py +++ b/app/services/file_manager/file_metadata/__init__.py @@ -1,3 +1,3 @@ -# Copyright (C) 2022-2023 Indoc Systems +# Copyright (C) 2022-2024 Indoc Systems # # Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/app/services/file_manager/file_metadata/file_metadata_client.py b/app/services/file_manager/file_metadata/file_metadata_client.py index 56379d87..8fdbd77f 100644 --- a/app/services/file_manager/file_metadata/file_metadata_client.py +++ b/app/services/file_manager/file_metadata/file_metadata_client.py @@ -1,4 +1,4 @@ -# Copyright (C) 2022-2023 Indoc Systems +# Copyright (C) 2022-2024 Indoc Systems # # Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/app/services/file_manager/file_move/__init__.py b/app/services/file_manager/file_move/__init__.py index 96b7c430..346c7325 100644 --- a/app/services/file_manager/file_move/__init__.py +++ b/app/services/file_manager/file_move/__init__.py @@ -1,3 +1,3 @@ -# Copyright (C) 2022-2023 Indoc Systems +# Copyright (C) 2022-2024 Indoc Systems # # Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/app/services/file_manager/file_move/file_move_client.py b/app/services/file_manager/file_move/file_move_client.py index 706d32c0..4bd2dfb4 100644 --- a/app/services/file_manager/file_move/file_move_client.py +++ b/app/services/file_manager/file_move/file_move_client.py @@ -1,4 +1,4 @@ -# Copyright (C) 2022-2023 Indoc Systems +# Copyright (C) 2022-2024 Indoc Systems # # Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/app/services/file_manager/file_tag.py b/app/services/file_manager/file_tag.py index 428b3920..2a5db71c 100644 --- a/app/services/file_manager/file_tag.py +++ b/app/services/file_manager/file_tag.py @@ -1,4 +1,4 @@ -# Copyright (C) 2022-2023 Indoc Systems +# Copyright (C) 2022-2024 Indoc Systems # # Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/app/services/file_manager/file_upload/__init__.py b/app/services/file_manager/file_upload/__init__.py index 96b7c430..346c7325 100644 --- a/app/services/file_manager/file_upload/__init__.py +++ b/app/services/file_manager/file_upload/__init__.py @@ -1,3 +1,3 @@ -# Copyright (C) 2022-2023 Indoc Systems +# Copyright (C) 2022-2024 Indoc Systems # # Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/app/services/file_manager/file_upload/exception.py b/app/services/file_manager/file_upload/exception.py index 5c7c5b71..ac47fa51 100644 --- a/app/services/file_manager/file_upload/exception.py +++ b/app/services/file_manager/file_upload/exception.py @@ -1,4 +1,4 @@ -# Copyright (C) 2022-2023 Indoc Systems +# Copyright (C) 2022-2024 Indoc Systems # # Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/app/services/file_manager/file_upload/file_upload.py b/app/services/file_manager/file_upload/file_upload.py index ad8ea08c..15cd5219 100644 --- a/app/services/file_manager/file_upload/file_upload.py +++ b/app/services/file_manager/file_upload/file_upload.py @@ -1,4 +1,4 @@ -# Copyright (C) 2022-2023 Indoc Systems +# Copyright (C) 2022-2024 Indoc Systems # # Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/app/services/file_manager/file_upload/models.py b/app/services/file_manager/file_upload/models.py index aaf36ffb..2a0ac3d9 100644 --- a/app/services/file_manager/file_upload/models.py +++ b/app/services/file_manager/file_upload/models.py @@ -1,4 +1,4 @@ -# Copyright (C) 2022-2023 Indoc Systems +# Copyright (C) 2022-2024 Indoc Systems # # Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/app/services/file_manager/file_upload/upload_client.py b/app/services/file_manager/file_upload/upload_client.py index 0640f3e5..86dc08e3 100644 --- a/app/services/file_manager/file_upload/upload_client.py +++ b/app/services/file_manager/file_upload/upload_client.py @@ -1,4 +1,4 @@ -# Copyright (C) 2022-2023 Indoc Systems +# Copyright (C) 2022-2024 Indoc Systems # # Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/app/services/file_manager/file_upload/upload_validator.py b/app/services/file_manager/file_upload/upload_validator.py index 9dd40531..70475caf 100644 --- a/app/services/file_manager/file_upload/upload_validator.py +++ b/app/services/file_manager/file_upload/upload_validator.py @@ -1,4 +1,4 @@ -# Copyright (C) 2022-2023 Indoc Systems +# Copyright (C) 2022-2024 Indoc Systems # # Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/app/services/logger_services/__init__.py b/app/services/logger_services/__init__.py index 96b7c430..346c7325 100644 --- a/app/services/logger_services/__init__.py +++ b/app/services/logger_services/__init__.py @@ -1,3 +1,3 @@ -# Copyright (C) 2022-2023 Indoc Systems +# Copyright (C) 2022-2024 Indoc Systems # # Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/app/services/logger_services/log_functions.py b/app/services/logger_services/log_functions.py index d27af885..feab591a 100644 --- a/app/services/logger_services/log_functions.py +++ b/app/services/logger_services/log_functions.py @@ -1,4 +1,4 @@ -# Copyright (C) 2022-2023 Indoc Systems +# Copyright (C) 2022-2024 Indoc Systems # # Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/app/services/output_manager/__init__.py b/app/services/output_manager/__init__.py index 96b7c430..346c7325 100644 --- a/app/services/output_manager/__init__.py +++ b/app/services/output_manager/__init__.py @@ -1,3 +1,3 @@ -# Copyright (C) 2022-2023 Indoc Systems +# Copyright (C) 2022-2024 Indoc Systems # # Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/app/services/output_manager/error_handler.py b/app/services/output_manager/error_handler.py index 4fa9586c..a7e2c79f 100644 --- a/app/services/output_manager/error_handler.py +++ b/app/services/output_manager/error_handler.py @@ -1,4 +1,4 @@ -# Copyright (C) 2022-2023 Indoc Systems +# Copyright (C) 2022-2024 Indoc Systems # # Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/app/services/output_manager/help_page.py b/app/services/output_manager/help_page.py index 9ce631f0..bfabf8e2 100644 --- a/app/services/output_manager/help_page.py +++ b/app/services/output_manager/help_page.py @@ -1,4 +1,4 @@ -# Copyright (C) 2022-2023 Indoc Systems +# Copyright (C) 2022-2024 Indoc Systems # # Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/app/services/output_manager/message_handler.py b/app/services/output_manager/message_handler.py index 3b2e11bf..cdc57677 100644 --- a/app/services/output_manager/message_handler.py +++ b/app/services/output_manager/message_handler.py @@ -1,4 +1,4 @@ -# Copyright (C) 2022-2023 Indoc Systems +# Copyright (C) 2022-2024 Indoc Systems # # Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/app/services/project_manager/__init__.py b/app/services/project_manager/__init__.py index 96b7c430..346c7325 100644 --- a/app/services/project_manager/__init__.py +++ b/app/services/project_manager/__init__.py @@ -1,3 +1,3 @@ -# Copyright (C) 2022-2023 Indoc Systems +# Copyright (C) 2022-2024 Indoc Systems # # Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/app/services/project_manager/project.py b/app/services/project_manager/project.py index 79c08dca..ec98c503 100644 --- a/app/services/project_manager/project.py +++ b/app/services/project_manager/project.py @@ -1,4 +1,4 @@ -# Copyright (C) 2022-2023 Indoc Systems +# Copyright (C) 2022-2024 Indoc Systems # # Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/app/services/user_authentication/__init__.py b/app/services/user_authentication/__init__.py index 96b7c430..346c7325 100644 --- a/app/services/user_authentication/__init__.py +++ b/app/services/user_authentication/__init__.py @@ -1,3 +1,3 @@ -# Copyright (C) 2022-2023 Indoc Systems +# Copyright (C) 2022-2024 Indoc Systems # # Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/app/services/user_authentication/decorator.py b/app/services/user_authentication/decorator.py index 9d321060..bfb7aec6 100644 --- a/app/services/user_authentication/decorator.py +++ b/app/services/user_authentication/decorator.py @@ -1,4 +1,4 @@ -# Copyright (C) 2022-2023 Indoc Systems +# Copyright (C) 2022-2024 Indoc Systems # # Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/app/services/user_authentication/token_manager.py b/app/services/user_authentication/token_manager.py index 314adbe8..361973b3 100644 --- a/app/services/user_authentication/token_manager.py +++ b/app/services/user_authentication/token_manager.py @@ -1,4 +1,4 @@ -# Copyright (C) 2022-2023 Indoc Systems +# Copyright (C) 2022-2024 Indoc Systems # # Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/app/services/user_authentication/user_login_logout.py b/app/services/user_authentication/user_login_logout.py index b4ca2bef..550f6140 100644 --- a/app/services/user_authentication/user_login_logout.py +++ b/app/services/user_authentication/user_login_logout.py @@ -1,4 +1,4 @@ -# Copyright (C) 2022-2023 Indoc Systems +# Copyright (C) 2022-2024 Indoc Systems # # Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/app/utils/__init__.py b/app/utils/__init__.py index 96b7c430..346c7325 100644 --- a/app/utils/__init__.py +++ b/app/utils/__init__.py @@ -1,3 +1,3 @@ -# Copyright (C) 2022-2023 Indoc Systems +# Copyright (C) 2022-2024 Indoc Systems # # Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/app/utils/aggregated.py b/app/utils/aggregated.py index 912b99c0..f629c090 100644 --- a/app/utils/aggregated.py +++ b/app/utils/aggregated.py @@ -1,4 +1,4 @@ -# Copyright (C) 2022-2023 Indoc Systems +# Copyright (C) 2022-2024 Indoc Systems # # Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/tests/__init__.py b/tests/__init__.py index 96b7c430..346c7325 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,3 +1,3 @@ -# Copyright (C) 2022-2023 Indoc Systems +# Copyright (C) 2022-2024 Indoc Systems # # Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/tests/app/commands/test_dataset.py b/tests/app/commands/test_dataset.py index 3d6d5fca..e5bdd497 100644 --- a/tests/app/commands/test_dataset.py +++ b/tests/app/commands/test_dataset.py @@ -1,4 +1,4 @@ -# Copyright (C) 2022-2023 Indoc Systems +# Copyright (C) 2022-2024 Indoc Systems # # Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/tests/app/commands/test_entry_point.py b/tests/app/commands/test_entry_point.py index 37adc20d..a2a56224 100644 --- a/tests/app/commands/test_entry_point.py +++ b/tests/app/commands/test_entry_point.py @@ -1,4 +1,4 @@ -# Copyright (C) 2022-2023 Indoc Systems +# Copyright (C) 2022-2024 Indoc Systems # # Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/tests/app/commands/test_file.py b/tests/app/commands/test_file.py index 9ac78b45..b3f1df9f 100644 --- a/tests/app/commands/test_file.py +++ b/tests/app/commands/test_file.py @@ -1,4 +1,4 @@ -# Copyright (C) 2022-2023 Indoc Systems +# Copyright (C) 2022-2024 Indoc Systems # # Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/tests/app/commands/test_project_command.py b/tests/app/commands/test_project_command.py index d021dd78..d20ce449 100644 --- a/tests/app/commands/test_project_command.py +++ b/tests/app/commands/test_project_command.py @@ -1,4 +1,4 @@ -# Copyright (C) 2022-2023 Indoc Systems +# Copyright (C) 2022-2024 Indoc Systems # # Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/tests/app/commands/test_user.py b/tests/app/commands/test_user.py index 77b4b76d..d1f02d10 100644 --- a/tests/app/commands/test_user.py +++ b/tests/app/commands/test_user.py @@ -1,4 +1,4 @@ -# Copyright (C) 2023 Indoc Systems +# Copyright (C) 2023-2024 Indoc Systems # # Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/tests/app/configs/__init__.py b/tests/app/configs/__init__.py index cad2bb80..91574f61 100644 --- a/tests/app/configs/__init__.py +++ b/tests/app/configs/__init__.py @@ -1,3 +1,3 @@ -# Copyright (C) 2023 Indoc Systems +# Copyright (C) 2023-2024 Indoc Systems # # Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/tests/app/configs/test_user_config.py b/tests/app/configs/test_user_config.py index 71b73970..5d54e006 100644 --- a/tests/app/configs/test_user_config.py +++ b/tests/app/configs/test_user_config.py @@ -1,4 +1,4 @@ -# Copyright (C) 2023 Indoc Systems +# Copyright (C) 2023-2024 Indoc Systems # # Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/tests/app/services/dataset_manager/test_dataset_detail.py b/tests/app/services/dataset_manager/test_dataset_detail.py index caaa3459..04b74529 100644 --- a/tests/app/services/dataset_manager/test_dataset_detail.py +++ b/tests/app/services/dataset_manager/test_dataset_detail.py @@ -1,4 +1,4 @@ -# Copyright (C) 2022-2023 Indoc Systems +# Copyright (C) 2022-2024 Indoc Systems # # Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/tests/app/services/dataset_manager/test_dataset_download.py b/tests/app/services/dataset_manager/test_dataset_download.py index 49537bbe..05bf664a 100644 --- a/tests/app/services/dataset_manager/test_dataset_download.py +++ b/tests/app/services/dataset_manager/test_dataset_download.py @@ -1,4 +1,4 @@ -# Copyright (C) 2022-2023 Indoc Systems +# Copyright (C) 2022-2024 Indoc Systems # # Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/tests/app/services/dataset_manager/test_dataset_list.py b/tests/app/services/dataset_manager/test_dataset_list.py index 43c5be1d..1d2187fc 100644 --- a/tests/app/services/dataset_manager/test_dataset_list.py +++ b/tests/app/services/dataset_manager/test_dataset_list.py @@ -1,4 +1,4 @@ -# Copyright (C) 2022-2023 Indoc Systems +# Copyright (C) 2022-2024 Indoc Systems # # Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/tests/app/services/file_manager/file_metadata/test_file_metadata_client.py b/tests/app/services/file_manager/file_metadata/test_file_metadata_client.py index 669cd06a..48c16216 100644 --- a/tests/app/services/file_manager/file_metadata/test_file_metadata_client.py +++ b/tests/app/services/file_manager/file_metadata/test_file_metadata_client.py @@ -1,4 +1,4 @@ -# Copyright (C) 2022-2023 Indoc Systems +# Copyright (C) 2022-2024 Indoc Systems # # Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/tests/app/services/file_manager/file_move/test_file_move_client.py b/tests/app/services/file_manager/file_move/test_file_move_client.py index 678ad01f..b6c52b1d 100644 --- a/tests/app/services/file_manager/file_move/test_file_move_client.py +++ b/tests/app/services/file_manager/file_move/test_file_move_client.py @@ -1,4 +1,4 @@ -# Copyright (C) 2022-2023 Indoc Systems +# Copyright (C) 2022-2024 Indoc Systems # # Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/tests/app/services/file_manager/file_upload/test_file_upload.py b/tests/app/services/file_manager/file_upload/test_file_upload.py index cff579fa..17b3e474 100644 --- a/tests/app/services/file_manager/file_upload/test_file_upload.py +++ b/tests/app/services/file_manager/file_upload/test_file_upload.py @@ -1,4 +1,4 @@ -# Copyright (C) 2022-2023 Indoc Systems +# Copyright (C) 2022-2024 Indoc Systems # # Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/tests/app/services/file_manager/file_upload/test_model.py b/tests/app/services/file_manager/file_upload/test_model.py index 2a53c17f..4f23a822 100644 --- a/tests/app/services/file_manager/file_upload/test_model.py +++ b/tests/app/services/file_manager/file_upload/test_model.py @@ -1,4 +1,4 @@ -# Copyright (C) 2022-2023 Indoc Systems +# Copyright (C) 2022-2024 Indoc Systems # # Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/tests/app/services/file_manager/file_upload/test_upload_client.py b/tests/app/services/file_manager/file_upload/test_upload_client.py index 53c3cd76..21454ee3 100644 --- a/tests/app/services/file_manager/file_upload/test_upload_client.py +++ b/tests/app/services/file_manager/file_upload/test_upload_client.py @@ -1,4 +1,4 @@ -# Copyright (C) 2022-2023 Indoc Systems +# Copyright (C) 2022-2024 Indoc Systems # # Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/tests/app/services/project_manager/test_project.py b/tests/app/services/project_manager/test_project.py index d7df8fdf..91747282 100644 --- a/tests/app/services/project_manager/test_project.py +++ b/tests/app/services/project_manager/test_project.py @@ -1,4 +1,4 @@ -# Copyright (C) 2022-2023 Indoc Systems +# Copyright (C) 2022-2024 Indoc Systems # # Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/tests/app/services/user_authentication/test_token_manager.py b/tests/app/services/user_authentication/test_token_manager.py index 049a26d5..cc6015ce 100644 --- a/tests/app/services/user_authentication/test_token_manager.py +++ b/tests/app/services/user_authentication/test_token_manager.py @@ -1,4 +1,4 @@ -# Copyright (C) 2023 Indoc Systems +# Copyright (C) 2023-2024 Indoc Systems # # Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/tests/app/services/user_authentication/test_user_login_logout.py b/tests/app/services/user_authentication/test_user_login_logout.py index 5af15cdf..8fbc860f 100644 --- a/tests/app/services/user_authentication/test_user_login_logout.py +++ b/tests/app/services/user_authentication/test_user_login_logout.py @@ -1,4 +1,4 @@ -# Copyright (C) 2022-2023 Indoc Systems +# Copyright (C) 2022-2024 Indoc Systems # # Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/tests/app/utils/test_aggregated.py b/tests/app/utils/test_aggregated.py index d449a5f4..d7a8aa4c 100644 --- a/tests/app/utils/test_aggregated.py +++ b/tests/app/utils/test_aggregated.py @@ -1,4 +1,4 @@ -# Copyright (C) 2022-2023 Indoc Systems +# Copyright (C) 2022-2024 Indoc Systems # # Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/tests/conftest.py b/tests/conftest.py index 6755b434..1a4a444d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,4 +1,4 @@ -# Copyright (C) 2022-2023 Indoc Systems +# Copyright (C) 2022-2024 Indoc Systems # # Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/tests/fixtures/__init__.py b/tests/fixtures/__init__.py index cad2bb80..91574f61 100644 --- a/tests/fixtures/__init__.py +++ b/tests/fixtures/__init__.py @@ -1,3 +1,3 @@ -# Copyright (C) 2023 Indoc Systems +# Copyright (C) 2023-2024 Indoc Systems # # Contact Indoc Systems for any questions regarding the use of this source code. diff --git a/tests/fixtures/fake.py b/tests/fixtures/fake.py index 7db1d77e..e8834908 100644 --- a/tests/fixtures/fake.py +++ b/tests/fixtures/fake.py @@ -1,4 +1,4 @@ -# Copyright (C) 2023 Indoc Systems +# Copyright (C) 2023-2024 Indoc Systems # # Contact Indoc Systems for any questions regarding the use of this source code. From 127d1af0685d53334c27f8d58dc2922fc2106f58 Mon Sep 17 00:00:00 2001 From: zhiren Date: Tue, 9 Jan 2024 11:51:28 -0500 Subject: [PATCH 18/22] bumpup the version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index ca5c4d3b..ec5fba78 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "app" -version = "2.9.5" +version = "2.9.6" description = "This service is designed to support pilot platform" authors = ["Indoc Systems"] From 5865f9d8060c449b2214639f6dca3b5728a31ebe Mon Sep 17 00:00:00 2001 From: Color Zhan Date: Fri, 19 Jan 2024 09:05:40 -0500 Subject: [PATCH 19/22] PILOT-4188: update the name of resumable upload manifest (#123) * add the new name in git ignore * add the new name in git ignore * remove the resumable logs after successful uploading * fix up test cases * update the logic of removing output file into a dedicate function --------- Co-authored-by: zhiren --- .gitignore | 1 + app/commands/file.py | 11 ++++++++--- app/resources/custom_error.py | 2 +- app/resources/custom_help.py | 2 +- app/utils/aggregated.py | 11 +++++++++++ pyproject.toml | 2 +- tests/app/commands/test_file.py | 15 +++++++++------ 7 files changed, 32 insertions(+), 12 deletions(-) diff --git a/.gitignore b/.gitignore index e33d6c8d..fbc35fb3 100644 --- a/.gitignore +++ b/.gitignore @@ -155,4 +155,5 @@ integration_tests # cli manifest data ./manifest.json manifest.json +resumable_upload_log.json test diff --git a/app/commands/file.py b/app/commands/file.py index 9dae8f01..2e821aa8 100644 --- a/app/commands/file.py +++ b/app/commands/file.py @@ -30,6 +30,7 @@ from app.utils.aggregated import get_file_info_by_geid from app.utils.aggregated import get_zone from app.utils.aggregated import identify_target_folder +from app.utils.aggregated import remove_the_output_file from app.utils.aggregated import search_item @@ -110,9 +111,9 @@ def cli(): @click.option( '--output-path', '-o', - default='./manifest.json', + default='./resumable_upload_log.json', required=False, - help='The output path for the manifest file of resumable upload.', + help='The output path for the manifest file of resumable upload log.', show_default=True, ) @doc(file_help.file_help_page(file_help.FileHELP.FILE_UPLOAD)) @@ -226,6 +227,8 @@ def file_put(**kwargs): # noqa: C901 srv_manifest.attach_manifest(attribute, item_ids[0], zone) if attribute else None message_handler.SrvOutPutHandler.all_file_uploaded() + remove_the_output_file(output_path) + @click.command(name='resume') @click.option( @@ -241,7 +244,7 @@ def file_put(**kwargs): # noqa: C901 '-r', default=None, required=True, - help='The manifest file for resumable upload', + help='The resumable upload log file', show_default=True, ) @doc(file_help.file_help_page(file_help.FileHELP.FILE_RESUME)) @@ -278,6 +281,8 @@ def file_resume(**kwargs): # noqa: C901 srv_manifest.attach_manifest(attribute, item_id, zone) if attribute else None message_handler.SrvOutPutHandler.all_file_uploaded() + remove_the_output_file(resumable_manifest_file) + def validate_upload_event(event): """validate upload request, raise error when filed.""" diff --git a/app/resources/custom_error.py b/app/resources/custom_error.py index ec566355..8f862a6b 100644 --- a/app/resources/custom_error.py +++ b/app/resources/custom_error.py @@ -63,7 +63,7 @@ class Error: 'The upload ID may be invalid, or the upload may have been aborted or completed.' ), 'MANIFEST_OF_FOLDER_FILE_EXIST': ( - 'The manifest file of folder %s already exist. ' 'Do you want to overwrite the existing manifest file?' + 'The manifest file of folder %s already exist. ' 'To continue and overwrite the resumable upload log, enter' ), 'INVALID_CHUNK_UPLOAD': ( '\nThe chunk number %d is not the same with previous etag.\n' diff --git a/app/resources/custom_help.py b/app/resources/custom_help.py index 47ed6f6c..ba86ddd9 100644 --- a/app/resources/custom_help.py +++ b/app/resources/custom_help.py @@ -36,7 +36,7 @@ class HelpPage: 'FILE_LIST': 'List files and folders inside a given Project/folder.', 'FILE_SYNC': 'Download files/folders from a given Project/folder/file in core zone.', 'FILE_UPLOAD': 'Upload files/folders to a given Project path.', - 'FILE_RESUME': 'Resume the upload process with given manifest file.', + 'FILE_RESUME': 'Resume the upload process with a resumable upload log.', 'FILE_Z': 'Target Zone (i.e., core/greenroom).', 'FILE_ATTRIBUTE_P': 'Project Code', 'FILE_ATTRIBUTE_N': 'Attribute Template Name', diff --git a/app/utils/aggregated.py b/app/utils/aggregated.py index f629c090..965f682b 100644 --- a/app/utils/aggregated.py +++ b/app/utils/aggregated.py @@ -12,6 +12,7 @@ import httpx import requests +import app.services.logger_services.log_functions as logger from app.configs.app_config import AppConfig from app.configs.config import ConfigClass from app.configs.user_config import UserConfig @@ -154,3 +155,13 @@ def batch_generator(iterable: List[Any], batch_size=1): max_size = len(iterable) for start_index in range(0, max_size, batch_size): yield iterable[start_index : min(start_index + batch_size, max_size)] + + +def remove_the_output_file(filepath: str) -> None: + """Remove the output file after each successful operation to avoid confusion.""" + try: + os.remove(filepath) + except FileNotFoundError: + pass + except OSError: + logger.warning(f'Unable to remove "{filepath}".') diff --git a/pyproject.toml b/pyproject.toml index ec5fba78..606b776e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "app" -version = "2.9.6" +version = "2.9.7" description = "This service is designed to support pilot platform" authors = ["Indoc Systems"] diff --git a/tests/app/commands/test_file.py b/tests/app/commands/test_file.py index b3f1df9f..3b6adb0e 100644 --- a/tests/app/commands/test_file.py +++ b/tests/app/commands/test_file.py @@ -62,6 +62,7 @@ def test_resumable_upload_command_success(mocker, cli_runner): mocker.patch('builtins.open', mocked_open_data) mocker.patch('json.load', return_value={'file_objects': {'test_item_id': {'file_name': 'test.json'}}, 'zone': 1}) mocker.patch('app.commands.file.resume_upload', return_value=None) + mocker.patch('os.remove', return_value=None) result = cli_runner.invoke(file_resume, ['--resumable-manifest', 'test.json', '--thread', 1]) assert result.exit_code == 0 @@ -85,6 +86,8 @@ def test_resumable_upload_command_with_file_attribute_success(mocker, cli_runner 'app.services.file_manager.file_manifests.SrvFileManifests.attach_manifest', return_value=None ) + mocker.patch('os.remove', return_value=None) + result = cli_runner.invoke(file_resume, ['--resumable-manifest', 'test.json', '--thread', 1]) assert result.exit_code == 0 attribute_fun_mock.assert_called_once() @@ -253,13 +256,13 @@ def test_download_file_metadata_file_duplicate_success(mocker, cli_runner): with runner.isolated_filesystem(): file_meta_client = FileMetaClient('zone', file_path, metadata_loc, metadata_loc, metadata_loc) # create all file to make duplicationn - makedirs(dirname(file_meta_client.general_location), exist_ok=True) + makedirs(dirname(file_meta_client.general_location), exist_ok=True, mode=0o0700) with open(file_meta_client.general_location, 'w') as f: f.write(file_meta_client.general_location) - makedirs(dirname(file_meta_client.attribute_location), exist_ok=True) + makedirs(dirname(file_meta_client.attribute_location), exist_ok=True, mode=0o0700) with open(file_meta_client.attribute_location, 'w') as f: f.write(file_meta_client.attribute_location) - makedirs(dirname(file_meta_client.tag_location), exist_ok=True) + makedirs(dirname(file_meta_client.tag_location), exist_ok=True, mode=0o0700) with open(file_meta_client.tag_location, 'w') as f: f.write(file_meta_client.tag_location) @@ -303,13 +306,13 @@ def test_download_file_metadata_file_duplicate_abort(mocker, cli_runner): with runner.isolated_filesystem(): file_meta_client = FileMetaClient('zone', file_path, metadata_loc, metadata_loc, metadata_loc) # create all file to make duplicationn - makedirs(dirname(file_meta_client.general_location), exist_ok=True) + makedirs(dirname(file_meta_client.general_location), exist_ok=True, mode=0o0700) with open(file_meta_client.general_location, 'w') as f: f.write(file_meta_client.general_location) - makedirs(dirname(file_meta_client.attribute_location), exist_ok=True) + makedirs(dirname(file_meta_client.attribute_location), exist_ok=True, mode=0o0700) with open(file_meta_client.attribute_location, 'w') as f: f.write(file_meta_client.attribute_location) - makedirs(dirname(file_meta_client.tag_location), exist_ok=True) + makedirs(dirname(file_meta_client.tag_location), exist_ok=True, mode=0o0700) with open(file_meta_client.tag_location, 'w') as f: f.write(file_meta_client.tag_location) From e52935795cf57377900d0b2fce1437560dbd65b0 Mon Sep 17 00:00:00 2001 From: Color Zhan Date: Mon, 29 Jan 2024 16:07:16 -0500 Subject: [PATCH 20/22] Pilot 4414: add a new logic to allow move/rename action to create non-existing parent folders (#124) * add a new logic to allow cli to create non-exist folder when moving or renaming * add a new boolean option to skip prompt confirmation in pipeline * add the test cases for file move client * add a test case to check the error handling of an aggregated function * bumpup versions --------- Co-authored-by: zhiren --- app/commands/file.py | 12 ++- app/resources/custom_help.py | 1 + .../file_move/file_move_client.py | 76 ++++++++++++++++++- app/services/output_manager/help_page.py | 1 + .../output_manager/message_handler.py | 5 ++ app/utils/aggregated.py | 28 +++++++ pyproject.toml | 2 +- .../file_move/test_file_move_client.py | 45 +++++++++++ tests/app/utils/test_aggregated.py | 22 ++++++ 9 files changed, 189 insertions(+), 3 deletions(-) diff --git a/app/commands/file.py b/app/commands/file.py index 2e821aa8..f38b0f13 100644 --- a/app/commands/file.py +++ b/app/commands/file.py @@ -521,6 +521,15 @@ def file_metadata_download(**kwargs): help=file_help.file_help_page(file_help.FileHELP.FILE_MOVE_Z), show_default=False, ) +@click.option( + '-y', + '--yes', + default=False, + required=False, + is_flag=True, + help=file_help.file_help_page(file_help.FileHELP.FILE_MOVE_Y), + show_default=True, +) @require_valid_token() @doc(file_help.file_help_page(file_help.FileHELP.FILE_MOVE)) def file_move(**kwargs): @@ -528,9 +537,10 @@ def file_move(**kwargs): src_item_path = kwargs.get('src_item_path') dest_item_path = kwargs.get('dest_item_path') zone = kwargs.get('zone') + skip_confirm = kwargs.get('yes') zone = get_zone(zone) if zone else AppConfig.Env.green_zone.lower() - file_meta_client = FileMoveClient(zone, project_code, src_item_path, dest_item_path) + file_meta_client = FileMoveClient(zone, project_code, src_item_path, dest_item_path, skip_confirm=skip_confirm) file_meta_client.move_file() message_handler.SrvOutPutHandler.move_action_success(src_item_path, dest_item_path) diff --git a/app/resources/custom_help.py b/app/resources/custom_help.py index ba86ddd9..75e77563 100644 --- a/app/resources/custom_help.py +++ b/app/resources/custom_help.py @@ -62,6 +62,7 @@ class HelpPage: 'FILE_META_T': 'The location of tag metadata file', 'FILE_MOVE': 'Move/Rename files/folders to a given Project path.', 'FILE_MOVE_Z': 'Target Zone (i.e., core/greenroom).', + 'FILE_MOVE_Y': 'Skip the prompt confirmation and create non-existing folders.', }, 'config': { 'SET_CONFIG': 'Chose config file and set for cli.', diff --git a/app/services/file_manager/file_move/file_move_client.py b/app/services/file_manager/file_move/file_move_client.py index 4bd2dfb4..41833891 100644 --- a/app/services/file_manager/file_move/file_move_client.py +++ b/app/services/file_manager/file_move/file_move_client.py @@ -2,10 +2,22 @@ # # Contact Indoc Systems for any questions regarding the use of this source code. +import uuid +from sys import exit + +import click +from click import Abort + import app.services.output_manager.message_handler as message_handler from app.configs.app_config import AppConfig from app.configs.user_config import UserConfig +from app.services.output_manager.error_handler import ECustomizedError +from app.services.output_manager.error_handler import SrvErrorHandler +from app.services.output_manager.error_handler import customized_error_msg +from app.services.user_authentication.decorator import require_valid_token +from app.utils.aggregated import check_item_duplication from app.utils.aggregated import resilient_session +from app.utils.aggregated import search_item class FileMoveClient: @@ -21,6 +33,7 @@ def __init__( project_code: str, src_item_path: str, dest_item_path: str, + skip_confirm: bool = False, ) -> None: """ Summary: @@ -32,19 +45,80 @@ def __init__( dest_item_path (str): destination item path. """ - self.zone = zone + self.zone = {'greenroom': 0, 'core': 1}.get(zone) self.project_code = project_code self.src_item_path = src_item_path self.dest_item_path = dest_item_path + self.skip_confirm = skip_confirm self.user = UserConfig() + def create_object_path_if_not_exist(self, folder_path: str) -> dict: + """Create object path is not on platfrom. + + it will create every non-exist folder along path. + """ + + path_list = folder_path.split('/') + # first check every folder in path exist or not + # the loop start with index 1 since we assume cli will not + # create any name folder or project folder + check_list = [] + for index in range(1, len(path_list) - 1): + check_list.append('/'.join(path_list[: index + 1])) + if len(check_list) == 0: + return + + # confirm if user want to create folder or not + exist_path = check_item_duplication(check_list, self.zone, self.project_code) + not_exist_path = sorted(set(check_list) - set(exist_path)) + if not_exist_path: + try: + if not self.skip_confirm: + click.confirm(customized_error_msg(ECustomizedError.CREATE_FOLDER_IF_NOT_EXIST), abort=True) + except Abort: + message_handler.SrvOutPutHandler.move_cancelled() + exit(1) + else: + return + + # get the current exist parent folder for reference + exist_parent = not_exist_path[0].rsplit('/', 1)[0] + exist_parent_item = search_item(self.project_code, self.zone, exist_parent).get('result') + exist_parent_id = exist_parent_item.get('id') + to_create = {'folders': [], 'parent_id': exist_parent_id} + for path in not_exist_path: + parent_path, folder_name = path.rsplit('/', 1) + current_item_id = str(uuid.uuid4()) + to_create['folders'].append( + { + 'name': folder_name, + 'parent': exist_parent_id, + 'parent_path': parent_path, + 'container_code': self.project_code, + 'container_type': 'project', + 'zone': self.zone, + 'item_id': current_item_id, + } + ) + exist_parent_id = current_item_id + + url = AppConfig.Connections.url_bff + '/v1/folders/batch' + headers = {'Authorization': 'Bearer ' + UserConfig().access_token} + response = resilient_session().post(url, json=to_create, headers=headers) + if response.status_code != 200: + SrvErrorHandler.default_handle(response.text, True) + return response.json().get('result') + + @require_valid_token() def move_file(self) -> None: """ Summary: Move file. """ + self.create_object_path_if_not_exist(self.dest_item_path) + try: url = AppConfig.Connections.url_bff + f'/v1/{self.project_code}/files' payload = { diff --git a/app/services/output_manager/help_page.py b/app/services/output_manager/help_page.py index bfabf8e2..32cb60ac 100644 --- a/app/services/output_manager/help_page.py +++ b/app/services/output_manager/help_page.py @@ -81,6 +81,7 @@ class FileHELP(enum.Enum): FILE_MOVE = 'FILE_MOVE' FILE_MOVE_Z = 'FILE_MOVE_Z' + FILE_MOVE_Y = 'FILE_MOVE_Y' def file_help_page(FileHELP: FileHELP): diff --git a/app/services/output_manager/message_handler.py b/app/services/output_manager/message_handler.py index cdc57677..501d0dc3 100644 --- a/app/services/output_manager/message_handler.py +++ b/app/services/output_manager/message_handler.py @@ -208,6 +208,11 @@ def move_action_success(src, dest): """e.g. Move action succeed.""" return logger.succeed(f'Successfully moved {src} to {dest}') + @staticmethod + def move_cancelled(): + """e.g. Move action cancelled.""" + return logger.warning('Move cancelled.') + @staticmethod def move_action_failed(src, dest, error): """e.g. Move action failed.""" diff --git a/app/utils/aggregated.py b/app/utils/aggregated.py index 965f682b..95ac1a97 100644 --- a/app/utils/aggregated.py +++ b/app/utils/aggregated.py @@ -74,6 +74,34 @@ def get_file_info_by_geid(geid: list): return res.json()['result'] +@require_valid_token() +def check_item_duplication(item_list: List[str], zone: int, project_code: str) -> List[str]: + ''' + Summary: + Check if the item already exists in the project in batch. + Parameters: + - item_list: list of item path to check + - zone: zone of the project + - project_code: project code + Returns: + - list of item path that already exists in the project + ''' + + url = AppConfig.Connections.url_base + '/portal/v1/files/exists' + headers = {'Authorization': 'Bearer ' + UserConfig().access_token} + payload = { + 'locations': item_list, + 'container_code': project_code, + 'container_type': 'project', + 'zone': zone, + } + response = resilient_session().post(url, json=payload, headers=headers) + if response.status_code != 200: + SrvErrorHandler.default_handle(response.text, True) + + return response.json().get('result') + + def fit_terminal_width(string_to_format): string_to_format = string_to_format.rsplit('...') current_len = 0 diff --git a/pyproject.toml b/pyproject.toml index 606b776e..aba8fe43 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "app" -version = "2.9.7" +version = "2.9.8" description = "This service is designed to support pilot platform" authors = ["Indoc Systems"] diff --git a/tests/app/services/file_manager/file_move/test_file_move_client.py b/tests/app/services/file_manager/file_move/test_file_move_client.py index b6c52b1d..e22776e6 100644 --- a/tests/app/services/file_manager/file_move/test_file_move_client.py +++ b/tests/app/services/file_manager/file_move/test_file_move_client.py @@ -2,6 +2,8 @@ # # Contact Indoc Systems for any questions regarding the use of this source code. +import pytest + from app.configs.app_config import AppConfig from app.services.file_manager.file_move.file_move_client import FileMoveClient from tests.conftest import decoded_token @@ -71,3 +73,46 @@ def test_file_move_error_with_wrong_input_422(mocker, httpx_mock, capfd): except SystemExit: out, _ = capfd.readouterr() assert out == 'Failed to move src_item_path to dest_item_path: \nerror_msg\n' + + +@pytest.mark.parametrize('skip_confirmation', [True, False]) +def test_move_file_dest_parent_not_exist_success(mocker, httpx_mock, skip_confirmation): + project_code = 'test_code' + item_info = {'result': {'id': 'test_id', 'name': 'test_name'}} + + mocker.patch( + 'app.services.user_authentication.token_manager.SrvTokenManager.decode_access_token', + return_value=decoded_token(), + ) + + # mock duplicated item + mocker.patch( + 'app.services.file_manager.file_move.file_move_client.check_item_duplication', + return_value=[], + ) + mocker.patch( + 'app.services.file_manager.file_move.file_move_client.search_item', + return_value={'result': {'id': 'test_id', 'name': 'test_name'}}, + ) + click_mocker = mocker.patch('app.services.file_manager.file_move.file_move_client.click.confirm', return_value=None) + httpx_mock.add_response( + url=AppConfig.Connections.url_bff + '/v1/folders/batch', + method='POST', + json={'result': []}, + ) + + httpx_mock.add_response( + url=AppConfig.Connections.url_bff + f'/v1/{project_code}/files', + method='PATCH', + json={'result': item_info}, + ) + + file_move_client = FileMoveClient( + 'zone', project_code, 'src_item_path', 'dest_item_path/test_folder/test_file', skip_confirm=skip_confirmation + ) + res = file_move_client.move_file() + assert res == item_info + if not skip_confirmation: + click_mocker.assert_called_once() + else: + click_mocker.assert_not_called() diff --git a/tests/app/utils/test_aggregated.py b/tests/app/utils/test_aggregated.py index d7a8aa4c..34a02646 100644 --- a/tests/app/utils/test_aggregated.py +++ b/tests/app/utils/test_aggregated.py @@ -4,7 +4,10 @@ import pytest +from app.configs.app_config import AppConfig +from app.utils.aggregated import check_item_duplication from app.utils.aggregated import search_item +from tests.conftest import decoded_token test_project_code = 'testproject' @@ -100,3 +103,22 @@ def test_search_file_error_handling_with_401(requests_mock, mocker, capsys): search_item(test_project_code, 'zone', 'folder_relative_path', 'project') out, _ = capsys.readouterr() assert out.rstrip() == 'Your login session has expired. Please try again or log in again.' + + +def test_check_duplicate_fail_with_error_code(httpx_mock, mocker, capsys): + mocker.patch( + 'app.services.user_authentication.token_manager.SrvTokenManager.decode_access_token', + return_value=decoded_token(), + ) + + httpx_mock.add_response( + url=AppConfig.Connections.url_base + '/portal/v1/files/exists', + method='POST', + json={'error': 'internal server error'}, + status_code=500, + ) + + with pytest.raises(SystemExit): + check_item_duplication(['test_path'], 0, 'test_project_code') + out, _ = capsys.readouterr() + assert out.rstrip() == '{"error": "internal server error"}' From 4f55e2874ddfbb299990d5c988757c34fe8a4db5 Mon Sep 17 00:00:00 2001 From: zhiren Date: Wed, 21 Feb 2024 14:57:20 -0500 Subject: [PATCH 21/22] bumup version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index aba8fe43..07a4eeb4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "app" -version = "2.9.8" +version = "2.10.0" description = "This service is designed to support pilot platform" authors = ["Indoc Systems"] From b47255aa54df5a63eea4b1357d13725defe8411c Mon Sep 17 00:00:00 2001 From: zhiren Date: Wed, 21 Feb 2024 15:25:21 -0500 Subject: [PATCH 22/22] bump up to correct version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 07a4eeb4..5fa6eaeb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "app" -version = "2.10.0" +version = "2.9.9" description = "This service is designed to support pilot platform" authors = ["Indoc Systems"]