From 885fd9d99671db130425c56340f61e3df7939f5c Mon Sep 17 00:00:00 2001 From: Color Zhan Date: Thu, 25 Apr 2024 11:17:29 -0400 Subject: [PATCH 1/9] Pilot 4952: Update cli listing command to allow user list specific files/folders under given root level folder (#135) * add [p] prefix for project folder when listing items under project * group itemprefix class with itemtype class * add double quotation when item name contains space * change enum type PROJECTFOLDER to SHAREDFOLDER * replace PROJECTFOLDER with SHAREDFOLDER * fixup test cases * fixup logic for different prefix * add prefix for name folder * bump up version to 3.0.0 * fixup test cases * fixup test cases * replace hardcoded string with enum value in test cases * remove the reference of project_folder * fixup test cases * update error message with new top level folder structure --------- Co-authored-by: zhiren Former-commit-id: 61f8a8b10f03f9da4418a563375dbbd16af9cd57 --- app/models/item.py | 11 ++- app/resources/custom_error.py | 2 +- app/resources/custom_help.py | 6 +- app/services/file_manager/file_list.py | 21 ++---- app/services/output_manager/error_handler.py | 2 +- app/utils/aggregated.py | 16 ++-- pyproject.toml | 2 +- tests/app/commands/test_file.py | 74 ++++++++----------- .../file_upload/test_file_upload.py | 32 ++++++-- tests/app/utils/test_aggregated.py | 20 ++++- 10 files changed, 94 insertions(+), 92 deletions(-) diff --git a/app/models/item.py b/app/models/item.py index a7a91029..db1068d8 100644 --- a/app/models/item.py +++ b/app/models/item.py @@ -9,9 +9,10 @@ class ItemType(str, Enum): """The class to reflect the type of item in database.""" FILE = 'file' - Folder = 'folder' + FOLDER = 'folder' NAMEFOLDER = 'name_folder' SHAREDFOLDER = 'project_folder' + ROOTFOLDER = 'root_folder' @classmethod def get_type_from_keyword(self, keyword: str): @@ -22,17 +23,19 @@ def get_type_from_keyword(self, keyword: str): """ alternative_mapping = { - 'projectfolder': self.SHAREDFOLDER, + 'shared': self.SHAREDFOLDER, + 'users': self.NAMEFOLDER, } - return alternative_mapping.get(keyword, self.NAMEFOLDER) + return alternative_mapping.get(keyword, '') def get_prefix_by_type(self) -> str: """Get the prefix for the folder type.""" prefix = { - self.NAMEFOLDER: '', + self.NAMEFOLDER: 'users/', self.SHAREDFOLDER: 'shared/', + self.ROOTFOLDER: '', } return prefix.get(self.value, '') diff --git a/app/resources/custom_error.py b/app/resources/custom_error.py index 893d676a..38d91406 100644 --- a/app/resources/custom_error.py +++ b/app/resources/custom_error.py @@ -92,7 +92,7 @@ class Error: 'INVALID_ACTION': 'Invalid action: %s', 'DUPLICATE_TAG_ERROR': 'Cannot add duplicate tags', 'INVALID_FOLDER': 'Provided folder does not exist', - 'INVALID_NAMEFOLDER': 'User name folder is missing or provided user name folder does not exist', + 'INVALID_PROJECT_PATH': 'root folder is missing or provided name/shared folder does not exist', 'INVALID_DOWNLOAD': 'Invalid download, file/folder not exist or folder is empty: %s', # file metadata related error 'LOCAL_METADATA_FILE_EXISTS': 'Following metadata file already exists in the local directory: ', diff --git a/app/resources/custom_help.py b/app/resources/custom_help.py index 3e4b17e6..f0edb329 100644 --- a/app/resources/custom_help.py +++ b/app/resources/custom_help.py @@ -43,11 +43,7 @@ class HelpPage: 'FILE_SYNC_ZIP': 'Download files as a zip.', 'FILE_SYNC_I': 'Enable downloading by geid.', 'FILE_SYNC_Z': 'Target Zone (i.e., core/greenroom).', - 'FILE_UPLOAD_P': ( - 'Project folder path starting from Project Code(i.e. /user/folder). ' - 'A new key word `projectfolder` is required to specify project folder(i.e. ' - '/projectfolder/folder1)' - ), + 'FILE_UPLOAD_P': 'Project folder path starting from Project Code(i.e. /users/admin/folder). ', 'FILE_UPLOAD_A': 'Add attributes to the file using a File Attribute Template.', 'FILE_UPLOAD_T': 'Add tags to the file using a Tag file.', 'FILE_UPLOAD_M': 'The message used to comment on the purpose of uploading your processed file.', diff --git a/app/services/file_manager/file_list.py b/app/services/file_manager/file_list.py index 2131903a..d543d952 100644 --- a/app/services/file_manager/file_list.py +++ b/app/services/file_manager/file_list.py @@ -15,7 +15,6 @@ from app.services.output_manager.error_handler import SrvErrorHandler from app.services.user_authentication.decorator import require_valid_token from app.utils.aggregated import fit_terminal_width -from app.utils.aggregated import search_item class SrvFileList(metaclass=MetaService): @@ -23,18 +22,14 @@ class SrvFileList(metaclass=MetaService): @require_valid_token() def list_files(self, paths, zone, page, page_size): + # path is formatted as // + # split the path in to project_code, root_folder, and folder1 project_path = paths.strip('/').split('/') - project_code = project_path[0] + project_code, source_type = project_path[0], 'project' folder_rel_path = '/'.join(project_path[1:]) - if len(project_path) == 1: - source_type = 'project' - else: - source_type = 'project' - res = search_item(project_code, zone, folder_rel_path) - parent_folder = res.get('result') - # if the target folder is project folder add the default path - if parent_folder.get('type') == 'project_folder': - folder_rel_path = 'shared/' + folder_rel_path + if len(project_path) > 1: + root_folder = ItemType.get_type_from_keyword(project_path[1]) + folder_rel_path = folder_rel_path.replace(project_path[1], root_folder.get_prefix_by_type()[:-1], 1) # now query the backend to get the file list get_url = AppConfig.Connections.url_bff + f'/v1/{project_code}/files/query' @@ -68,10 +63,6 @@ def list_files(self, paths, zone, page, page_size): if item_type == ItemType.FILE: files = files + f.get('name') + ' ...' else: - # add [p] in front of the project folder - if item_type == ItemType.SHAREDFOLDER: - f['name'] = f'[p]{f.get("name")}' - folders = folders + f"\033[34m{f.get('name')}\033[0m ..." f_string = folders + files diff --git a/app/services/output_manager/error_handler.py b/app/services/output_manager/error_handler.py index 53e5d480..963c1206 100644 --- a/app/services/output_manager/error_handler.py +++ b/app/services/output_manager/error_handler.py @@ -61,7 +61,7 @@ class ECustomizedError(enum.Enum): RESERVED_FOLDER = 'RESERVED_FOLDER' INVALID_ACTION = 'INVALID_ACTION' INVALID_FOLDER = 'INVALID_FOLDER' - INVALID_NAMEFOLDER = 'INVALID_NAMEFOLDER' + INVALID_PROJECT_PATH = 'INVALID_PROJECT_PATH' INVALID_DOWNLOAD = 'INVALID_DOWNLOAD' DUPLICATE_TAG_ERROR = 'DUPLICATE_TAG_ERROR' # file metadata related error diff --git a/app/utils/aggregated.py b/app/utils/aggregated.py index e338ae5f..afd1f671 100644 --- a/app/utils/aggregated.py +++ b/app/utils/aggregated.py @@ -173,8 +173,8 @@ def identify_target_folder(project_path: str) -> Tuple[str, ItemType, str]: contain invalid characters and return the project code and target folder Parameters: - project_path: - - for project folder the input folder path (eg. /projectfolder/) - - for name folder the input folder path will be (eg. /) + - for project folder the input folder path (eg. /shared/) + - for name folder the input folder path will be (eg. /users/) Return: - project_code: the project code - folder_type: the folder type @@ -186,17 +186,11 @@ def identify_target_folder(project_path: str) -> Tuple[str, ItemType, str]: # check folder type if is project folder or name folder # there will be a extra string for project folder between project code and folder name - if len(temp_paths) == 2: - folder_type = ItemType.NAMEFOLDER - folder_name = temp_paths[1] - elif len(temp_paths) >= 3: + if len(temp_paths) >= 3: folder_type = ItemType.get_type_from_keyword(temp_paths[1]) - if folder_type == ItemType.SHAREDFOLDER: - folder_name = temp_paths[2] - else: - folder_name = os.path.join(temp_paths[1], temp_paths[2]) + folder_name = temp_paths[2] else: - SrvErrorHandler.customized_handle(ECustomizedError.INVALID_NAMEFOLDER, True) + SrvErrorHandler.customized_handle(ECustomizedError.INVALID_PROJECT_PATH, True) target_folder = '' # first check if folder names are valid diff --git a/pyproject.toml b/pyproject.toml index 0ad2804c..c5a8b4c7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "app" -version = "2.10.1" +version = "3.0.0" description = "This service is designed to support pilot platform" authors = ["Indoc Systems"] diff --git a/tests/app/commands/test_file.py b/tests/app/commands/test_file.py index ab9e64bc..2331a535 100644 --- a/tests/app/commands/test_file.py +++ b/tests/app/commands/test_file.py @@ -16,6 +16,7 @@ from app.commands.file import file_move from app.commands.file import file_put from app.commands.file import file_resume +from app.models.item import ItemType from app.services.file_manager.file_metadata.file_metadata_client import FileMetaClient from app.services.file_manager.file_upload.models import FileObject from app.services.output_manager.error_handler import ECustomizedError @@ -45,7 +46,15 @@ def test_file_upload_command_success_with_attribute(mocker, cli_runner): result = cli_runner.invoke( file_put, - ['--project-path', 'test_project/admin', '--thread', 1, '--attribute', 'template.json', 'test.txt'], + [ + '--project-path', + f'test_project/{ItemType.NAMEFOLDER.get_prefix_by_type()}admin', + '--thread', + 1, + '--attribute', + 'template.json', + 'test.txt', + ], ) assert result.exit_code == 0 @@ -134,69 +143,55 @@ def test_file_list_with_pagination_with_folder_success(requests_mock, mocker, cl return_value=decoded_token(), ) - mocker.patch( - 'app.services.file_manager.file_list.search_item', - return_value={ - 'result': { - 'type': 'folder', - 'id': 'id', - } - }, - ) requests_mock.get( 'http://bff_cli' + '/v1/testproject/files/query', json={ 'code': 200, 'error_msg': '', 'result': [ - {'type': 'file', 'name': 'file1'}, - {'type': 'file', 'name': 'file2'}, + {'type': ItemType.FILE.value, 'name': 'file1'}, + {'type': ItemType.FILE.value, 'name': 'file2'}, ], }, ) mocker.patch.object(questionary, 'select') questionary.select.return_value.ask.return_value = 'exit' - result = cli_runner.invoke(file_list, ['testproject/admin', '-z', 'greenroom']) + result = cli_runner.invoke(file_list, ['testproject/users/admin', '-z', 'greenroom']) outputs = result.output.split('\n') assert outputs[0] == 'file1 file2 ' -@pytest.mark.parametrize('parent_folder_type', ['name_folder', 'project_folder']) -def test_file_list_with_pagination_with_name_project_folder(requests_mock, mocker, cli_runner, parent_folder_type): +def test_file_list_with_pagination_with_root_folder(requests_mock, mocker, cli_runner): mocker.patch( 'app.services.user_authentication.token_manager.SrvTokenManager.decode_access_token', return_value=decoded_token(), ) - mocker.patch( - 'app.services.file_manager.file_list.search_item', - return_value={ - 'result': { - 'type': parent_folder_type, - 'id': 'id', - } - }, - ) + folder = 'folder1' + folder_with_underline = 'folder_1' + folder_with_space = 'folder 1' + root_folder = 'root_folder' requests_mock.get( 'http://bff_cli' + '/v1/testproject/files/query', json={ 'code': 200, 'error_msg': '', 'result': [ - {'type': 'folder', 'name': 'folder1'}, - {'type': 'name_folder', 'name': 'name_folder1'}, - {'type': 'project_folder', 'name': 'project folder1'}, - {'type': 'folder', 'name': 'test folder2'}, - {'type': 'project_folder', 'name': 'project folder2'}, + {'type': ItemType.FOLDER.value, 'name': folder}, + {'type': ItemType.NAMEFOLDER.value, 'name': folder_with_underline}, + {'type': ItemType.SHAREDFOLDER.value, 'name': folder_with_underline}, + {'type': ItemType.FOLDER.value, 'name': folder_with_space}, + {'type': ItemType.SHAREDFOLDER.value, 'name': folder_with_space}, + {'type': ItemType.ROOTFOLDER.value, 'name': root_folder}, ], }, ) mocker.patch.object(questionary, 'select') questionary.select.return_value.ask.return_value = 'exit' - result = cli_runner.invoke(file_list, ['testproject/admin', '-z', 'greenroom']) + result = cli_runner.invoke(file_list, ['testproject/users/admin', '-z', 'greenroom']) outputs = result.output.split('\n') - assert outputs[0] == 'folder1 name_folder1 [p]"project folder1" ' - assert outputs[1] == '"test folder2" [p]"project folder2" ' + assert outputs[0] == f'{folder} {folder_with_underline} {folder_with_underline} "{folder_with_space}" ' + assert outputs[1] == f'"{folder_with_space}" {root_folder} ' def test_empty_file_list_with_pagination(requests_mock, mocker, cli_runner): @@ -205,15 +200,6 @@ def test_empty_file_list_with_pagination(requests_mock, mocker, cli_runner): return_value=decoded_token(), ) - mocker.patch( - 'app.services.file_manager.file_list.search_item', - return_value={ - 'result': { - 'type': 'folder', - 'id': 'id', - } - }, - ) requests_mock.get( 'http://bff_cli' + '/v1/testproject/files/query', json={'code': 200, 'error_msg': '', 'result': []}, @@ -222,10 +208,10 @@ def test_empty_file_list_with_pagination(requests_mock, mocker, cli_runner): questionary.select.return_value.ask.return_value = 'exit' result = cli_runner.invoke(file_list, ['testproject/admin', '-z', 'greenroom']) outputs = result.output.split('\n') - assert outputs[0] == ' ' + assert outputs[0] == '' -@pytest.mark.parametrize('parent_folder_type', ['name_folder', 'project_folder']) +@pytest.mark.parametrize('parent_folder_type', [ItemType.NAMEFOLDER.value, ItemType.SHAREDFOLDER.value]) def test_file_download_success(requests_mock, mocker, cli_runner, parent_folder_type): mocker.patch( 'app.services.user_authentication.token_manager.SrvTokenManager.decode_access_token', @@ -246,7 +232,7 @@ def test_file_download_success(requests_mock, mocker, cli_runner, parent_folder_ { 'code': 200, 'result': { - 'type': 'file', + 'type': ItemType.FILE.value, 'id': 'id', }, }, diff --git a/tests/app/services/file_manager/file_upload/test_file_upload.py b/tests/app/services/file_manager/file_upload/test_file_upload.py index fb385057..34e56d96 100644 --- a/tests/app/services/file_manager/file_upload/test_file_upload.py +++ b/tests/app/services/file_manager/file_upload/test_file_upload.py @@ -36,7 +36,7 @@ def test_assemble_path_at_name_folder(mocker): current_file_path, parent_folder, create_folder_flag, _ = assemble_path( local_file_path, target_folder, project_code, ItemType.NAMEFOLDER, zone ) - assert current_file_path == 'admin/file.txt' + assert current_file_path == f'{ItemType.NAMEFOLDER.get_prefix_by_type()}admin/file.txt' assert parent_folder.get('name') == 'admin' assert create_folder_flag is False @@ -53,6 +53,16 @@ def test_assemble_path_at_exsting_folder(mocker): 'id': 'test', 'parent_id': 'test_parent', 'parent_path': '', + 'name': ItemType.NAMEFOLDER.get_prefix_by_type().strip('/'), + 'zone': 0, + 'type': 'folder', + } + }, + { + 'result': { + 'id': 'test', + 'parent_id': 'test_parent', + 'parent_path': ItemType.NAMEFOLDER.get_prefix_by_type(), 'name': 'admin', 'zone': 0, 'type': 'folder', @@ -62,7 +72,7 @@ def test_assemble_path_at_exsting_folder(mocker): 'result': { 'id': 'test', 'parent_id': 'test_parent', - 'parent_path': 'admin', + 'parent_path': ItemType.NAMEFOLDER.get_prefix_by_type() + 'admin', 'name': 'test_folder_exist', 'zone': 0, 'type': 'folder', @@ -74,7 +84,7 @@ def test_assemble_path_at_exsting_folder(mocker): current_file_path, parent_folder, create_folder_flag, _ = assemble_path( local_file_path, target_folder, project_code, ItemType.NAMEFOLDER, zone ) - assert current_file_path == 'admin/test_folder_exist/file.txt' + assert current_file_path == f'{ItemType.NAMEFOLDER.get_prefix_by_type()}admin/test_folder_exist/file.txt' assert parent_folder.get('name') == 'test_folder_exist' assert create_folder_flag is False @@ -91,6 +101,16 @@ def test_assemble_path_at_non_existing_folder(mocker): 'id': 'test', 'parent_id': 'test_parent', 'parent_path': '', + 'name': ItemType.NAMEFOLDER.get_prefix_by_type().strip('/'), + 'zone': 0, + 'type': 'folder', + } + }, + { + 'result': { + 'id': 'test', + 'parent_id': 'test_parent', + 'parent_path': ItemType.NAMEFOLDER.get_prefix_by_type(), 'name': 'admin', 'zone': 0, 'type': 'folder', @@ -105,7 +125,7 @@ def test_assemble_path_at_non_existing_folder(mocker): current_file_path, parent_folder, create_folder_flag, _ = assemble_path( local_file_path, target_folder, project_code, ItemType.NAMEFOLDER, zone ) - assert current_file_path == 'admin/test_folder_not_exist' + assert current_file_path == f'{ItemType.NAMEFOLDER.get_prefix_by_type()}admin/test_folder_not_exist' assert parent_folder.get('name') == 'admin' assert create_folder_flag is True @@ -133,9 +153,9 @@ def test_assemble_path_at_project_folder(mocker): current_file_path, parent_folder, create_folder_flag, target_folder = assemble_path( local_file_path, target_folder, project_code, ItemType.SHAREDFOLDER, zone ) - assert current_file_path == 'shared/project_folder/file.txt' + assert current_file_path == f'{ItemType.SHAREDFOLDER.get_prefix_by_type()}project_folder/file.txt' assert parent_folder.get('name') == 'project_folder' - assert target_folder == 'shared/project_folder' + assert target_folder == f'{ItemType.SHAREDFOLDER.get_prefix_by_type()}project_folder' assert create_folder_flag is False diff --git a/tests/app/utils/test_aggregated.py b/tests/app/utils/test_aggregated.py index 91454cbb..1a892dc7 100644 --- a/tests/app/utils/test_aggregated.py +++ b/tests/app/utils/test_aggregated.py @@ -136,10 +136,22 @@ def test_validate_folder_name(folder_name): @pytest.mark.parametrize( 'input_path,expected_result', [ - ('project_code/username', ('project_code', ItemType.NAMEFOLDER, 'username')), - ('project_code/username/folder1', ('project_code', ItemType.NAMEFOLDER, 'username/folder1')), - ('project_code/projectfolder/folder1', ('project_code', ItemType.SHAREDFOLDER, 'folder1')), - ('project_code/projectfolder/folder1/folder2', ('project_code', ItemType.SHAREDFOLDER, 'folder1/folder2')), + ( + f'project_code/{ItemType.NAMEFOLDER.get_prefix_by_type()}username', + ('project_code', ItemType.NAMEFOLDER, 'username'), + ), + ( + f'project_code/{ItemType.NAMEFOLDER.get_prefix_by_type()}username/folder1', + ('project_code', ItemType.NAMEFOLDER, 'username/folder1'), + ), + ( + f'project_code/{ItemType.SHAREDFOLDER.get_prefix_by_type()}folder1', + ('project_code', ItemType.SHAREDFOLDER, 'folder1'), + ), + ( + f'project_code/{ItemType.SHAREDFOLDER.get_prefix_by_type()}folder1/folder2', + ('project_code', ItemType.SHAREDFOLDER, 'folder1/folder2'), + ), ], ) def test_identify_target_folder_success_with_different_path(mocker, input_path, expected_result): From c7f8d90c92a7442a0dcb40f9095a107897fd9e02 Mon Sep 17 00:00:00 2001 From: Color Zhan Date: Thu, 25 Apr 2024 14:33:28 -0400 Subject: [PATCH 2/9] Pilot 4951: Update the logic of upload command for root level folder(eg. users, shared) (#134) * add [p] prefix for project folder when listing items under project * group itemprefix class with itemtype class * add double quotation when item name contains space * change enum type PROJECTFOLDER to SHAREDFOLDER * replace PROJECTFOLDER with SHAREDFOLDER * fixup test cases * fixup logic for different prefix * add prefix for name folder * bump up version to 3.0.0 * fixup test cases * fixup test cases * replace hardcoded string with enum value in test cases * remove the reference of project_folder * fixup test cases * update help page --------- Co-authored-by: zhiren Former-commit-id: 74b12c15417dd9af47823cf8451c463b6ea128ee --- app/resources/custom_help.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/resources/custom_help.py b/app/resources/custom_help.py index f0edb329..aa750847 100644 --- a/app/resources/custom_help.py +++ b/app/resources/custom_help.py @@ -9,9 +9,9 @@ class HelpPage: page = { 'update': { 'version': pkg_resources.get_distribution('app').version, - '1': 'Add new feature for folder merging', - '2': 'Secure the config file', - '3': 'Optimize logic, input and error message', + '1': 'Add a new feature for file metadata downloading.', + '2': 'Add a new feature for file moving/renaming.', + '3': 'Support shared folder within project.', }, 'dataset': { 'DATASET_DOWNLOAD': 'Download a dataset or a particular version of a dataset.', From c97a558fb53d56a4a3d8195b0e7f9d2cfe87dfd7 Mon Sep 17 00:00:00 2001 From: Color Zhan Date: Thu, 25 Apr 2024 16:01:16 -0400 Subject: [PATCH 3/9] PILOT 4808: CLI differentiates the item path of move/rename between Project Folders and Name Folders (#137) * add [p] prefix for project folder when listing items under project * group itemprefix class with itemtype class * add double quotation when item name contains space * udpate move/rename commands to distinguish project folder/name folder * change enum type PROJECTFOLDER to SHAREDFOLDER * update move/rename logic but will need to fix upload/list command for prefix changes * replace PROJECTFOLDER with SHAREDFOLDER * replace PROJECTFOLDER with SHAREDFOLDER * fixup test cases * fixup logic for different prefix * add prefix for name folder * bump up version to 3.0.0 * fixup test cases * fixup test cases * remove SHAREFOLDER specific logic * fixup test cases * bumpup version * Pilot 4807: CLI differentiates the item path of downloading metadata between Project Folders and Name Folders (#138) * use keyword and prefix to config the path under project folder or name folder * fixup test cases * remove old comments * remove response is used before assignment --------- Co-authored-by: zhiren --------- Co-authored-by: zhiren Former-commit-id: da53de040dae45a13aa0114716ef3efd79a31a7d --- app/commands/file.py | 16 ++- app/models/item.py | 2 +- .../file_metadata/file_metadata_client.py | 12 +- .../file_move/file_move_client.py | 5 - app/services/project_manager/project.py | 2 +- pyproject.toml | 2 +- tests/app/commands/test_file.py | 9 +- .../test_file_metadata_client.py | 109 ++++++++---------- .../file_upload/test_file_upload.py | 6 +- 9 files changed, 79 insertions(+), 84 deletions(-) diff --git a/app/commands/file.py b/app/commands/file.py index e83baa27..319f2221 100644 --- a/app/commands/file.py +++ b/app/commands/file.py @@ -12,6 +12,7 @@ import app.services.output_manager.help_page as file_help import app.services.output_manager.message_handler as message_handler from app.configs.app_config import AppConfig +from app.models.item import ItemType from app.services.file_manager.file_download.download_client import SrvFileDownload from app.services.file_manager.file_list import SrvFileList from app.services.file_manager.file_manifests import SrvFileManifests @@ -544,8 +545,21 @@ def file_move(**kwargs): zone = kwargs.get('zone') skip_confirm = kwargs.get('yes') + if len(src_item_path.split('/')) == 1 and len(dest_item_path.split('/')) == 1: + raise Exception('Invalid path') + + # tranlate keyword to correct object path + src_keyword, src_path = src_item_path.split('/', 1) + dest_keyword, dest_path = dest_item_path.split('/', 1) + + src_type = ItemType.get_type_from_keyword(src_keyword) + dest_type = ItemType.get_type_from_keyword(dest_keyword) + + src_path = src_type.get_prefix_by_type() + src_path + dest_path = dest_type.get_prefix_by_type() + dest_path + zone = get_zone(zone) if zone else AppConfig.Env.green_zone.lower() - file_meta_client = FileMoveClient(zone, project_code, src_item_path, dest_item_path, skip_confirm=skip_confirm) + file_meta_client = FileMoveClient(zone, project_code, src_path, dest_path, skip_confirm=skip_confirm) file_meta_client.move_file() message_handler.SrvOutPutHandler.move_action_success(src_item_path, dest_item_path) diff --git a/app/models/item.py b/app/models/item.py index db1068d8..b7c84b6c 100644 --- a/app/models/item.py +++ b/app/models/item.py @@ -27,7 +27,7 @@ def get_type_from_keyword(self, keyword: str): 'users': self.NAMEFOLDER, } - return alternative_mapping.get(keyword, '') + return alternative_mapping.get(keyword, self.FOLDER) def get_prefix_by_type(self) -> str: """Get the prefix for the folder type.""" diff --git a/app/services/file_manager/file_metadata/file_metadata_client.py b/app/services/file_manager/file_metadata/file_metadata_client.py index ad3833a3..fad9cc02 100644 --- a/app/services/file_manager/file_metadata/file_metadata_client.py +++ b/app/services/file_manager/file_metadata/file_metadata_client.py @@ -19,6 +19,7 @@ import app.services.logger_services.log_functions as logger import app.services.output_manager.message_handler as message_handler +from app.models.item import ItemType from app.services.output_manager.error_handler import ECustomizedError from app.services.output_manager.error_handler import customized_error_msg from app.utils.aggregated import get_attribute_template_by_id @@ -109,14 +110,15 @@ def download_file_metadata(self) -> List[Dict[str, Any]]: """ project_code, object_path = self.file_path.split('/', 1) + root_folder, object_path = object_path.split('/', 1) + root_type = ItemType.get_type_from_keyword(root_folder) + object_path = join(root_type.get_prefix_by_type(), object_path) item_res = search_item(project_code, self.zone, object_path) - # double check if the file is in shared folder if item_res.get('code') == 404: - item_res = search_item(project_code, self.zone, f'shared/{object_path}') - if item_res.get('code') == 404: - logger.error(f'Cannot find item {self.file_path} at {self.zone}.') - exit(1) + logger.error(f'Cannot find item {self.file_path} at {self.zone}.') + exit(1) + # filter out item metadata item_res = item_res.get('result', {}) extra_info = item_res.pop('extended', {}).get('extra') tags = extra_info.get('tags', []) diff --git a/app/services/file_manager/file_move/file_move_client.py b/app/services/file_manager/file_move/file_move_client.py index 17478598..2c998428 100644 --- a/app/services/file_manager/file_move/file_move_client.py +++ b/app/services/file_manager/file_move/file_move_client.py @@ -60,11 +60,6 @@ def create_object_path_if_not_exist(self, folder_path: str) -> dict: """ path_list = folder_path.split('/') - # first get the root folder to check if it is name folder - # or project folder - root_item = search_item(self.project_code, self.zone, path_list[0]).get('result') - if root_item.get('type') == 'project_folder': - path_list[0] = '/'.join([root_item.get('parent_path'), path_list[0]]) # first check every folder in path exist or not # the loop start with index 1 since we assume cli will not diff --git a/app/services/project_manager/project.py b/app/services/project_manager/project.py index ec98c503..ef832bc0 100644 --- a/app/services/project_manager/project.py +++ b/app/services/project_manager/project.py @@ -48,4 +48,4 @@ def list_projects(self, page, page_size, order, order_by): else: SrvErrorHandler.default_handle(response.content, True) except Exception: - SrvErrorHandler.default_handle(response.content, True) + SrvErrorHandler.default_handle('Error when listing projects', True) diff --git a/pyproject.toml b/pyproject.toml index c5a8b4c7..d2b215b9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "app" -version = "3.0.0" +version = "3.0.1" description = "This service is designed to support pilot platform" authors = ["Indoc Systems"] diff --git a/tests/app/commands/test_file.py b/tests/app/commands/test_file.py index 2331a535..8b558768 100644 --- a/tests/app/commands/test_file.py +++ b/tests/app/commands/test_file.py @@ -208,7 +208,7 @@ def test_empty_file_list_with_pagination(requests_mock, mocker, cli_runner): questionary.select.return_value.ask.return_value = 'exit' result = cli_runner.invoke(file_list, ['testproject/admin', '-z', 'greenroom']) outputs = result.output.split('\n') - assert outputs[0] == '' + assert outputs[0] == ' ' @pytest.mark.parametrize('parent_folder_type', [ItemType.NAMEFOLDER.value, ItemType.SHAREDFOLDER.value]) @@ -365,7 +365,10 @@ def test_file_move_success(mocker, cli_runner): return_value=None, ) - result = cli_runner.invoke(file_move, ['test_project', 'src_item_path', 'dest_item_path']) + src_path = 'src_item_path/test' + dest_path = 'dest_item_path/test' + result = cli_runner.invoke(file_move, ['test_project', src_path, dest_path]) + outputs = result.output.split('\n') - assert outputs[0] == 'Successfully moved src_item_path to dest_item_path' + assert outputs[0] == f'Successfully moved {src_path} to {dest_path}' file_move_mock.assert_called_once() diff --git a/tests/app/services/file_manager/file_metadata/test_file_metadata_client.py b/tests/app/services/file_manager/file_metadata/test_file_metadata_client.py index dafeeaac..05b72110 100644 --- a/tests/app/services/file_manager/file_metadata/test_file_metadata_client.py +++ b/tests/app/services/file_manager/file_metadata/test_file_metadata_client.py @@ -2,19 +2,30 @@ # # Contact Indoc Systems for any questions regarding the use of this source code. +import pytest + from app.configs.app_config import AppConfig +from app.models.item import ItemType from app.services.file_manager.file_metadata.file_metadata_client import FileMetaClient from tests.conftest import decoded_token -def test_file_metadata_client_get_detail_success(mocker, httpx_mock): +@pytest.mark.parametrize( + 'root_folder', + [ItemType.NAMEFOLDER, ItemType.SHAREDFOLDER], +) +def test_file_metadata_client_get_detail_success(mocker, httpx_mock, root_folder: ItemType): + file_name = 'test_file.txt' + project_code = 'project_code' item_info = { 'id': 'test', 'parent_id': 'test_parent', - 'parent_path': '', - 'name': 'admin', + 'parent_path': root_folder.get_prefix_by_type(), + 'name': file_name, 'zone': 0, 'status': 'ACTIVE', + 'container_code': project_code, + 'container_type': 'project', } tags = ['test'] attri_template_uid = 'template_uid' @@ -41,9 +52,11 @@ def test_file_metadata_client_get_detail_success(mocker, httpx_mock): return_value=None, ) - file_meta_client = FileMetaClient('zone', 'project_code/object_path', 'general', 'attr', 'tag') - assert file_meta_client.project_code == 'project_code' - assert file_meta_client.object_path == 'object_path' + file_meta_client = FileMetaClient( + 'zone', f'{project_code}/{root_folder.get_prefix_by_type()}/{file_name}', 'general', 'attr', 'tag' + ) + assert file_meta_client.project_code == project_code + assert file_meta_client.object_path == f'{root_folder.get_prefix_by_type()}/{file_name}' item_info, res_attributes, tags = file_meta_client.download_file_metadata() assert item_info == item_info @@ -51,14 +64,22 @@ def test_file_metadata_client_get_detail_success(mocker, httpx_mock): assert tags == tags -def test_file_metadata_client_get_detail_success_with_no_tag_attributes(mocker, httpx_mock): +@pytest.mark.parametrize( + 'root_folder', + [ItemType.NAMEFOLDER, ItemType.SHAREDFOLDER], +) +def test_file_metadata_client_get_detail_success_with_no_tag_attributes(mocker, root_folder: ItemType): + file_name = 'test_file.txt' + project_code = 'project_code' item_info = { 'id': 'test', 'parent_id': 'test_parent', - 'parent_path': '', - 'name': 'admin', + 'parent_path': root_folder.get_prefix_by_type(), + 'name': file_name, 'zone': 0, 'status': 'ACTIVE', + 'container_code': project_code, + 'container_type': 'project', } mocker.patch( @@ -76,9 +97,11 @@ def test_file_metadata_client_get_detail_success_with_no_tag_attributes(mocker, return_value=None, ) - file_meta_client = FileMetaClient('zone', 'project_code/object_path', 'general', 'attr', 'tag') - assert file_meta_client.project_code == 'project_code' - assert file_meta_client.object_path == 'object_path' + file_meta_client = FileMetaClient( + 'zone', f'{project_code}/{root_folder.get_prefix_by_type()}/{file_name}', 'general', 'attr', 'tag' + ) + assert file_meta_client.project_code == project_code + assert file_meta_client.object_path == f'{root_folder.get_prefix_by_type()}/{file_name}' item_info, res_attributes, tags = file_meta_client.download_file_metadata() assert item_info == item_info @@ -86,20 +109,11 @@ def test_file_metadata_client_get_detail_success_with_no_tag_attributes(mocker, assert tags == tags -def test_metadata_download_from_project_folder(mocker, httpx_mock): - item_info = { - 'id': 'test', - 'parent_id': 'test_parent', - 'parent_path': 'shared/path', - 'name': 'admin', - 'zone': 0, - 'status': 'ACTIVE', - } - tags = ['test'] - attri_template_uid = 'template_uid' - attri_template_name = 'template_name' - attributes = {attri_template_uid: {'attr_1': 'value'}} - +@pytest.mark.parametrize( + 'root_folder', + [ItemType.NAMEFOLDER, ItemType.SHAREDFOLDER], +) +def test_metadata_download_fail_when_file_doesnot_exist(mocker, capfd, root_folder: ItemType): mocker.patch( 'app.services.user_authentication.token_manager.SrvTokenManager.decode_access_token', return_value=decoded_token(), @@ -107,53 +121,20 @@ def test_metadata_download_from_project_folder(mocker, httpx_mock): search_mock = mocker.patch( 'app.services.file_manager.file_metadata.file_metadata_client.search_item', - ) - search_mock.side_effect = [ - {'result': {}, 'code': 404}, - {'result': {**item_info, 'extended': {'extra': {'tags': tags, 'attributes': attributes}}}}, - ] - httpx_mock.add_response( - url=AppConfig.Connections.url_portal + f'/v1/data/manifest/{attri_template_uid}', - method='GET', - json={'result': {'id': attri_template_uid, 'name': attri_template_name}}, + return_value={'result': {}, 'code': 404}, ) - mocker.patch( - 'app.services.file_manager.file_metadata.file_metadata_client.FileMetaClient.save_file_metadata', - return_value=None, + file_meta_client = FileMetaClient( + 'zone', f'project_code/{root_folder.get_prefix_by_type()}/file_name', 'general', 'attr', 'tag' ) - file_meta_client = FileMetaClient('zone', 'project_code/object_path', 'general', 'attr', 'tag') - assert file_meta_client.project_code == 'project_code' - assert file_meta_client.object_path == 'object_path' - - item_info, res_attributes, tags = file_meta_client.download_file_metadata() - assert item_info == item_info - assert res_attributes == {attri_template_name: attributes.get(attri_template_uid)} - assert tags == tags - assert search_mock.call_count == 2 - - -def test_metadata_download_fail_when_file_doesnot_exist(mocker, capfd): - mocker.patch( - 'app.services.user_authentication.token_manager.SrvTokenManager.decode_access_token', - return_value=decoded_token(), - ) - - search_mock = mocker.patch( - 'app.services.file_manager.file_metadata.file_metadata_client.search_item', - ) - search_mock.side_effect = [{'result': {}, 'code': 404}, {'result': {}, 'code': 404}] - - file_meta_client = FileMetaClient('zone', 'project_code/object_path', 'general', 'attr', 'tag') - try: file_meta_client.download_file_metadata() except SystemExit: - assert search_mock.call_count == 2 + assert search_mock.call_count == 1 out, _ = capfd.readouterr() - expect = 'Cannot find item project_code/object_path at zone.\n' + expect = f'Cannot find item project_code/{root_folder.get_prefix_by_type()}/file_name at zone.\n' assert out == expect else: AssertionError('SystemExit not raised') diff --git a/tests/app/services/file_manager/file_upload/test_file_upload.py b/tests/app/services/file_manager/file_upload/test_file_upload.py index 34e56d96..907d51b9 100644 --- a/tests/app/services/file_manager/file_upload/test_file_upload.py +++ b/tests/app/services/file_manager/file_upload/test_file_upload.py @@ -132,7 +132,7 @@ def test_assemble_path_at_non_existing_folder(mocker): def test_assemble_path_at_project_folder(mocker): local_file_path = './test/file.txt' - target_folder = 'project_folder' + target_folder = ItemType.SHAREDFOLDER.value project_code = 'test_project' zone = 0 @@ -143,9 +143,9 @@ def test_assemble_path_at_project_folder(mocker): 'id': 'test', 'parent_id': 'test_parent', 'parent_path': '', - 'name': 'project_folder', + 'name': target_folder, 'zone': 0, - 'type': 'project_folder', + 'type': target_folder, } }, ) From a459705b51a07d98a2f8528ddbd490e7cbe75ffd Mon Sep 17 00:00:00 2001 From: Color Zhan Date: Fri, 26 Apr 2024 10:16:08 -0400 Subject: [PATCH 4/9] Pilot 4806: CLI differentiates the input path of downloading file between Project Folders and Name Folders (#140) * add [p] prefix for project folder when listing items under project * group itemprefix class with itemtype class * add double quotation when item name contains space * udpate move/rename commands to distinguish project folder/name folder * change enum type PROJECTFOLDER to SHAREDFOLDER * update move/rename logic but will need to fix upload/list command for prefix changes * replace PROJECTFOLDER with SHAREDFOLDER * replace PROJECTFOLDER with SHAREDFOLDER * fixup test cases * fixup logic for different prefix * add prefix for name folder * bump up version to 3.0.0 * fixup test cases * fixup test cases * replace hardcoded string with enum value in test cases * remove the reference of project_folder * fixup test cases * replace hardcode project folder with enum class * add more download test cases * fixup test cases * remove SHAREFOLDER specific logic * fixup test cases * remove old comments --------- Co-authored-by: zhiren Former-commit-id: a1291ad220fe91f15e48b90145b2c3d7f0475af1 --- app/commands/file.py | 10 +- app/models/item.py | 5 + .../file_download/download_client.py | 9 +- .../file_manager/file_download/model.py | 5 +- tests/app/commands/test_file.py | 7 +- .../test_file_download_client.py | 143 ++++++++++++++++++ 6 files changed, 161 insertions(+), 18 deletions(-) create mode 100644 tests/app/services/file_manager/file_download/test_file_download_client.py diff --git a/app/commands/file.py b/app/commands/file.py index 319f2221..b40f284c 100644 --- a/app/commands/file.py +++ b/app/commands/file.py @@ -430,14 +430,12 @@ def file_download(**kwargs): else: item_res = [] for path in paths: - project_code, root_folder = path.strip('/').split('/')[:2] - target_path = '/'.join(path.split('/')[1::]) - # search the root to check for name folder or project folder - root_item = search_item(project_code, zone, root_folder).get('result', {}) - target_path = 'shared/' + target_path if root_item.get('type') == 'project_folder' else target_path + project_code, root_folder, object_path = path.strip('/').split('/', 2) + root_type = ItemType.get_type_from_keyword(root_folder) + object_path = os.path.join(root_type.get_prefix_by_type(), object_path) # search the target item and download to local - item = search_item(project_code, zone, target_path) + item = search_item(project_code, zone, object_path) if item.get('code') == 200 and item.get('result'): item_status = 'success' item_result = item.get('result') diff --git a/app/models/item.py b/app/models/item.py index b7c84b6c..0b8ba0a4 100644 --- a/app/models/item.py +++ b/app/models/item.py @@ -39,3 +39,8 @@ def get_prefix_by_type(self) -> str: } return prefix.get(self.value, '') + + +class ItemZone(str, Enum): + GREENROOM = 'greenroom' + CORE = 'core' diff --git a/app/services/file_manager/file_download/download_client.py b/app/services/file_manager/file_download/download_client.py index d9f0ca4c..3e5b7b4a 100644 --- a/app/services/file_manager/file_download/download_client.py +++ b/app/services/file_manager/file_download/download_client.py @@ -15,6 +15,7 @@ import app.services.output_manager.message_handler as mhandler from app.configs.app_config import AppConfig from app.configs.user_config import UserConfig +from app.models.item import ItemZone from app.models.service_meta_class import MetaService from app.services.output_manager.error_handler import ECustomizedError from app.services.output_manager.error_handler import SrvErrorHandler @@ -53,7 +54,7 @@ def print_prepare_msg(self, message): click.secho(f"{message}{'.'*i}\r", fg='white', nl=False) def get_download_url(self, zone): - if zone == 'greenroom': + if zone == ItemZone.GREENROOM.value: url = self.appconfig.Connections.url_download_greenroom else: url = self.appconfig.Connections.url_download_core @@ -101,11 +102,11 @@ def prepare_download(self): file_path = download_info.get('file_path') pre_status = EFileStatus(response.get('status')) elif res.status_code == 403: - SrvErrorHandler.customized_handle(ECustomizedError.NO_FILE_PERMMISION, self.interactive) + SrvErrorHandler.customized_handle(ECustomizedError.NO_FILE_PERMMISION, if_exit=self.interactive) elif res.status_code == 400 and 'number of file must greater than 0' in res_json.get('error_msg'): - SrvErrorHandler.customized_handle(ECustomizedError.FOLDER_EMPTY, self.interactive) + SrvErrorHandler.customized_handle(ECustomizedError.FOLDER_EMPTY, if_exit=self.interactive) else: - SrvErrorHandler.customized_handle(ECustomizedError.DOWNLOAD_FAIL, self.interactive) + SrvErrorHandler.customized_handle(ECustomizedError.DOWNLOAD_FAIL, if_exit=self.interactive) return pre_status, file_path diff --git a/app/services/file_manager/file_download/model.py b/app/services/file_manager/file_download/model.py index b39745c2..fa990db5 100644 --- a/app/services/file_manager/file_download/model.py +++ b/app/services/file_manager/file_download/model.py @@ -5,11 +5,8 @@ from enum import Enum -class EFileStatus(Enum): +class EFileStatus(str, Enum): WAITING = 'WAITING' RUNNING = 'RUNNING' SUCCEED = 'SUCCEED' FAILED = 'FAILED' - - def __str__(self): - return '%s' % self.name diff --git a/tests/app/commands/test_file.py b/tests/app/commands/test_file.py index 8b558768..82241184 100644 --- a/tests/app/commands/test_file.py +++ b/tests/app/commands/test_file.py @@ -224,7 +224,7 @@ def test_file_download_success(requests_mock, mocker, cli_runner, parent_folder_ { 'code': 200, 'result': { - 'type': parent_folder_type, + 'type': parent_folder_type.value, 'name': 'test', 'id': 'id', }, @@ -244,13 +244,12 @@ def test_file_download_success(requests_mock, mocker, cli_runner, parent_folder_ return_value=None, ) - project_code, target_folder = 'testproject', 'test/test.txt' + project_code, target_folder = 'testproject', parent_folder_type.get_prefix_by_type() + 'test/test.txt' result = cli_runner.invoke(file_download, [f'{project_code}/{target_folder}', './']) outputs = result.output.split('\n') assert outputs[0] == '' - except_target_folder = 'test/test.txt' if parent_folder_type == 'name_folder' else 'shared/test/test.txt' - search_mock.assert_called_with(project_code, 'greenroom', except_target_folder) + search_mock.assert_called_with(project_code, 'greenroom', target_folder) download_mock.assert_called_once() diff --git a/tests/app/services/file_manager/file_download/test_file_download_client.py b/tests/app/services/file_manager/file_download/test_file_download_client.py new file mode 100644 index 00000000..64687b4c --- /dev/null +++ b/tests/app/services/file_manager/file_download/test_file_download_client.py @@ -0,0 +1,143 @@ +# Copyright (C) 2022-2024 Indoc Systems +# +# Contact Indoc Systems for any questions regarding the use of this source code. + +import click +import jwt +import pytest +from pytest_httpx import IteratorStream + +from app.configs.app_config import AppConfig +from app.models.item import ItemZone +from app.services.file_manager.file_download.download_client import SrvFileDownload +from app.services.file_manager.file_download.model import EFileStatus +from app.services.output_manager.error_handler import ECustomizedError +from app.services.output_manager.error_handler import customized_error_msg +from tests.conftest import decoded_token + + +@pytest.mark.parametrize('file_status', [EFileStatus.SUCCEED, EFileStatus.FAILED, EFileStatus.WAITING]) +def test_file_download_client_prepare_download_success(mocker, httpx_mock, file_status: EFileStatus): + test_file_path = 'test_file_path' + hash_token = jwt.encode({'file_path': test_file_path}, key='unittest', algorithm='HS256').decode('utf-8') + + mocker.patch( + 'app.services.user_authentication.token_manager.SrvTokenManager.decode_access_token', + return_value=decoded_token(), + ) + + download_client = SrvFileDownload(0, False) + download_client.file_geid = ['test'] + download_client.project_code = 'test_project' + + httpx_mock.add_response( + url=download_client.appconfig.Connections.url_v2_download_pre % (download_client.project_code), + method='POST', + status_code=200, + json={ + 'result': { + 'payload': { + 'hash_code': hash_token, + }, + 'status': file_status.value, + } + }, + ) + + pre_status, file_path = download_client.prepare_download() + assert pre_status == file_status + assert file_path == test_file_path + + +@pytest.mark.parametrize( + 'status_code', + [ + 403, + 400, + 500, + ], +) +def test_file_download_client_prepare_download_failed(mocker, httpx_mock, capfd, status_code: int): + mocker.patch( + 'app.services.user_authentication.token_manager.SrvTokenManager.decode_access_token', + return_value=decoded_token(), + ) + + download_client = SrvFileDownload(0, True) + download_client.file_geid = ['test'] + download_client.project_code = 'test_project' + + httpx_mock.add_response( + url=download_client.appconfig.Connections.url_v2_download_pre % (download_client.project_code), + method='POST', + status_code=status_code, + json={'error_msg': 'number of file must greater than 0'}, + ) + + try: + download_client.pre_download() + except SystemExit: + out, _ = capfd.readouterr() + + expect = { + 500: ECustomizedError.DOWNLOAD_FAIL, + 403: ECustomizedError.NO_FILE_PERMMISION, + 400: ECustomizedError.FOLDER_EMPTY, + } + assert out == customized_error_msg(expect.get(status_code)) + '\n' + else: + AssertionError('SystemExit not raised') + + +@pytest.mark.parametrize( + 'zone', + [ + ItemZone.GREENROOM.value, + ItemZone.CORE.value, + ], +) +def test_file_download_url_based_on_different_zones(zone: str): + download_client = SrvFileDownload(0, True) + download_client.file_geid = ['test'] + download_client.project_code = 'test_project' + + url = download_client.get_download_url(zone) + except_url = { + ItemZone.GREENROOM.value: AppConfig.Connections.url_download_greenroom, + ItemZone.CORE.value: AppConfig.Connections.url_download_core, + }.get(zone) + + assert url == except_url + + +@pytest.mark.parametrize( + 'total_size_presented', + [True, False], +) +def test_file_stream_download(mocker, httpx_mock, total_size_presented): + file_url = 'http://test.com' + file_content = b'123' + + mocker.patch( + 'app.services.user_authentication.token_manager.SrvTokenManager.decode_access_token', + return_value=decoded_token(), + ) + + httpx_mock.add_response( + url=file_url, + method='GET', + status_code=200, + content=IteratorStream([file_content]), + ) + + runner = click.testing.CliRunner() + with runner.isolated_filesystem(): + download_client = SrvFileDownload(0, True) + download_client.file_geid = ['test'] + download_client.project_code = 'test_project' + download_client.total_size = len(file_content) if total_size_presented else None + + download_client.download_file(file_url, 'test_file') + + with open('test_file', 'r') as f: + assert f.read() == '123' From d359591b14e17bf09c465e30bedc5fe752e7ae23 Mon Sep 17 00:00:00 2001 From: zhiren Date: Wed, 1 May 2024 15:14:43 -0400 Subject: [PATCH 5/9] fixup test cases --- tests/app/commands/test_file.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/app/commands/test_file.py b/tests/app/commands/test_file.py index 82241184..e48080f1 100644 --- a/tests/app/commands/test_file.py +++ b/tests/app/commands/test_file.py @@ -211,7 +211,7 @@ def test_empty_file_list_with_pagination(requests_mock, mocker, cli_runner): assert outputs[0] == ' ' -@pytest.mark.parametrize('parent_folder_type', [ItemType.NAMEFOLDER.value, ItemType.SHAREDFOLDER.value]) +@pytest.mark.parametrize('parent_folder_type', [ItemType.NAMEFOLDER, ItemType.SHAREDFOLDER]) def test_file_download_success(requests_mock, mocker, cli_runner, parent_folder_type): mocker.patch( 'app.services.user_authentication.token_manager.SrvTokenManager.decode_access_token', From 63998c82251ce4ba2967ef28ecafa91c70ed927c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Du=C5=A1an=20Andri=C4=87?= Date: Wed, 1 May 2024 16:02:06 -0400 Subject: [PATCH 6/9] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b469ea7e..d0071553 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# command_line_tool_ctl +# Pilot CLI [![Run Tests](https://github.com/PilotDataPlatform/cli/actions/workflows/run-tests.yml/badge.svg?branch=develop)](https://github.com/PilotDataPlatform/cli/actions/workflows/run-tests.yml) [![Python](https://img.shields.io/badge/python-3.7-brightgreen.svg)](https://www.python.org/) From ccd874bd36125c7c7b182d336ddd532822a9f916 Mon Sep 17 00:00:00 2001 From: zhiren Date: Tue, 14 May 2024 16:48:47 -0400 Subject: [PATCH 7/9] add prefix based on folder type when uploading into name or shared folder --- app/services/file_manager/file_upload/file_upload.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/app/services/file_manager/file_upload/file_upload.py b/app/services/file_manager/file_upload/file_upload.py index 952ec259..9a0f3ff8 100644 --- a/app/services/file_manager/file_upload/file_upload.py +++ b/app/services/file_manager/file_upload/file_upload.py @@ -69,10 +69,9 @@ def assemble_path( - target_folder: result object path on platform ''' - current_file_path = target_folder + '/' + f.rstrip('/').split('/')[-1] # set name folder as first parent folder - root_folder = target_folder.split('/')[0] + root_folder = folder_type.get_prefix_by_type() + target_folder.split('/')[0] parent_folder = search_item(project_code, zone, root_folder).get('result', {}) # if f input is a file then current_folder_node is target_folder From f3881330dbca9e830fc48ca0db5be1fab07ef9af Mon Sep 17 00:00:00 2001 From: zhiren Date: Tue, 14 May 2024 17:31:12 -0400 Subject: [PATCH 8/9] bumpup version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index d2b215b9..0304928a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "app" -version = "3.0.1" +version = "3.0.2" description = "This service is designed to support pilot platform" authors = ["Indoc Systems"] From b8a62d39e2c941c91c0fad0e716a92ad9984b079 Mon Sep 17 00:00:00 2001 From: zhiren Date: Wed, 22 May 2024 10:19:03 -0400 Subject: [PATCH 9/9] prepare release branch --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 0304928a..db648cfe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "app" -version = "3.0.2" +version = "3.0.3" description = "This service is designed to support pilot platform" authors = ["Indoc Systems"]