From 89e2d99d80b3ddbdd52901db1c408497225be079 Mon Sep 17 00:00:00 2001 From: zhiren Date: Tue, 12 Mar 2024 11:11:20 -0400 Subject: [PATCH 1/5] add new folder type for namefolder and project folder --- app/commands/file.py | 5 ++-- app/models/folder.py | 22 ++++++++++++++ app/resources/custom_help.py | 4 ++- .../file_manager/file_upload/file_upload.py | 14 ++++----- app/utils/aggregated.py | 30 +++++++++++++++---- 5 files changed, 59 insertions(+), 16 deletions(-) create mode 100644 app/models/folder.py diff --git a/app/commands/file.py b/app/commands/file.py index b3590f3d..e83baa27 100644 --- a/app/commands/file.py +++ b/app/commands/file.py @@ -167,8 +167,7 @@ def file_put(**kwargs): # noqa: C901 message_handler.SrvOutPutHandler.cancel_upload() exit(1) - project_path = click.prompt('ProjectCode') if not project_path else project_path - project_code, target_folder = identify_target_folder(project_path) + project_code, folder_type, target_folder = identify_target_folder(project_path) srv_manifest = SrvFileManifests() upload_val_event = { 'zone': zone, @@ -207,8 +206,8 @@ def file_put(**kwargs): # noqa: C901 f, target_folder, project_code, + folder_type, zone, - zipping, ) upload_event = { diff --git a/app/models/folder.py b/app/models/folder.py new file mode 100644 index 00000000..1ec155a5 --- /dev/null +++ b/app/models/folder.py @@ -0,0 +1,22 @@ +# Copyright (C) 2023-2024 Indoc Systems +# +# Contact Indoc Systems for any questions regarding the use of this source code. + +from enum import Enum + + +class FolderType(str, Enum): + """Available folder types.""" + + NAMEFOLDER = 'namefolder' + PROJECTFOLDER = 'projectfolder' + + def get_prefix(self) -> str: + """Get the prefix for the folder type.""" + + prefix = { + 'namefolder': 'namefolder/', + 'projectfolder': 'shared/', + } + + return prefix.get(self.value) diff --git a/app/resources/custom_help.py b/app/resources/custom_help.py index 75e77563..6bf37de8 100644 --- a/app/resources/custom_help.py +++ b/app/resources/custom_help.py @@ -43,7 +43,9 @@ class HelpPage: 'FILE_SYNC_ZIP': 'Download files as a zip.', 'FILE_SYNC_I': 'Enable downloading by geid.', 'FILE_SYNC_Z': 'Target Zone (i.e., core/greenroom).', - 'FILE_UPLOAD_P': 'Project folder path starting from Project Code. (i.e., indoctestproject/user/folder)', + 'FILE_UPLOAD_P': ( + 'Project folder path starting from Project Code. ' '(i.e., indoctestproject/namefolder//folder)' + ), 'FILE_UPLOAD_A': 'Add attributes to the file using a File Attribute Template.', 'FILE_UPLOAD_T': 'Add tags to the file using a Tag file.', 'FILE_UPLOAD_M': 'The message used to comment on the purpose of uploading your processed file.', diff --git a/app/services/file_manager/file_upload/file_upload.py b/app/services/file_manager/file_upload/file_upload.py index 15cd5219..548d88ea 100644 --- a/app/services/file_manager/file_upload/file_upload.py +++ b/app/services/file_manager/file_upload/file_upload.py @@ -18,6 +18,7 @@ import app.services.logger_services.log_functions as logger import app.services.output_manager.message_handler as mhandler from app.configs.app_config import AppConfig +from app.models.folder import FolderType from app.services.file_manager.file_upload.models import FileObject from app.services.file_manager.file_upload.models import ItemStatus from app.services.file_manager.file_upload.models import UploadType @@ -43,7 +44,7 @@ def compress_folder_to_zip(path): def assemble_path( - f: str, target_folder: str, project_code: str, zone: str, zipping: bool = False + f: str, target_folder: str, project_code: str, folder_type: FolderType, zone: str ) -> Tuple[str, Dict, bool, str]: ''' Summary: @@ -61,7 +62,6 @@ def assemble_path( - target_folder(str): the folder on the platform - project_code(str): the unique identifier of project - zone(str): the zone label eg.greenroom/core - - zipping(bool): default False. The flag to indicate if upload as a zip Return: - current_file_path: the format file path on platform - parent_folder: the item information of longest parent folder @@ -79,11 +79,11 @@ def assemble_path( # otherwise it is target_folder + f input name current_folder_node = target_folder if os.path.isfile(f) else current_file_path create_folder_flag = False - # always add `shared/` as prefix to folder/file if - # they directly under the project root folder - if parent_folder.get('type') == 'project_folder': - current_folder_node = 'shared/' + current_folder_node - target_folder = 'shared/' + target_folder + # add prefix to folder + current_folder_node = folder_type.get_prefix() + current_folder_node + target_folder = folder_type.get_prefix() + target_folder + + raise if len(current_file_path.split('/')) > 2: sub_path = target_folder.split('/') diff --git a/app/utils/aggregated.py b/app/utils/aggregated.py index 83bd7520..f8e3f4c1 100644 --- a/app/utils/aggregated.py +++ b/app/utils/aggregated.py @@ -8,6 +8,7 @@ from typing import Any from typing import Dict from typing import List +from typing import Tuple import httpx import requests @@ -16,6 +17,7 @@ from app.configs.app_config import AppConfig from app.configs.config import ConfigClass from app.configs.user_config import UserConfig +from app.models.folder import FolderType from app.services.output_manager.error_handler import ECustomizedError from app.services.output_manager.error_handler import SrvErrorHandler from app.services.user_authentication.decorator import require_valid_token @@ -164,19 +166,37 @@ def get_file_in_folder(path): return files_list -def identify_target_folder(project_path): - project_code = project_path.split('/')[0] - if len(project_path.split('/')) > 1: - target_folder = '/'.join(project_path.split('/')[1:]) +def identify_target_folder(project_path: str) -> Tuple[str, FolderType, str]: + ''' + Summary: + the function will validate if input folder path doesn't + contain invalid characters and return the project code and target folder + Parameters: + - project_path: the input folder path (eg. //) + Return: + - project_code: the project code + - folder_type: the folder type + - target_folder: the target folder + ''' + # split into project_code, folder_type, folder + temp_paths = project_path.split('/', 3) + project_code, folder_type, folder_name = temp_paths + if len(temp_paths) == 3: + # first check if folder names are valid + target_folder = '/'.join(folder_name.split('/')) for f in target_folder.split('/'): f = f.strip(' ') valid = validate_folder_name(f) if not valid: SrvErrorHandler.customized_handle(ECustomizedError.INVALID_FOLDERNAME, True) + + # check folder type if is project folder or name folder + folder_type = FolderType(folder_type) + else: SrvErrorHandler.customized_handle(ECustomizedError.INVALID_NAMEFOLDER, True) target_folder = '' - return project_code, target_folder + return project_code, folder_type, target_folder def batch_generator(iterable: List[Any], batch_size=1): From 9ea06775cfd8cdbeb35542e4fecc9681263f85ac Mon Sep 17 00:00:00 2001 From: zhiren Date: Tue, 19 Mar 2024 16:32:38 -0400 Subject: [PATCH 2/5] update upload logic to use projectfolder as the keyword to distinguish between project folder and name folder uploading --- app/models/folder.py | 2 +- .../file_manager/file_upload/file_upload.py | 6 +-- app/utils/aggregated.py | 42 ++++++++++++------- 3 files changed, 30 insertions(+), 20 deletions(-) diff --git a/app/models/folder.py b/app/models/folder.py index 1ec155a5..180c337c 100644 --- a/app/models/folder.py +++ b/app/models/folder.py @@ -15,7 +15,7 @@ def get_prefix(self) -> str: """Get the prefix for the folder type.""" prefix = { - 'namefolder': 'namefolder/', + 'namefolder': '', 'projectfolder': 'shared/', } diff --git a/app/services/file_manager/file_upload/file_upload.py b/app/services/file_manager/file_upload/file_upload.py index 548d88ea..4dda080b 100644 --- a/app/services/file_manager/file_upload/file_upload.py +++ b/app/services/file_manager/file_upload/file_upload.py @@ -72,8 +72,8 @@ def assemble_path( current_file_path = target_folder + '/' + f.rstrip('/').split('/')[-1] # set name folder as first parent folder - name_folder = target_folder.split('/')[0] - parent_folder = search_item(project_code, zone, name_folder).get('result', {}) + root_folder = target_folder.split('/')[0] + parent_folder = search_item(project_code, zone, root_folder).get('result', {}) # if f input is a file then current_folder_node is target_folder # otherwise it is target_folder + f input name @@ -83,8 +83,6 @@ def assemble_path( current_folder_node = folder_type.get_prefix() + current_folder_node target_folder = folder_type.get_prefix() + target_folder - raise - if len(current_file_path.split('/')) > 2: sub_path = target_folder.split('/') for index in range(len(sub_path) - 1): diff --git a/app/utils/aggregated.py b/app/utils/aggregated.py index f8e3f4c1..662db673 100644 --- a/app/utils/aggregated.py +++ b/app/utils/aggregated.py @@ -172,30 +172,42 @@ def identify_target_folder(project_path: str) -> Tuple[str, FolderType, str]: the function will validate if input folder path doesn't contain invalid characters and return the project code and target folder Parameters: - - project_path: the input folder path (eg. //) + - project_path: + - for project folder the input folder path (eg. /projectfolder/) + - for name folder the input folder path will be (eg. /) Return: - project_code: the project code - folder_type: the folder type - target_folder: the target folder ''' # split into project_code, folder_type, folder - temp_paths = project_path.split('/', 3) - project_code, folder_type, folder_name = temp_paths - if len(temp_paths) == 3: - # first check if folder names are valid - target_folder = '/'.join(folder_name.split('/')) - for f in target_folder.split('/'): - f = f.strip(' ') - valid = validate_folder_name(f) - if not valid: - SrvErrorHandler.customized_handle(ECustomizedError.INVALID_FOLDERNAME, True) - - # check folder type if is project folder or name folder - folder_type = FolderType(folder_type) - + temp_paths = project_path.split('/', 2) + project_code, folder_type, folder_name = temp_paths[0], '', '' + + # check folder type if is project folder or name folder + # there will be a extra string for project folder between project code and folder name + if len(temp_paths) == 2: + folder_type = FolderType.NAMEFOLDER + folder_name = temp_paths[1] + elif len(temp_paths) >= 3: + if temp_paths[1] == FolderType.PROJECTFOLDER.value: + folder_type = FolderType.PROJECTFOLDER + folder_name = temp_paths[2] + else: + folder_type = FolderType.NAMEFOLDER + folder_name = os.path.join(temp_paths[1], temp_paths[2]) else: SrvErrorHandler.customized_handle(ECustomizedError.INVALID_NAMEFOLDER, True) target_folder = '' + + # first check if folder names are valid + target_folder = '/'.join(folder_name.split('/')) + for f in target_folder.split('/'): + f = f.strip(' ') + valid = validate_folder_name(f) + if not valid: + SrvErrorHandler.customized_handle(ECustomizedError.INVALID_FOLDERNAME, True) + return project_code, folder_type, target_folder From 72e6b4de45c10bb3b2670eb597be4c06562b6158 Mon Sep 17 00:00:00 2001 From: zhiren Date: Tue, 19 Mar 2024 16:33:38 -0400 Subject: [PATCH 3/5] add more test cases for folder type --- tests/app/commands/test_file.py | 8 +++---- .../file_upload/test_file_upload.py | 13 ++++------- tests/app/utils/test_aggregated.py | 23 +++++++++++++++++++ 3 files changed, 31 insertions(+), 13 deletions(-) diff --git a/tests/app/commands/test_file.py b/tests/app/commands/test_file.py index d90912d1..3abe2cab 100644 --- a/tests/app/commands/test_file.py +++ b/tests/app/commands/test_file.py @@ -24,10 +24,6 @@ def test_file_upload_command_success_with_attribute(mocker, cli_runner): - project_code = 'test_project' - target_folder = 'admin' - - mocker.patch('app.commands.file.identify_target_folder', return_value=(project_code, target_folder)) mocker.patch('app.commands.file.validate_upload_event', return_value={'source_file': '', 'attribute': 'test'}) mocker.patch('app.commands.file.assemble_path', return_value=('test', {'id': 'id'}, True, 'test')) @@ -48,8 +44,10 @@ def test_file_upload_command_success_with_attribute(mocker, cli_runner): json.dump({'template': {'attr1': 'value'}}, f) result = cli_runner.invoke( - file_put, ['--project-path', 'test', '--thread', 1, '--attribute', 'template.json', 'test.txt'] + file_put, + ['--project-path', 'test_project/admin', '--thread', 1, '--attribute', 'template.json', 'test.txt'], ) + assert result.exit_code == 0 simple_upload_mock.assert_called_once() attribute_mock.assert_called_once() diff --git a/tests/app/services/file_manager/file_upload/test_file_upload.py b/tests/app/services/file_manager/file_upload/test_file_upload.py index 17b3e474..b4a9ffb2 100644 --- a/tests/app/services/file_manager/file_upload/test_file_upload.py +++ b/tests/app/services/file_manager/file_upload/test_file_upload.py @@ -3,6 +3,7 @@ # Contact Indoc Systems for any questions regarding the use of this source code. from app.configs.app_config import AppConfig +from app.models.folder import FolderType from app.services.file_manager.file_upload.file_upload import assemble_path from app.services.file_manager.file_upload.file_upload import resume_upload from app.services.file_manager.file_upload.file_upload import simple_upload @@ -17,7 +18,6 @@ def test_assemble_path_at_name_folder(mocker): target_folder = 'admin' project_code = 'test_project' zone = 0 - resumable_id = None mocker.patch( 'app.services.file_manager.file_upload.file_upload.search_item', @@ -34,7 +34,7 @@ def test_assemble_path_at_name_folder(mocker): ) current_file_path, parent_folder, create_folder_flag, _ = assemble_path( - local_file_path, target_folder, project_code, zone, resumable_id + local_file_path, target_folder, project_code, FolderType.NAMEFOLDER, zone ) assert current_file_path == 'admin/file.txt' assert parent_folder.get('name') == 'admin' @@ -46,7 +46,6 @@ def test_assemble_path_at_exsting_folder(mocker): target_folder = 'admin/test_folder_exist' project_code = 'test_project' zone = 0 - resumable_id = None node_list = [ { @@ -74,7 +73,7 @@ def test_assemble_path_at_exsting_folder(mocker): mocker.patch('app.services.file_manager.file_upload.file_upload.search_item', side_effect=node_list) current_file_path, parent_folder, create_folder_flag, _ = assemble_path( - local_file_path, target_folder, project_code, zone, resumable_id + local_file_path, target_folder, project_code, FolderType.NAMEFOLDER, zone ) assert current_file_path == 'admin/test_folder_exist/file.txt' assert parent_folder.get('name') == 'test_folder_exist' @@ -86,7 +85,6 @@ def test_assemble_path_at_non_existing_folder(mocker): target_folder = 'admin/test_folder_not_exist' project_code = 'test_project' zone = 0 - resumable_id = None node_list = [ { @@ -106,7 +104,7 @@ def test_assemble_path_at_non_existing_folder(mocker): mocker.patch('app.services.file_manager.file_upload.file_upload.click.confirm', return_value=None) current_file_path, parent_folder, create_folder_flag, _ = assemble_path( - local_file_path, target_folder, project_code, zone, resumable_id + local_file_path, target_folder, project_code, FolderType.NAMEFOLDER, zone ) assert current_file_path == 'admin/test_folder_not_exist' assert parent_folder.get('name') == 'admin' @@ -118,7 +116,6 @@ def test_assemble_path_at_project_folder(mocker): target_folder = 'project_folder' project_code = 'test_project' zone = 0 - resumable_id = None mocker.patch( 'app.services.file_manager.file_upload.file_upload.search_item', @@ -135,7 +132,7 @@ def test_assemble_path_at_project_folder(mocker): ) current_file_path, parent_folder, create_folder_flag, target_folder = assemble_path( - local_file_path, target_folder, project_code, zone, resumable_id + local_file_path, target_folder, project_code, FolderType.PROJECTFOLDER, zone ) assert current_file_path == 'shared/project_folder/file.txt' assert parent_folder.get('name') == 'project_folder' diff --git a/tests/app/utils/test_aggregated.py b/tests/app/utils/test_aggregated.py index 00ec31e5..88194ca2 100644 --- a/tests/app/utils/test_aggregated.py +++ b/tests/app/utils/test_aggregated.py @@ -5,7 +5,9 @@ import pytest from app.configs.app_config import AppConfig +from app.models.folder import FolderType from app.utils.aggregated import check_item_duplication +from app.utils.aggregated import identify_target_folder from app.utils.aggregated import search_item from app.utils.aggregated import validate_folder_name from tests.conftest import decoded_token @@ -129,3 +131,24 @@ def test_check_duplicate_fail_with_error_code(httpx_mock, mocker, capsys): def test_validate_folder_name(folder_name): valid = validate_folder_name(folder_name) assert valid is False + + +@pytest.mark.parametrize( + 'input_path,expected_result', + [ + ('project_code/username', ('project_code', FolderType.NAMEFOLDER, 'username')), + ('project_code/username/folder1', ('project_code', FolderType.NAMEFOLDER, 'username/folder1')), + ('project_code/projectfolder/folder1', ('project_code', FolderType.PROJECTFOLDER, 'folder1')), + ('project_code/projectfolder/folder1/folder2', ('project_code', FolderType.PROJECTFOLDER, 'folder1/folder2')), + ], +) +def test_identify_target_folder_success_with_different_path(mocker, input_path, expected_result): + mocker.patch('app.utils.aggregated.validate_folder_name', return_value=True) + result = identify_target_folder(input_path) + assert result == expected_result + + +def test_identify_target_folder_fail_with_invalid_input(mocker): + mocker.patch('app.utils.aggregated.validate_folder_name', return_value=False) + with pytest.raises(SystemExit): + identify_target_folder('project_code') From 30e6820b8e131c9d924a7472cbe319dfa35211ee Mon Sep 17 00:00:00 2001 From: zhiren Date: Tue, 19 Mar 2024 16:34:07 -0400 Subject: [PATCH 4/5] bump up to next version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index f04e7c59..07a4eeb4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "app" -version = "2.10.0a0" +version = "2.10.0" description = "This service is designed to support pilot platform" authors = ["Indoc Systems"] From 366aff96721c6fc91fdad4c1e0c01c62a739d304 Mon Sep 17 00:00:00 2001 From: zhiren Date: Thu, 21 Mar 2024 09:40:03 -0400 Subject: [PATCH 5/5] update help message for project folder --- app/resources/custom_help.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/resources/custom_help.py b/app/resources/custom_help.py index 6bf37de8..3e4b17e6 100644 --- a/app/resources/custom_help.py +++ b/app/resources/custom_help.py @@ -44,7 +44,9 @@ class HelpPage: 'FILE_SYNC_I': 'Enable downloading by geid.', 'FILE_SYNC_Z': 'Target Zone (i.e., core/greenroom).', 'FILE_UPLOAD_P': ( - 'Project folder path starting from Project Code. ' '(i.e., indoctestproject/namefolder//folder)' + 'Project folder path starting from Project Code(i.e. /user/folder). ' + 'A new key word `projectfolder` is required to specify project folder(i.e. ' + '/projectfolder/folder1)' ), 'FILE_UPLOAD_A': 'Add attributes to the file using a File Attribute Template.', 'FILE_UPLOAD_T': 'Add tags to the file using a Tag file.',