Pilot 4952: Update cli listing command to allow user list specific files/folders under given root level folder

app/models/item.py

@@ -9,9 +9,10 @@ class ItemType(str, Enum):
     """The class to reflect the type of item in database."""
 
     FILE = 'file'
-    Folder = 'folder'
+    FOLDER = 'folder'
     NAMEFOLDER = 'name_folder'
     SHAREDFOLDER = 'project_folder'
+    ROOTFOLDER = 'root_folder'
 
     @classmethod
     def get_type_from_keyword(self, keyword: str):
@@ -22,17 +23,19 @@ def get_type_from_keyword(self, keyword: str):
         """
 
         alternative_mapping = {
-            'projectfolder': self.SHAREDFOLDER,
+            'shared': self.SHAREDFOLDER,
+            'users': self.NAMEFOLDER,
         }
 
-        return alternative_mapping.get(keyword, self.NAMEFOLDER)
+        return alternative_mapping.get(keyword, '')
 
     def get_prefix_by_type(self) -> str:
         """Get the prefix for the folder type."""
 
         prefix = {
-            self.NAMEFOLDER: '',
+            self.NAMEFOLDER: 'users/',
             self.SHAREDFOLDER: 'shared/',
+            self.ROOTFOLDER: '',
         }
 
         return prefix.get(self.value, '')

app/resources/custom_error.py

@@ -92,7 +92,7 @@ class Error:
         'INVALID_ACTION': 'Invalid action: %s',
         'DUPLICATE_TAG_ERROR': 'Cannot add duplicate tags',
         'INVALID_FOLDER': 'Provided folder does not exist',
-        'INVALID_NAMEFOLDER': 'User name folder is missing or provided user name folder does not exist',
+        'INVALID_PROJECT_PATH': 'root folder is missing or provided name/shared folder does not exist',
         'INVALID_DOWNLOAD': 'Invalid download, file/folder not exist or folder is empty: %s',
         # file metadata related error
         'LOCAL_METADATA_FILE_EXISTS': 'Following metadata file already exists in the local directory: ',

app/resources/custom_help.py

@@ -43,11 +43,7 @@ class HelpPage:
             'FILE_SYNC_ZIP': 'Download files as a zip.',
             'FILE_SYNC_I': 'Enable downloading by geid.',
             'FILE_SYNC_Z': 'Target Zone (i.e., core/greenroom).',
-            'FILE_UPLOAD_P': (
-                'Project folder path starting from Project Code(i.e. <project>/user/folder). '
-                'A new key word `projectfolder` is required to specify project folder(i.e. '
-                '<project>/projectfolder/folder1)'
-            ),
+            'FILE_UPLOAD_P': 'Project folder path starting from Project Code(i.e. <project>/users/admin/folder). ',
             'FILE_UPLOAD_A': 'Add attributes to the file using a File Attribute Template.',
             'FILE_UPLOAD_T': 'Add tags to the file using a Tag file.',
             'FILE_UPLOAD_M': 'The message used to comment on the purpose of uploading your processed file.',

app/services/file_manager/file_list.py

@@ -15,26 +15,21 @@
 from app.services.output_manager.error_handler import SrvErrorHandler
 from app.services.user_authentication.decorator import require_valid_token
 from app.utils.aggregated import fit_terminal_width
-from app.utils.aggregated import search_item
 
 
 class SrvFileList(metaclass=MetaService):
     user = UserConfig()
 
     @require_valid_token()
     def list_files(self, paths, zone, page, page_size):
+        # path is formatted as <project_code>/<root_folder>/<folder1>
+        # split the path in to project_code, root_folder, and folder1
         project_path = paths.strip('/').split('/')
-        project_code = project_path[0]
+        project_code, source_type = project_path[0], 'project'
         folder_rel_path = '/'.join(project_path[1:])
-        if len(project_path) == 1:
-            source_type = 'project'
-        else:
-            source_type = 'project'
-            res = search_item(project_code, zone, folder_rel_path)
-            parent_folder = res.get('result')
-            # if the target folder is project folder add the default path
-            if parent_folder.get('type') == 'project_folder':
-                folder_rel_path = 'shared/' + folder_rel_path
+        if len(project_path) > 1:
+            root_folder = ItemType.get_type_from_keyword(project_path[1])
+            folder_rel_path = folder_rel_path.replace(project_path[1], root_folder.get_prefix_by_type()[:-1], 1)
 
         # now query the backend to get the file list
         get_url = AppConfig.Connections.url_bff + f'/v1/{project_code}/files/query'
@@ -68,10 +63,6 @@ def list_files(self, paths, zone, page, page_size):
             if item_type == ItemType.FILE:
                 files = files + f.get('name') + ' ...'
             else:
-                # add [p] in front of the project folder
-                if item_type == ItemType.SHAREDFOLDER:
-                    f['name'] = f'[p]{f.get("name")}'
-
                 folders = folders + f"\033[34m{f.get('name')}\033[0m ..."
 
         f_string = folders + files

app/services/output_manager/error_handler.py

@@ -61,7 +61,7 @@ class ECustomizedError(enum.Enum):
     RESERVED_FOLDER = 'RESERVED_FOLDER'
     INVALID_ACTION = 'INVALID_ACTION'
     INVALID_FOLDER = 'INVALID_FOLDER'
-    INVALID_NAMEFOLDER = 'INVALID_NAMEFOLDER'
+    INVALID_PROJECT_PATH = 'INVALID_PROJECT_PATH'
     INVALID_DOWNLOAD = 'INVALID_DOWNLOAD'
     DUPLICATE_TAG_ERROR = 'DUPLICATE_TAG_ERROR'
     # file metadata related error

app/utils/aggregated.py

@@ -173,8 +173,8 @@ def identify_target_folder(project_path: str) -> Tuple[str, ItemType, str]:
         contain invalid characters and return the project code and target folder
     Parameters:
         - project_path:
-            - for project folder the input folder path (eg. <project_code>/projectfolder/<folder_name>)
-            - for name folder the input folder path will be (eg. <project_code>/<folder_name>)
+            - for project folder the input folder path (eg. <project_code>/shared/<folder_name>)
+            - for name folder the input folder path will be (eg. <project_code>/users/<folder_name>)
     Return:
         - project_code: the project code
         - folder_type: the folder type
@@ -186,17 +186,11 @@ def identify_target_folder(project_path: str) -> Tuple[str, ItemType, str]:
 
     # check folder type if is project folder or name folder
     # there will be a extra string for project folder between project code and folder name
-    if len(temp_paths) == 2:
-        folder_type = ItemType.NAMEFOLDER
-        folder_name = temp_paths[1]
-    elif len(temp_paths) >= 3:
+    if len(temp_paths) >= 3:
         folder_type = ItemType.get_type_from_keyword(temp_paths[1])
-        if folder_type == ItemType.SHAREDFOLDER:
-            folder_name = temp_paths[2]
-        else:
-            folder_name = os.path.join(temp_paths[1], temp_paths[2])
+        folder_name = temp_paths[2]
     else:
-        SrvErrorHandler.customized_handle(ECustomizedError.INVALID_NAMEFOLDER, True)
+        SrvErrorHandler.customized_handle(ECustomizedError.INVALID_PROJECT_PATH, True)
         target_folder = ''
 
     # first check if folder names are valid

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "app"
-version = "2.10.1"
+version = "3.0.0"
 description = "This service is designed to support pilot platform"
 authors = ["Indoc Systems"]
 

tests/app/commands/test_file.py

@@ -16,6 +16,7 @@
 from app.commands.file import file_move
 from app.commands.file import file_put
 from app.commands.file import file_resume
+from app.models.item import ItemType
 from app.services.file_manager.file_metadata.file_metadata_client import FileMetaClient
 from app.services.file_manager.file_upload.models import FileObject
 from app.services.output_manager.error_handler import ECustomizedError
@@ -45,7 +46,15 @@ def test_file_upload_command_success_with_attribute(mocker, cli_runner):
 
         result = cli_runner.invoke(
             file_put,
-            ['--project-path', 'test_project/admin', '--thread', 1, '--attribute', 'template.json', 'test.txt'],
+            [
+                '--project-path',
+                f'test_project/{ItemType.NAMEFOLDER.get_prefix_by_type()}admin',
+                '--thread',
+                1,
+                '--attribute',
+                'template.json',
+                'test.txt',
+            ],
         )
 
     assert result.exit_code == 0
@@ -134,69 +143,55 @@ def test_file_list_with_pagination_with_folder_success(requests_mock, mocker, cl
         return_value=decoded_token(),
     )
 
-    mocker.patch(
-        'app.services.file_manager.file_list.search_item',
-        return_value={
-            'result': {
-                'type': 'folder',
-                'id': 'id',
-            }
-        },
-    )
     requests_mock.get(
         'http://bff_cli' + '/v1/testproject/files/query',
         json={
             'code': 200,
             'error_msg': '',
             'result': [
-                {'type': 'file', 'name': 'file1'},
-                {'type': 'file', 'name': 'file2'},
+                {'type': ItemType.FILE.value, 'name': 'file1'},
+                {'type': ItemType.FILE.value, 'name': 'file2'},
             ],
         },
     )
     mocker.patch.object(questionary, 'select')
     questionary.select.return_value.ask.return_value = 'exit'
-    result = cli_runner.invoke(file_list, ['testproject/admin', '-z', 'greenroom'])
+    result = cli_runner.invoke(file_list, ['testproject/users/admin', '-z', 'greenroom'])
     outputs = result.output.split('\n')
     assert outputs[0] == 'file1  file2   '
 
 
-@pytest.mark.parametrize('parent_folder_type', ['name_folder', 'project_folder'])
-def test_file_list_with_pagination_with_name_project_folder(requests_mock, mocker, cli_runner, parent_folder_type):
+def test_file_list_with_pagination_with_root_folder(requests_mock, mocker, cli_runner):
     mocker.patch(
         'app.services.user_authentication.token_manager.SrvTokenManager.decode_access_token',
         return_value=decoded_token(),
     )
 
-    mocker.patch(
-        'app.services.file_manager.file_list.search_item',
-        return_value={
-            'result': {
-                'type': parent_folder_type,
-                'id': 'id',
-            }
-        },
-    )
+    folder = 'folder1'
+    folder_with_underline = 'folder_1'
+    folder_with_space = 'folder 1'
+    root_folder = 'root_folder'
     requests_mock.get(
         'http://bff_cli' + '/v1/testproject/files/query',
         json={
             'code': 200,
             'error_msg': '',
             'result': [
-                {'type': 'folder', 'name': 'folder1'},
-                {'type': 'name_folder', 'name': 'name_folder1'},
-                {'type': 'project_folder', 'name': 'project folder1'},
-                {'type': 'folder', 'name': 'test folder2'},
-                {'type': 'project_folder', 'name': 'project folder2'},
+                {'type': ItemType.FOLDER.value, 'name': folder},
+                {'type': ItemType.NAMEFOLDER.value, 'name': folder_with_underline},
+                {'type': ItemType.SHAREDFOLDER.value, 'name': folder_with_underline},
+                {'type': ItemType.FOLDER.value, 'name': folder_with_space},
+                {'type': ItemType.SHAREDFOLDER.value, 'name': folder_with_space},
+                {'type': ItemType.ROOTFOLDER.value, 'name': root_folder},
             ],
         },
     )
     mocker.patch.object(questionary, 'select')
     questionary.select.return_value.ask.return_value = 'exit'
-    result = cli_runner.invoke(file_list, ['testproject/admin', '-z', 'greenroom'])
+    result = cli_runner.invoke(file_list, ['testproject/users/admin', '-z', 'greenroom'])
     outputs = result.output.split('\n')
-    assert outputs[0] == 'folder1  name_folder1  [p]"project folder1"  '
-    assert outputs[1] == '"test folder2"  [p]"project folder2"   '
+    assert outputs[0] == f'{folder}  {folder_with_underline}  {folder_with_underline}  "{folder_with_space}"  '
+    assert outputs[1] == f'"{folder_with_space}"  {root_folder}   '
 
 
 def test_empty_file_list_with_pagination(requests_mock, mocker, cli_runner):
@@ -205,15 +200,6 @@ def test_empty_file_list_with_pagination(requests_mock, mocker, cli_runner):
         return_value=decoded_token(),
     )
 
-    mocker.patch(
-        'app.services.file_manager.file_list.search_item',
-        return_value={
-            'result': {
-                'type': 'folder',
-                'id': 'id',
-            }
-        },
-    )
     requests_mock.get(
         'http://bff_cli' + '/v1/testproject/files/query',
         json={'code': 200, 'error_msg': '', 'result': []},
@@ -222,10 +208,10 @@ def test_empty_file_list_with_pagination(requests_mock, mocker, cli_runner):
     questionary.select.return_value.ask.return_value = 'exit'
     result = cli_runner.invoke(file_list, ['testproject/admin', '-z', 'greenroom'])
     outputs = result.output.split('\n')
-    assert outputs[0] == ' '
+    assert outputs[0] == ''
 
 
-@pytest.mark.parametrize('parent_folder_type', ['name_folder', 'project_folder'])
+@pytest.mark.parametrize('parent_folder_type', [ItemType.NAMEFOLDER.value, ItemType.SHAREDFOLDER.value])
 def test_file_download_success(requests_mock, mocker, cli_runner, parent_folder_type):
     mocker.patch(
         'app.services.user_authentication.token_manager.SrvTokenManager.decode_access_token',
@@ -246,7 +232,7 @@ def test_file_download_success(requests_mock, mocker, cli_runner, parent_folder_
             {
                 'code': 200,
                 'result': {
-                    'type': 'file',
+                    'type': ItemType.FILE.value,
                     'id': 'id',
                 },
             },

tests/app/services/file_manager/file_upload/test_file_upload.py

@@ -36,7 +36,7 @@ def test_assemble_path_at_name_folder(mocker):
     current_file_path, parent_folder, create_folder_flag, _ = assemble_path(
         local_file_path, target_folder, project_code, ItemType.NAMEFOLDER, zone
     )
-    assert current_file_path == 'admin/file.txt'
+    assert current_file_path == f'{ItemType.NAMEFOLDER.get_prefix_by_type()}admin/file.txt'
     assert parent_folder.get('name') == 'admin'
     assert create_folder_flag is False
 
@@ -53,6 +53,16 @@ def test_assemble_path_at_exsting_folder(mocker):
                 'id': 'test',
                 'parent_id': 'test_parent',
                 'parent_path': '',
+                'name': ItemType.NAMEFOLDER.get_prefix_by_type().strip('/'),
+                'zone': 0,
+                'type': 'folder',
+            }
+        },
+        {
+            'result': {
+                'id': 'test',
+                'parent_id': 'test_parent',
+                'parent_path': ItemType.NAMEFOLDER.get_prefix_by_type(),
                 'name': 'admin',
                 'zone': 0,
                 'type': 'folder',
@@ -62,7 +72,7 @@ def test_assemble_path_at_exsting_folder(mocker):
             'result': {
                 'id': 'test',
                 'parent_id': 'test_parent',
-                'parent_path': 'admin',
+                'parent_path': ItemType.NAMEFOLDER.get_prefix_by_type() + 'admin',
                 'name': 'test_folder_exist',
                 'zone': 0,
                 'type': 'folder',
@@ -74,7 +84,7 @@ def test_assemble_path_at_exsting_folder(mocker):
     current_file_path, parent_folder, create_folder_flag, _ = assemble_path(
         local_file_path, target_folder, project_code, ItemType.NAMEFOLDER, zone
     )
-    assert current_file_path == 'admin/test_folder_exist/file.txt'
+    assert current_file_path == f'{ItemType.NAMEFOLDER.get_prefix_by_type()}admin/test_folder_exist/file.txt'
     assert parent_folder.get('name') == 'test_folder_exist'
     assert create_folder_flag is False
 
@@ -91,6 +101,16 @@ def test_assemble_path_at_non_existing_folder(mocker):
                 'id': 'test',
                 'parent_id': 'test_parent',
                 'parent_path': '',
+                'name': ItemType.NAMEFOLDER.get_prefix_by_type().strip('/'),
+                'zone': 0,
+                'type': 'folder',
+            }
+        },
+        {
+            'result': {
+                'id': 'test',
+                'parent_id': 'test_parent',
+                'parent_path': ItemType.NAMEFOLDER.get_prefix_by_type(),
                 'name': 'admin',
                 'zone': 0,
                 'type': 'folder',
@@ -105,7 +125,7 @@ def test_assemble_path_at_non_existing_folder(mocker):
     current_file_path, parent_folder, create_folder_flag, _ = assemble_path(
         local_file_path, target_folder, project_code, ItemType.NAMEFOLDER, zone
     )
-    assert current_file_path == 'admin/test_folder_not_exist'
+    assert current_file_path == f'{ItemType.NAMEFOLDER.get_prefix_by_type()}admin/test_folder_not_exist'
     assert parent_folder.get('name') == 'admin'
     assert create_folder_flag is True
 
@@ -133,9 +153,9 @@ def test_assemble_path_at_project_folder(mocker):
     current_file_path, parent_folder, create_folder_flag, target_folder = assemble_path(
         local_file_path, target_folder, project_code, ItemType.SHAREDFOLDER, zone
     )
-    assert current_file_path == 'shared/project_folder/file.txt'
+    assert current_file_path == f'{ItemType.SHAREDFOLDER.get_prefix_by_type()}project_folder/file.txt'
     assert parent_folder.get('name') == 'project_folder'
-    assert target_folder == 'shared/project_folder'
+    assert target_folder == f'{ItemType.SHAREDFOLDER.get_prefix_by_type()}project_folder'
     assert create_folder_flag is False