Rc/staging release 20240522

README.md

@@ -1,4 +1,4 @@
-# command_line_tool_ctl
+# Pilot CLI
 
 [![Run Tests](https://github.com/PilotDataPlatform/cli/actions/workflows/run-tests.yml/badge.svg?branch=develop)](https://github.com/PilotDataPlatform/cli/actions/workflows/run-tests.yml)
 [![Python](https://img.shields.io/badge/python-3.7-brightgreen.svg)](https://www.python.org/)

app/commands/file.py

@@ -12,6 +12,7 @@
 import app.services.output_manager.help_page as file_help
 import app.services.output_manager.message_handler as message_handler
 from app.configs.app_config import AppConfig
+from app.models.item import ItemType
 from app.services.file_manager.file_download.download_client import SrvFileDownload
 from app.services.file_manager.file_list import SrvFileList
 from app.services.file_manager.file_manifests import SrvFileManifests
@@ -429,14 +430,12 @@ def file_download(**kwargs):
     else:
         item_res = []
         for path in paths:
-            project_code, root_folder = path.strip('/').split('/')[:2]
-            target_path = '/'.join(path.split('/')[1::])
-            # search the root to check for name folder or project folder
-            root_item = search_item(project_code, zone, root_folder).get('result', {})
-            target_path = 'shared/' + target_path if root_item.get('type') == 'project_folder' else target_path
+            project_code, root_folder, object_path = path.strip('/').split('/', 2)
+            root_type = ItemType.get_type_from_keyword(root_folder)
+            object_path = os.path.join(root_type.get_prefix_by_type(), object_path)
 
             # search the target item and download to local
-            item = search_item(project_code, zone, target_path)
+            item = search_item(project_code, zone, object_path)
             if item.get('code') == 200 and item.get('result'):
                 item_status = 'success'
                 item_result = item.get('result')
@@ -544,8 +543,21 @@ def file_move(**kwargs):
     zone = kwargs.get('zone')
     skip_confirm = kwargs.get('yes')
 
+    if len(src_item_path.split('/')) == 1 and len(dest_item_path.split('/')) == 1:
+        raise Exception('Invalid path')
+
+    # tranlate keyword to correct object path
+    src_keyword, src_path = src_item_path.split('/', 1)
+    dest_keyword, dest_path = dest_item_path.split('/', 1)
+
+    src_type = ItemType.get_type_from_keyword(src_keyword)
+    dest_type = ItemType.get_type_from_keyword(dest_keyword)
+
+    src_path = src_type.get_prefix_by_type() + src_path
+    dest_path = dest_type.get_prefix_by_type() + dest_path
+
     zone = get_zone(zone) if zone else AppConfig.Env.green_zone.lower()
-    file_meta_client = FileMoveClient(zone, project_code, src_item_path, dest_item_path, skip_confirm=skip_confirm)
+    file_meta_client = FileMoveClient(zone, project_code, src_path, dest_path, skip_confirm=skip_confirm)
     file_meta_client.move_file()
 
     message_handler.SrvOutPutHandler.move_action_success(src_item_path, dest_item_path)

app/models/item.py

@@ -9,9 +9,10 @@ class ItemType(str, Enum):
     """The class to reflect the type of item in database."""
 
     FILE = 'file'
-    Folder = 'folder'
+    FOLDER = 'folder'
     NAMEFOLDER = 'name_folder'
     SHAREDFOLDER = 'project_folder'
+    ROOTFOLDER = 'root_folder'
 
     @classmethod
     def get_type_from_keyword(self, keyword: str):
@@ -22,17 +23,24 @@ def get_type_from_keyword(self, keyword: str):
         """
 
         alternative_mapping = {
-            'projectfolder': self.SHAREDFOLDER,
+            'shared': self.SHAREDFOLDER,
+            'users': self.NAMEFOLDER,
         }
 
-        return alternative_mapping.get(keyword, self.NAMEFOLDER)
+        return alternative_mapping.get(keyword, self.FOLDER)
 
     def get_prefix_by_type(self) -> str:
         """Get the prefix for the folder type."""
 
         prefix = {
-            self.NAMEFOLDER: '',
+            self.NAMEFOLDER: 'users/',
             self.SHAREDFOLDER: 'shared/',
+            self.ROOTFOLDER: '',
         }
 
         return prefix.get(self.value, '')
+
+
+class ItemZone(str, Enum):
+    GREENROOM = 'greenroom'
+    CORE = 'core'

app/resources/custom_error.py

@@ -92,7 +92,7 @@ class Error:
         'INVALID_ACTION': 'Invalid action: %s',
         'DUPLICATE_TAG_ERROR': 'Cannot add duplicate tags',
         'INVALID_FOLDER': 'Provided folder does not exist',
-        'INVALID_NAMEFOLDER': 'User name folder is missing or provided user name folder does not exist',
+        'INVALID_PROJECT_PATH': 'root folder is missing or provided name/shared folder does not exist',
         'INVALID_DOWNLOAD': 'Invalid download, file/folder not exist or folder is empty: %s',
         # file metadata related error
         'LOCAL_METADATA_FILE_EXISTS': 'Following metadata file already exists in the local directory: ',

app/resources/custom_help.py

@@ -9,9 +9,9 @@ class HelpPage:
     page = {
         'update': {
             'version': pkg_resources.get_distribution('app').version,
-            '1': 'Add new feature for folder merging',
-            '2': 'Secure the config file',
-            '3': 'Optimize logic, input and error message',
+            '1': 'Add a new feature for file metadata downloading.',
+            '2': 'Add a new feature for file moving/renaming.',
+            '3': 'Support shared folder within project.',
         },
         'dataset': {
             'DATASET_DOWNLOAD': 'Download a dataset or a particular version of a dataset.',
@@ -43,11 +43,7 @@ class HelpPage:
             'FILE_SYNC_ZIP': 'Download files as a zip.',
             'FILE_SYNC_I': 'Enable downloading by geid.',
             'FILE_SYNC_Z': 'Target Zone (i.e., core/greenroom).',
-            'FILE_UPLOAD_P': (
-                'Project folder path starting from Project Code(i.e. <project>/user/folder). '
-                'A new key word `projectfolder` is required to specify project folder(i.e. '
-                '<project>/projectfolder/folder1)'
-            ),
+            'FILE_UPLOAD_P': 'Project folder path starting from Project Code(i.e. <project>/users/admin/folder). ',
             'FILE_UPLOAD_A': 'Add attributes to the file using a File Attribute Template.',
             'FILE_UPLOAD_T': 'Add tags to the file using a Tag file.',
             'FILE_UPLOAD_M': 'The message used to comment on the purpose of uploading your processed file.',

app/services/file_manager/file_download/download_client.py

@@ -15,6 +15,7 @@
 import app.services.output_manager.message_handler as mhandler
 from app.configs.app_config import AppConfig
 from app.configs.user_config import UserConfig
+from app.models.item import ItemZone
 from app.models.service_meta_class import MetaService
 from app.services.output_manager.error_handler import ECustomizedError
 from app.services.output_manager.error_handler import SrvErrorHandler
@@ -53,7 +54,7 @@ def print_prepare_msg(self, message):
                 click.secho(f"{message}{'.'*i}\r", fg='white', nl=False)
 
     def get_download_url(self, zone):
-        if zone == 'greenroom':
+        if zone == ItemZone.GREENROOM.value:
             url = self.appconfig.Connections.url_download_greenroom
         else:
             url = self.appconfig.Connections.url_download_core
@@ -101,11 +102,11 @@ def prepare_download(self):
             file_path = download_info.get('file_path')
             pre_status = EFileStatus(response.get('status'))
         elif res.status_code == 403:
-            SrvErrorHandler.customized_handle(ECustomizedError.NO_FILE_PERMMISION, self.interactive)
+            SrvErrorHandler.customized_handle(ECustomizedError.NO_FILE_PERMMISION, if_exit=self.interactive)
         elif res.status_code == 400 and 'number of file must greater than 0' in res_json.get('error_msg'):
-            SrvErrorHandler.customized_handle(ECustomizedError.FOLDER_EMPTY, self.interactive)
+            SrvErrorHandler.customized_handle(ECustomizedError.FOLDER_EMPTY, if_exit=self.interactive)
         else:
-            SrvErrorHandler.customized_handle(ECustomizedError.DOWNLOAD_FAIL, self.interactive)
+            SrvErrorHandler.customized_handle(ECustomizedError.DOWNLOAD_FAIL, if_exit=self.interactive)
 
         return pre_status, file_path
 

app/services/file_manager/file_download/model.py

@@ -5,11 +5,8 @@
 from enum import Enum
 
 
-class EFileStatus(Enum):
+class EFileStatus(str, Enum):
     WAITING = 'WAITING'
     RUNNING = 'RUNNING'
     SUCCEED = 'SUCCEED'
     FAILED = 'FAILED'
-
-    def __str__(self):
-        return '%s' % self.name

app/services/file_manager/file_list.py

@@ -15,26 +15,21 @@
 from app.services.output_manager.error_handler import SrvErrorHandler
 from app.services.user_authentication.decorator import require_valid_token
 from app.utils.aggregated import fit_terminal_width
-from app.utils.aggregated import search_item
 
 
 class SrvFileList(metaclass=MetaService):
     user = UserConfig()
 
     @require_valid_token()
     def list_files(self, paths, zone, page, page_size):
+        # path is formatted as <project_code>/<root_folder>/<folder1>
+        # split the path in to project_code, root_folder, and folder1
         project_path = paths.strip('/').split('/')
-        project_code = project_path[0]
+        project_code, source_type = project_path[0], 'project'
         folder_rel_path = '/'.join(project_path[1:])
-        if len(project_path) == 1:
-            source_type = 'project'
-        else:
-            source_type = 'project'
-            res = search_item(project_code, zone, folder_rel_path)
-            parent_folder = res.get('result')
-            # if the target folder is project folder add the default path
-            if parent_folder.get('type') == 'project_folder':
-                folder_rel_path = 'shared/' + folder_rel_path
+        if len(project_path) > 1:
+            root_folder = ItemType.get_type_from_keyword(project_path[1])
+            folder_rel_path = folder_rel_path.replace(project_path[1], root_folder.get_prefix_by_type()[:-1], 1)
 
         # now query the backend to get the file list
         get_url = AppConfig.Connections.url_bff + f'/v1/{project_code}/files/query'
@@ -68,10 +63,6 @@ def list_files(self, paths, zone, page, page_size):
             if item_type == ItemType.FILE:
                 files = files + f.get('name') + ' ...'
             else:
-                # add [p] in front of the project folder
-                if item_type == ItemType.SHAREDFOLDER:
-                    f['name'] = f'[p]{f.get("name")}'
-
                 folders = folders + f"\033[34m{f.get('name')}\033[0m ..."
 
         f_string = folders + files

app/services/file_manager/file_metadata/file_metadata_client.py

@@ -19,6 +19,7 @@
 
 import app.services.logger_services.log_functions as logger
 import app.services.output_manager.message_handler as message_handler
+from app.models.item import ItemType
 from app.services.output_manager.error_handler import ECustomizedError
 from app.services.output_manager.error_handler import customized_error_msg
 from app.utils.aggregated import get_attribute_template_by_id
@@ -109,14 +110,15 @@ def download_file_metadata(self) -> List[Dict[str, Any]]:
         """
 
         project_code, object_path = self.file_path.split('/', 1)
+        root_folder, object_path = object_path.split('/', 1)
+        root_type = ItemType.get_type_from_keyword(root_folder)
+        object_path = join(root_type.get_prefix_by_type(), object_path)
         item_res = search_item(project_code, self.zone, object_path)
-        # double check if the file is in shared folder
         if item_res.get('code') == 404:
-            item_res = search_item(project_code, self.zone, f'shared/{object_path}')
-            if item_res.get('code') == 404:
-                logger.error(f'Cannot find item {self.file_path} at {self.zone}.')
-                exit(1)
+            logger.error(f'Cannot find item {self.file_path} at {self.zone}.')
+            exit(1)
 
+        # filter out item metadata
         item_res = item_res.get('result', {})
         extra_info = item_res.pop('extended', {}).get('extra')
         tags = extra_info.get('tags', [])

app/services/file_manager/file_move/file_move_client.py

@@ -60,11 +60,6 @@ def create_object_path_if_not_exist(self, folder_path: str) -> dict:
         """
 
         path_list = folder_path.split('/')
-        # first get the root folder to check if it is name folder
-        # or project folder
-        root_item = search_item(self.project_code, self.zone, path_list[0]).get('result')
-        if root_item.get('type') == 'project_folder':
-            path_list[0] = '/'.join([root_item.get('parent_path'), path_list[0]])
 
         # first check every folder in path exist or not
         # the loop start with index 1 since we assume cli will not

app/services/file_manager/file_upload/file_upload.py

@@ -69,10 +69,9 @@ def assemble_path(
          - target_folder: result object path on platform
 
     '''
-
     current_file_path = target_folder + '/' + f.rstrip('/').split('/')[-1]
     # set name folder as first parent folder
-    root_folder = target_folder.split('/')[0]
+    root_folder = folder_type.get_prefix_by_type() + target_folder.split('/')[0]
     parent_folder = search_item(project_code, zone, root_folder).get('result', {})
 
     # if f input is a file then current_folder_node is target_folder

app/services/output_manager/error_handler.py

@@ -61,7 +61,7 @@ class ECustomizedError(enum.Enum):
     RESERVED_FOLDER = 'RESERVED_FOLDER'
     INVALID_ACTION = 'INVALID_ACTION'
     INVALID_FOLDER = 'INVALID_FOLDER'
-    INVALID_NAMEFOLDER = 'INVALID_NAMEFOLDER'
+    INVALID_PROJECT_PATH = 'INVALID_PROJECT_PATH'
     INVALID_DOWNLOAD = 'INVALID_DOWNLOAD'
     DUPLICATE_TAG_ERROR = 'DUPLICATE_TAG_ERROR'
     # file metadata related error

app/services/project_manager/project.py

@@ -48,4 +48,4 @@ def list_projects(self, page, page_size, order, order_by):
             else:
                 SrvErrorHandler.default_handle(response.content, True)
         except Exception:
-            SrvErrorHandler.default_handle(response.content, True)
+            SrvErrorHandler.default_handle('Error when listing projects', True)

app/utils/aggregated.py

@@ -173,8 +173,8 @@ def identify_target_folder(project_path: str) -> Tuple[str, ItemType, str]:
         contain invalid characters and return the project code and target folder
     Parameters:
         - project_path:
-            - for project folder the input folder path (eg. <project_code>/projectfolder/<folder_name>)
-            - for name folder the input folder path will be (eg. <project_code>/<folder_name>)
+            - for project folder the input folder path (eg. <project_code>/shared/<folder_name>)
+            - for name folder the input folder path will be (eg. <project_code>/users/<folder_name>)
     Return:
         - project_code: the project code
         - folder_type: the folder type
@@ -186,17 +186,11 @@ def identify_target_folder(project_path: str) -> Tuple[str, ItemType, str]:
 
     # check folder type if is project folder or name folder
     # there will be a extra string for project folder between project code and folder name
-    if len(temp_paths) == 2:
-        folder_type = ItemType.NAMEFOLDER
-        folder_name = temp_paths[1]
-    elif len(temp_paths) >= 3:
+    if len(temp_paths) >= 3:
         folder_type = ItemType.get_type_from_keyword(temp_paths[1])
-        if folder_type == ItemType.SHAREDFOLDER:
-            folder_name = temp_paths[2]
-        else:
-            folder_name = os.path.join(temp_paths[1], temp_paths[2])
+        folder_name = temp_paths[2]
     else:
-        SrvErrorHandler.customized_handle(ECustomizedError.INVALID_NAMEFOLDER, True)
+        SrvErrorHandler.customized_handle(ECustomizedError.INVALID_PROJECT_PATH, True)
         target_folder = ''
 
     # first check if folder names are valid

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "app"
-version = "2.10.2"
+version = "3.0.3"
 description = "This service is designed to support pilot platform"
 authors = ["Indoc Systems"]