Rc/staging release 20230921

.github/workflows/build-and-publish.yml

@@ -61,12 +61,21 @@ jobs:
       - name: Install dependencies
         run: poetry install --no-interaction --no-root
 
-      - name: Build binary
+      - name: Build default binary
         run: poetry run pyinstaller -F --distpath ./app/bundled_app/linux --specpath ./app/build/linux --workpath ./app/build/linux --paths=./.venv/lib/python3.9/site-packages ./app/pilotcli.py -n ${{ github.sha }}
 
-      - name: Rename output file
+      - name: Rename default output file
         run: mv "./app/bundled_app/linux/${{ github.sha }}" "./app/bundled_app/linux/pilotcli_linux"
 
+      - name: Enable cloud mode
+        run: touch ./app/ENABLE_CLOUD_MODE
+
+      - name: Build cloud binary
+        run: poetry run pyinstaller -F --distpath ./app/bundled_app/linux --specpath ./app/build/linux --workpath ./app/build/linux --paths=./.venv/lib/python3.9/site-packages --add-binary=$(pwd)/app/ENABLE_CLOUD_MODE:. ./app/pilotcli.py -n ${{ github.sha }}
+
+      - name: Rename cloud output file
+        run: mv "./app/bundled_app/linux/${{ github.sha }}" "./app/bundled_app/linux/pilotcli_cloud"
+
       - name: Set version in env
         run: poetry run echo "TAG_VERSION=`poetry version --short`" >> $GITHUB_ENV
 
@@ -82,7 +91,9 @@ jobs:
           draft: false
           prerelease: false
           target_commitish: ${{ needs.extract-branch-name.outputs.branch }}
-          files: ./app/bundled_app/linux/pilotcli_linux
+          files: |
+            ./app/bundled_app/linux/pilotcli_linux
+            ./app/bundled_app/linux/pilotcli_cloud
 
   push-binary-macos:
     needs: [ push-binary-linux ]

app/configs/user_config.py

@@ -5,6 +5,7 @@
 import configparser
 import os
 import stat
+import sys
 import time
 from pathlib import Path
 from typing import Iterable
@@ -26,11 +27,27 @@ class UserConfig(metaclass=Singleton):
     This user config is global.
     """
 
-    def __init__(self, config_path: Union[str, Path, None] = None, config_filename: Union[str, None] = None) -> None:
+    def __init__(
+        self,
+        config_path: Union[str, Path, None] = None,
+        config_filename: Union[str, None] = None,
+        is_cloud_mode: Union[bool, None] = None,
+    ) -> None:
+        """When `is_cloud_mode` is enabled, it omits the checks for file or folder ownership and correct access mode for
+        the user.
+
+        This adjustment is made to prevent complications with mounted NFS volumes where all files have root ownership.
+        """
+
         if config_path is None:
             config_path = ConfigClass.config_path
         if config_filename is None:
             config_filename = ConfigClass.config_file
+        if is_cloud_mode is None:
+            # Check when code is bundled using pyinstaller
+            # https://pyinstaller.org/en/stable/runtime-information.html#run-time-information
+            is_bundled = getattr(sys, 'frozen', False) and hasattr(sys, '_MEIPASS')
+            is_cloud_mode = is_bundled and (Path(sys._MEIPASS) / 'ENABLE_CLOUD_MODE').is_file()
 
         config_path = Path(config_path)
         if not config_path.exists():
@@ -39,7 +56,7 @@ def __init__(self, config_path: Union[str, Path, None] = None, config_filename:
         current_user_id = os.geteuid()
 
         error = self._check_user_permissions(config_path, current_user_id, (0o0500, 0o0700))
-        if error:
+        if error and not is_cloud_mode:
             SrvErrorHandler.customized_handle(ECustomizedError.CONFIG_INVALID_PERMISSIONS, True, error)
             return
 
@@ -48,10 +65,11 @@ def __init__(self, config_path: Union[str, Path, None] = None, config_filename:
             config_file.touch(mode=0o0600, exist_ok=False)
 
         error = self._check_user_permissions(config_file, current_user_id, (0o0400, 0o0600))
-        if error:
+        if error and not is_cloud_mode:
             SrvErrorHandler.customized_handle(ECustomizedError.CONFIG_INVALID_PERMISSIONS, True, error)
             return
 
+        self.is_cloud_mode = is_cloud_mode
         self.config_file = config_file
         self.config = configparser.ConfigParser()
         self.config.read(self.config_file)

app/resources/custom_error.py

@@ -53,6 +53,8 @@ class Error:
         ),
         'UPLOAD_CANCEL': 'Upload task was cancelled.',
         'UPLOAD_FAIL': 'Upload task was failed. Please check the console output.',
+        'UPLOAD_SKIP_DUPLICATION': 'Following files with the same '
+        'name already exist in the Project: \n%s.\nDo you want to skip uploading',
         'UPLOAD_ID_NOT_EXIST': (
             'The specified multipart upload does not exist. '
             'The upload ID may be invalid, or the upload may have been aborted or completed.'

app/resources/custom_help.py

@@ -6,10 +6,9 @@
 class HelpPage:
     page = {
         'update': {
-            'version': '2.6.0',
-            '1': 'The logic of normal upload and resumble are splited. '
-            'add new command for resumable upload as `pilotcli file resume -r manifest.json`',
-            '2': 'The manifest file will be output for both file/folder upload',
+            'version': '2.7.1',
+            '1': 'Add new feature for folder merging',
+            '2': 'Secure the config file',
             '3': 'Optimize logic, input and error message',
         },
         'dataset': {

app/services/file_manager/file_upload/file_upload.py

@@ -2,7 +2,6 @@
 #
 # Contact Indoc Systems for any questions regarding the use of this source code.
 
-import math
 import os
 import time
 import zipfile
@@ -24,6 +23,7 @@
 from app.services.output_manager.error_handler import ECustomizedError
 from app.services.output_manager.error_handler import SrvErrorHandler
 from app.services.output_manager.error_handler import customized_error_msg
+from app.utils.aggregated import batch_generator
 from app.utils.aggregated import get_file_in_folder
 from app.utils.aggregated import get_file_info_by_geid
 from app.utils.aggregated import search_item
@@ -174,18 +174,32 @@ def simple_upload(  # noqa: C901
         if file_object.total_size == 0:
             logger.warning(f'Skip the file with 0 size: {file_object.file_name}')
         else:
-            file_objects.append(FileObject(object_path, file))
+            file_objects.append(file_object)
+
+    # make the file duplication check to allow folde merging
+    non_duplicate_file_objects = []
+    if create_folder_flag is True:
+        non_duplicate_file_objects = file_objects
+    else:
+        mhandler.SrvOutPutHandler.file_duplication_check()
+        duplicated_file = []
+        for file_batchs in batch_generator(file_objects, batch_size=AppConfig.Env.upload_batch_size):
+            non_duplicates, duplicate_path = upload_client.check_upload_duplication(file_batchs)
+            non_duplicate_file_objects.extend(non_duplicates)
+            duplicated_file.extend(duplicate_path)
+
+        if len(non_duplicate_file_objects) == 0:
+            mhandler.SrvOutPutHandler.file_duplication_check_warning_with_all_same()
+        elif len(duplicated_file) > 0:
+            mhandler.SrvOutPutHandler.file_duplication_check_success()
+            duplicate_warning_format = '\n'.join(duplicated_file)
+            click.confirm(
+                customized_error_msg(ECustomizedError.UPLOAD_SKIP_DUPLICATION) % (duplicate_warning_format), abort=True
+            )
 
-    # here add the batch of 500 per loop, the pre upload api cannot
-    # process very large amount of file at same time. otherwise it will timeout
-    num_of_batchs = math.ceil(len(file_objects) / AppConfig.Env.upload_batch_size)
     # here is list of pre upload result. We decided to call pre upload api by batch
     pre_upload_infos = []
-    for batch in range(0, num_of_batchs):
-        start_index = batch * AppConfig.Env.upload_batch_size
-        end_index = (batch + 1) * AppConfig.Env.upload_batch_size
-        file_batchs = file_objects[start_index:end_index]
-
+    for file_batchs in batch_generator(non_duplicate_file_objects, batch_size=AppConfig.Env.upload_batch_size):
         # sending the pre upload request to generate
         # the placeholder in object storage
         pre_upload_infos.extend(upload_client.pre_upload(file_batchs, output_path))
@@ -266,12 +280,8 @@ def resume_upload(
 
     # here add the batch of 500 per loop, the pre upload api cannot
     # process very large amount of file at same time. otherwise it will timeout
-    num_of_batchs = math.ceil(len(all_files) / AppConfig.Env.upload_batch_size)
     # here is list of pre upload result. We decided to call pre upload api by batch
-    for batch in range(0, num_of_batchs):
-        start_index = batch * AppConfig.Env.upload_batch_size
-        end_index = (batch + 1) * AppConfig.Env.upload_batch_size
-        file_batchs = item_ids[start_index:end_index]
+    for file_batchs in batch_generator(item_ids, batch_size=AppConfig.Env.upload_batch_size):
         items = get_file_info_by_geid(file_batchs)
 
         # get the detail of item to see if the file is already uploaded

app/services/file_manager/file_upload/upload_client.py

@@ -147,6 +147,40 @@ def resume_upload(self, unfinished_file_objects: List[FileObject]) -> List[FileO
 
         return unfinished_file_objects
 
+    @require_valid_token()
+    def check_upload_duplication(self, file_objects: List[FileObject]) -> Tuple[List[FileObject], List[str]]:
+        """
+        Summary:
+            The function will call the api to check if the file has been uploaded.
+            if yes, it will skip the file.
+        Parameter:
+            - file_objects(List[FileObject]): the file will be uploaded.
+        return:
+            - non_exist_file_objects(List[FileObject]): the file that need to be uploaded.
+            - exist_files(List[str]): the file that has been uploaded. will be skipped
+        """
+        headers = {'Authorization': 'Bearer ' + self.user.access_token, 'Session-ID': self.user.session_id}
+        url = AppConfig.Connections.url_base + '/portal/v1/files/exists'
+
+        # generate a list of locations for uploaded files to check duplication
+        # at same time, generate a dict of mapping with object_path: FileObject
+        locations = [x.object_path for x in file_objects]
+        object_path_file_object_map = {x.object_path: x for x in file_objects}
+
+        payload = {'locations': locations, 'container_code': self.project_code, 'container_type': 'project', 'zone': 0}
+        response = resilient_session().post(url, json=payload, headers=headers)
+
+        # pop the file object if the file has been uploaded
+        # return the file objects that need to be uploaded
+        if response.status_code == 200:
+            exist_files = response.json().get('result', [])
+            for exist_file_path in exist_files:
+                object_path_file_object_map.pop(exist_file_path)
+        else:
+            SrvErrorHandler.default_handle('Error when checking file duplication', if_exit=True)
+
+        return list(object_path_file_object_map.values()), exist_files
+
     @require_valid_token()
     def pre_upload(self, file_objects: List[FileObject], output_path: str) -> List[FileObject]:
         """
@@ -179,7 +213,7 @@ def pre_upload(self, file_objects: List[FileObject], output_path: str) -> List[F
             ],
         }
 
-        response = resilient_session().post(url, json=payload, headers=headers, timeout=None)
+        response = resilient_session().post(url, json=payload, headers=headers)
         if response.status_code == 200:
             result = response.json().get('result')
             file_mapping = {x.object_path: x for x in file_objects}

app/services/output_manager/error_handler.py

@@ -37,6 +37,7 @@ class ECustomizedError(enum.Enum):
     PERMISSION_DENIED = 'PERMISSION_DENIED'
     UPLOAD_CANCEL = 'UPLOAD_CANCEL'
     UPLOAD_FAIL = 'UPLOAD_FAIL'
+    UPLOAD_SKIP_DUPLICATION = 'UPLOAD_SKIP_DUPLICATION'
     # the error when multipart upload id is not exist
     UPLOAD_ID_NOT_EXIST = 'UPLOAD_ID_NOT_EXIST'
     MANIFEST_OF_FOLDER_FILE_EXIST = 'MANIFEST_OF_FOLDER_FILE_EXIST'

app/services/output_manager/message_handler.py

@@ -141,6 +141,21 @@ def preupload_success():
         """e.g. pre-upload succeed."""
         return logger.info('Pre-upload complete.')
 
+    @staticmethod
+    def file_duplication_check():
+        """e.g. file duplication check."""
+        return logger.info('Checking for file duplication...')
+
+    @staticmethod
+    def file_duplication_check_success():
+        """e.g. file duplication check succeed."""
+        return logger.info('File duplication check complete.')
+
+    @staticmethod
+    def file_duplication_check_warning_with_all_same():
+        """e.g. file duplication check warning with all same."""
+        return logger.warning('All files are the same, no need to upload.')
+
     @staticmethod
     def resume_check_success():
         """e.g. notify the resumable check succeed."""

app/utils/aggregated.py

@@ -5,6 +5,8 @@
 import os
 import re
 import shutil
+from typing import Any
+from typing import List
 
 import httpx
 import requests
@@ -132,3 +134,9 @@ def identify_target_folder(project_path):
         SrvErrorHandler.customized_handle(ECustomizedError.INVALID_NAMEFOLDER, True)
         target_folder = ''
     return project_code, target_folder
+
+
+def batch_generator(iterable: List[Any], batch_size=1):
+    max_size = len(iterable)
+    for start_index in range(0, max_size, batch_size):
+        yield iterable[start_index : min(start_index + batch_size, max_size)]

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "app"
-version = "2.6.0"
+version = "2.7.1"
 description = "This service is designed to support pilot platform"
 authors = ["Indoc Systems"]
 

tests/app/configs/test_user_config.py

@@ -4,6 +4,7 @@
 
 import os
 import stat
+import sys
 
 import pytest
 
@@ -73,3 +74,42 @@ def test__init__exits_with_error_when_config_file_does_not_have_expected_access_
         )
 
         error_log.assert_called_with(expected_message)
+
+    def test__init__does_not_exit_with_error_when_config_folder_has_invalid_access_mode_and_is_cloud_mode_set_to_true(
+        self, tmp_path, fake
+    ):
+        config_folder = tmp_path / fake.pystr()
+        config_folder.mkdir(mode=0o0755)
+
+        UserConfig(config_folder, is_cloud_mode=True)
+
+    def test__init__does_not_exit_with_error_when_config_file_has_invalid_access_mode_and_is_cloud_mode_set_to_true(
+        self, tmp_path, fake
+    ):
+        config_folder = tmp_path / fake.pystr()
+        file_name = fake.pystr()
+        config_file = config_folder / file_name
+        config_folder.mkdir(mode=0o0700)
+        config_file.touch(mode=0o0644)
+
+        UserConfig(config_folder, file_name, is_cloud_mode=True)
+
+    def test__init__sets_is_cloud_mode_to_false_by_default(self, tmp_path, fake):
+        config_folder = tmp_path / fake.pystr()
+
+        user_config = UserConfig(config_folder)
+
+        assert user_config.is_cloud_mode is False
+
+    def test__init__sets_is_cloud_mode_to_true_when_pyinstaller_bundle_params_are_set_and_cloud_mode_file_is_present(
+        self, tmp_path, monkeypatch
+    ):
+        monkeypatch.setattr(sys, 'frozen', True, raising=False)
+        monkeypatch.setattr(sys, '_MEIPASS', str(tmp_path), raising=False)
+
+        cloud_mode_file = tmp_path / 'ENABLE_CLOUD_MODE'
+        cloud_mode_file.touch()
+
+        user_config = UserConfig()
+
+        assert user_config.is_cloud_mode is True