Rc/staging release 20231123

.github/workflows/build-and-publish.yml

@@ -59,7 +59,7 @@ jobs:
         run: timeout 10s poetry run pip --version || rm -rf .venv
 
       - name: Install dependencies
-        run: poetry install --no-interaction --no-root
+        run: poetry install --no-interaction
 
       - name: Build default binary
         run: poetry run pyinstaller -F --distpath ./app/bundled_app/linux --specpath ./app/build/linux --workpath ./app/build/linux --paths=./.venv/lib/python3.9/site-packages ./app/pilotcli.py -n ${{ github.sha }}
@@ -130,7 +130,7 @@ jobs:
         run: timeout 10s poetry run pip --version || rm -rf .venv
 
       - name: Install dependencies
-        run: poetry install --no-interaction --no-root
+        run: poetry install --no-interaction
 
       - name: Build binary
         run: poetry run pyinstaller -F --distpath ./app/bundled_app/macos --specpath ./app/build/macos --workpath ./app/build/macos --paths=./.venv/lib/python3.9/site-packages ./app/pilotcli.py -n ${{ github.sha }}

.github/workflows/run-tests.yml

@@ -41,7 +41,7 @@ jobs:
         run: timeout 10s poetry run pip --version || rm -rf .venv
 
       - name: Install dependencies
-        run: poetry install --no-interaction --no-root
+        run: poetry install --no-interaction
 
       - name: Run pre-commit hooks
         uses: pre-commit/action@v3.0.0

app/commands/file.py

@@ -4,8 +4,10 @@
 
 import json
 import os
+from sys import exit
 
 import click
+from click.exceptions import Abort
 
 import app.services.output_manager.help_page as file_help
 import app.services.output_manager.message_handler as message_handler
@@ -125,16 +127,26 @@ def file_put(**kwargs):  # noqa: C901
     zone = get_zone(zone) if zone else AppConfig.Env.green_zone.lower()
 
     toc = customized_error_msg(ECustomizedError.TOU_CONTENT).replace(' ', '...')
-    if zone.lower() == AppConfig.Env.core_zone.lower() and click.confirm(fit_terminal_width(toc), abort=True):
-        pass
+    try:
+        if zone.lower() == AppConfig.Env.core_zone.lower() and click.confirm(fit_terminal_width(toc), abort=True):
+            pass
+    except Abort:
+        message_handler.SrvOutPutHandler.cancel_upload()
+        exit(1)
 
     # check if user input at least one file/folder
     if len(paths) == 0:
         SrvErrorHandler.customized_handle(ECustomizedError.INVALID_PATHS, True)
 
     # check if the manifest file exists
-    if os.path.exists(output_path):
-        click.confirm(customized_error_msg(ECustomizedError.MANIFEST_OF_FOLDER_FILE_EXIST) % (output_path), abort=True)
+    try:
+        if os.path.exists(output_path):
+            click.confirm(
+                customized_error_msg(ECustomizedError.MANIFEST_OF_FOLDER_FILE_EXIST) % (output_path), abort=True
+            )
+    except Abort:
+        message_handler.SrvOutPutHandler.cancel_upload()
+        exit(1)
 
     project_path = click.prompt('ProjectCode') if not project_path else project_path
     project_code, target_folder = identify_target_folder(project_path)
@@ -349,7 +361,7 @@ def file_list(paths, zone, page, page_size, detached):
         srv_list.list_files_with_pagination(paths, zone, page, page_size)
 
 
-@click.command(name='sync')
+@click.command(name='download')
 @click.argument('paths', type=click.STRING, nargs=-1)
 @click.argument('output_path', type=click.Path(exists=True), nargs=1)
 @click.option(

app/pilotcli.py

@@ -2,6 +2,8 @@
 #
 # Contact Indoc Systems for any questions regarding the use of this source code.
 
+from multiprocessing import freeze_support
+
 import click
 import requests
 
@@ -38,4 +40,5 @@ def cli():
 
 
 if __name__ == '__main__':
+    freeze_support()  # Add support for multiprocessing after bundling with PyInstaller
     cli()

app/resources/custom_error.py

@@ -109,10 +109,10 @@ class Error:
         'USER_NOT_FOUND': 'User not found.',
         'CONTAINER_REGISTRY_OTHER': 'Encountered an error when interacting with container registry.',
         'TOU_CONTENT': (
-            'You are about to transfer data directly to the PILOT Core! '
-            'In accordance with the PILOT Terms of Use, please confirm that you have made your best efforts '
+            'You are about to transfer data directly to the Pilot Core. '
+            'In accordance with the Pilot Terms of Use, please confirm that you have made your best efforts '
             'to pseudonymize or anonymize the data and that you have the legal authority to transfer and make this '
-            'data available for dissemination and use within the PILOT. If you need to process the data to remove '
+            'data available for dissemination and use within Pilot. If you need to process the data to remove '
             'sensitive identifiers, please cancel this transfer and upload the data to the Green Room to perform '
             'these actions.'
         ),

app/resources/custom_help.py

@@ -2,11 +2,13 @@
 #
 # Contact Indoc Systems for any questions regarding the use of this source code.
 
+import pkg_resources
+
 
 class HelpPage:
     page = {
         'update': {
-            'version': '2.7.4',
+            'version': pkg_resources.get_distribution('app').version,
             '1': 'Add new feature for folder merging',
             '2': 'Secure the config file',
             '3': 'Optimize logic, input and error message',

app/services/file_manager/file_upload/file_upload.py

@@ -6,12 +6,14 @@
 import time
 import zipfile
 from multiprocessing.pool import ThreadPool
+from sys import exit
 from typing import Any
 from typing import Dict
 from typing import List
 from typing import Tuple
 
 import click
+from click.exceptions import Abort
 
 import app.services.logger_services.log_functions as logger
 import app.services.output_manager.message_handler as mhandler
@@ -91,8 +93,14 @@ def assemble_path(
             # find the longest existing folder as parent folder
             # if user input a path that need to create some folders
             if not res.get('result'):
+                try:
+                    click.confirm(customized_error_msg(ECustomizedError.CREATE_FOLDER_IF_NOT_EXIST), abort=True)
+                except Abort:
+                    mhandler.SrvOutPutHandler.cancel_upload()
+                    exit(1)
+
+                # stop scaning and use the current folder as parent folder
                 current_folder_node = folder_path
-                click.confirm(customized_error_msg(ECustomizedError.CREATE_FOLDER_IF_NOT_EXIST), abort=True)
                 create_folder_flag = True
                 break
             else:
@@ -194,9 +202,14 @@ def simple_upload(  # noqa: C901
         elif len(duplicated_file) > 0:
             mhandler.SrvOutPutHandler.file_duplication_check_success()
             duplicate_warning_format = '\n'.join(duplicated_file)
-            click.confirm(
-                customized_error_msg(ECustomizedError.UPLOAD_SKIP_DUPLICATION) % (duplicate_warning_format), abort=True
-            )
+            try:
+                click.confirm(
+                    customized_error_msg(ECustomizedError.UPLOAD_SKIP_DUPLICATION) % (duplicate_warning_format),
+                    abort=True,
+                )
+            except Abort:
+                mhandler.SrvOutPutHandler.cancel_upload()
+                exit(1)
 
     # here is list of pre upload result. We decided to call pre upload api by batch
     pre_upload_infos = []

app/services/file_manager/file_upload/models.py

@@ -29,13 +29,15 @@ class ItemStatus(str, Enum):
         Enum type for item status where:
             - REGISTERED means file is created by upload service but not complete yet. either in progress or fail.
             - ACTIVE means file uploading is complete.
-            - ARCHIVED means the file has been deleted
+            - TRASHED means the file has been moved to the trash bin.
+            - DELETED means the file has been permanently deleted.
         The status will be stored at metadata table.
     """
 
     REGISTERED = 'REGISTERED'
     ACTIVE = 'ACTIVE'
-    ARCHIVED = 'ARCHIVED'
+    TRASHED = 'TRASHED'
+    DELETED = 'DELETED'
 
     def __str__(self):
         return self.name

app/services/file_manager/file_upload/upload_client.py

@@ -350,7 +350,7 @@ def upload_chunk(self, file_object: FileObject, chunk_number: int, chunk: str, e
             # request upload service to generate presigned url for the chunk
             params = {
                 'bucket': self.bucket,
-                'key': file_object.object_path,
+                'key': file_object.item_id,
                 'upload_id': file_object.resumable_id,
                 'chunk_number': chunk_number,
             }

app/services/output_manager/message_handler.py

@@ -191,6 +191,10 @@ def start_uploading(filename):
         """e.g. Start Uploading: ./test_file."""
         logger.info(f'Starting upload of: {filename}')
 
+    @staticmethod
+    def cancel_upload():
+        logger.warning('Upload cancelled.')
+
     @staticmethod
     def start_requests():
         """e.g. start requests."""

app/utils/aggregated.py

@@ -60,7 +60,7 @@ def get_file_info_by_geid(geid: list):
 
 
 def fit_terminal_width(string_to_format):
-    string_to_format = string_to_format.split('...')
+    string_to_format = string_to_format.rsplit('...')
     current_len = 0
     sentence = ''
     terminal_width = shutil.get_terminal_size().columns

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "app"
-version = "2.7.4"
+version = "2.8.5"
 description = "This service is designed to support pilot platform"
 authors = ["Indoc Systems"]
 

tests/app/commands/test_entry_point.py

@@ -60,7 +60,7 @@ def test_file_commands(user_login_true):
         'upload': file_put,
         'attribute-list': file_check_manifest,
         'attribute-export': file_export_manifest,
-        'sync': file_download,
+        'download': file_download,
         'resume': file_resume,
     }
     file_commands_object = entry_point.commands.get('file')

tests/app/utils/test_aggregated.py

@@ -21,7 +21,7 @@ def test_search_file_should_return_200(requests_mock, mocker):
                 'parent': 'parent-id',
                 'parent_path': 'folder1',
                 'restore_path': None,
-                'archived': False,
+                'status': 'ACTIVE',
                 'type': 'file',
                 'zone': 0,
                 'name': 'test-file',
@@ -42,7 +42,7 @@ def test_search_file_should_return_200(requests_mock, mocker):
         'parent': 'parent-id',
         'parent_path': 'folder1',
         'restore_path': None,
-        'archived': False,
+        'status': 'ACTIVE',
         'type': 'file',
         'zone': 0,
         'name': 'test-file',