diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index 8ceb69c6..12fa811b 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -59,7 +59,7 @@ jobs: run: timeout 10s poetry run pip --version || rm -rf .venv - name: Install dependencies - run: poetry install --no-interaction --no-root + run: poetry install --no-interaction - name: Build default binary run: poetry run pyinstaller -F --distpath ./app/bundled_app/linux --specpath ./app/build/linux --workpath ./app/build/linux --paths=./.venv/lib/python3.9/site-packages ./app/pilotcli.py -n ${{ github.sha }} @@ -130,7 +130,7 @@ jobs: run: timeout 10s poetry run pip --version || rm -rf .venv - name: Install dependencies - run: poetry install --no-interaction --no-root + run: poetry install --no-interaction - name: Build binary run: poetry run pyinstaller -F --distpath ./app/bundled_app/macos --specpath ./app/build/macos --workpath ./app/build/macos --paths=./.venv/lib/python3.9/site-packages ./app/pilotcli.py -n ${{ github.sha }} diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 73ae9ac0..6dee81e8 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -41,7 +41,7 @@ jobs: run: timeout 10s poetry run pip --version || rm -rf .venv - name: Install dependencies - run: poetry install --no-interaction --no-root + run: poetry install --no-interaction - name: Run pre-commit hooks uses: pre-commit/action@v3.0.0 diff --git a/app/commands/file.py b/app/commands/file.py index a77a0dc0..c3bacfaf 100644 --- a/app/commands/file.py +++ b/app/commands/file.py @@ -4,8 +4,10 @@ import json import os +from sys import exit import click +from click.exceptions import Abort import app.services.output_manager.help_page as file_help import app.services.output_manager.message_handler as message_handler @@ -125,16 +127,26 @@ def file_put(**kwargs): # noqa: C901 zone = get_zone(zone) if zone else AppConfig.Env.green_zone.lower() toc = customized_error_msg(ECustomizedError.TOU_CONTENT).replace(' ', '...') - if zone.lower() == AppConfig.Env.core_zone.lower() and click.confirm(fit_terminal_width(toc), abort=True): - pass + try: + if zone.lower() == AppConfig.Env.core_zone.lower() and click.confirm(fit_terminal_width(toc), abort=True): + pass + except Abort: + message_handler.SrvOutPutHandler.cancel_upload() + exit(1) # check if user input at least one file/folder if len(paths) == 0: SrvErrorHandler.customized_handle(ECustomizedError.INVALID_PATHS, True) # check if the manifest file exists - if os.path.exists(output_path): - click.confirm(customized_error_msg(ECustomizedError.MANIFEST_OF_FOLDER_FILE_EXIST) % (output_path), abort=True) + try: + if os.path.exists(output_path): + click.confirm( + customized_error_msg(ECustomizedError.MANIFEST_OF_FOLDER_FILE_EXIST) % (output_path), abort=True + ) + except Abort: + message_handler.SrvOutPutHandler.cancel_upload() + exit(1) project_path = click.prompt('ProjectCode') if not project_path else project_path project_code, target_folder = identify_target_folder(project_path) @@ -349,7 +361,7 @@ def file_list(paths, zone, page, page_size, detached): srv_list.list_files_with_pagination(paths, zone, page, page_size) -@click.command(name='sync') +@click.command(name='download') @click.argument('paths', type=click.STRING, nargs=-1) @click.argument('output_path', type=click.Path(exists=True), nargs=1) @click.option( diff --git a/app/pilotcli.py b/app/pilotcli.py index 8d882d8e..a2a25f12 100644 --- a/app/pilotcli.py +++ b/app/pilotcli.py @@ -2,6 +2,8 @@ # # Contact Indoc Systems for any questions regarding the use of this source code. +from multiprocessing import freeze_support + import click import requests @@ -38,4 +40,5 @@ def cli(): if __name__ == '__main__': + freeze_support() # Add support for multiprocessing after bundling with PyInstaller cli() diff --git a/app/resources/custom_error.py b/app/resources/custom_error.py index a8eff967..16a1c4f4 100644 --- a/app/resources/custom_error.py +++ b/app/resources/custom_error.py @@ -109,10 +109,10 @@ class Error: 'USER_NOT_FOUND': 'User not found.', 'CONTAINER_REGISTRY_OTHER': 'Encountered an error when interacting with container registry.', 'TOU_CONTENT': ( - 'You are about to transfer data directly to the PILOT Core! ' - 'In accordance with the PILOT Terms of Use, please confirm that you have made your best efforts ' + 'You are about to transfer data directly to the Pilot Core. ' + 'In accordance with the Pilot Terms of Use, please confirm that you have made your best efforts ' 'to pseudonymize or anonymize the data and that you have the legal authority to transfer and make this ' - 'data available for dissemination and use within the PILOT. If you need to process the data to remove ' + 'data available for dissemination and use within Pilot. If you need to process the data to remove ' 'sensitive identifiers, please cancel this transfer and upload the data to the Green Room to perform ' 'these actions.' ), diff --git a/app/resources/custom_help.py b/app/resources/custom_help.py index 1b708a11..65b6aec3 100644 --- a/app/resources/custom_help.py +++ b/app/resources/custom_help.py @@ -2,11 +2,13 @@ # # Contact Indoc Systems for any questions regarding the use of this source code. +import pkg_resources + class HelpPage: page = { 'update': { - 'version': '2.7.4', + 'version': pkg_resources.get_distribution('app').version, '1': 'Add new feature for folder merging', '2': 'Secure the config file', '3': 'Optimize logic, input and error message', diff --git a/app/services/file_manager/file_upload/file_upload.py b/app/services/file_manager/file_upload/file_upload.py index 93224751..64c600e4 100644 --- a/app/services/file_manager/file_upload/file_upload.py +++ b/app/services/file_manager/file_upload/file_upload.py @@ -6,12 +6,14 @@ import time import zipfile from multiprocessing.pool import ThreadPool +from sys import exit from typing import Any from typing import Dict from typing import List from typing import Tuple import click +from click.exceptions import Abort import app.services.logger_services.log_functions as logger import app.services.output_manager.message_handler as mhandler @@ -91,8 +93,14 @@ def assemble_path( # find the longest existing folder as parent folder # if user input a path that need to create some folders if not res.get('result'): + try: + click.confirm(customized_error_msg(ECustomizedError.CREATE_FOLDER_IF_NOT_EXIST), abort=True) + except Abort: + mhandler.SrvOutPutHandler.cancel_upload() + exit(1) + + # stop scaning and use the current folder as parent folder current_folder_node = folder_path - click.confirm(customized_error_msg(ECustomizedError.CREATE_FOLDER_IF_NOT_EXIST), abort=True) create_folder_flag = True break else: @@ -194,9 +202,14 @@ def simple_upload( # noqa: C901 elif len(duplicated_file) > 0: mhandler.SrvOutPutHandler.file_duplication_check_success() duplicate_warning_format = '\n'.join(duplicated_file) - click.confirm( - customized_error_msg(ECustomizedError.UPLOAD_SKIP_DUPLICATION) % (duplicate_warning_format), abort=True - ) + try: + click.confirm( + customized_error_msg(ECustomizedError.UPLOAD_SKIP_DUPLICATION) % (duplicate_warning_format), + abort=True, + ) + except Abort: + mhandler.SrvOutPutHandler.cancel_upload() + exit(1) # here is list of pre upload result. We decided to call pre upload api by batch pre_upload_infos = [] diff --git a/app/services/file_manager/file_upload/models.py b/app/services/file_manager/file_upload/models.py index fa0875e9..8d0b481b 100644 --- a/app/services/file_manager/file_upload/models.py +++ b/app/services/file_manager/file_upload/models.py @@ -29,13 +29,15 @@ class ItemStatus(str, Enum): Enum type for item status where: - REGISTERED means file is created by upload service but not complete yet. either in progress or fail. - ACTIVE means file uploading is complete. - - ARCHIVED means the file has been deleted + - TRASHED means the file has been moved to the trash bin. + - DELETED means the file has been permanently deleted. The status will be stored at metadata table. """ REGISTERED = 'REGISTERED' ACTIVE = 'ACTIVE' - ARCHIVED = 'ARCHIVED' + TRASHED = 'TRASHED' + DELETED = 'DELETED' def __str__(self): return self.name diff --git a/app/services/file_manager/file_upload/upload_client.py b/app/services/file_manager/file_upload/upload_client.py index 52ed7933..bea34f29 100644 --- a/app/services/file_manager/file_upload/upload_client.py +++ b/app/services/file_manager/file_upload/upload_client.py @@ -350,7 +350,7 @@ def upload_chunk(self, file_object: FileObject, chunk_number: int, chunk: str, e # request upload service to generate presigned url for the chunk params = { 'bucket': self.bucket, - 'key': file_object.object_path, + 'key': file_object.item_id, 'upload_id': file_object.resumable_id, 'chunk_number': chunk_number, } diff --git a/app/services/output_manager/message_handler.py b/app/services/output_manager/message_handler.py index a6aab665..e7fbfd24 100644 --- a/app/services/output_manager/message_handler.py +++ b/app/services/output_manager/message_handler.py @@ -191,6 +191,10 @@ def start_uploading(filename): """e.g. Start Uploading: ./test_file.""" logger.info(f'Starting upload of: {filename}') + @staticmethod + def cancel_upload(): + logger.warning('Upload cancelled.') + @staticmethod def start_requests(): """e.g. start requests.""" diff --git a/app/utils/aggregated.py b/app/utils/aggregated.py index 725cb753..a6d13cfe 100644 --- a/app/utils/aggregated.py +++ b/app/utils/aggregated.py @@ -60,7 +60,7 @@ def get_file_info_by_geid(geid: list): def fit_terminal_width(string_to_format): - string_to_format = string_to_format.split('...') + string_to_format = string_to_format.rsplit('...') current_len = 0 sentence = '' terminal_width = shutil.get_terminal_size().columns diff --git a/pyproject.toml b/pyproject.toml index 7bf376f9..eab325d4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "app" -version = "2.7.4" +version = "2.8.5" description = "This service is designed to support pilot platform" authors = ["Indoc Systems"] diff --git a/tests/app/commands/test_entry_point.py b/tests/app/commands/test_entry_point.py index b95ea560..2085de4b 100644 --- a/tests/app/commands/test_entry_point.py +++ b/tests/app/commands/test_entry_point.py @@ -60,7 +60,7 @@ def test_file_commands(user_login_true): 'upload': file_put, 'attribute-list': file_check_manifest, 'attribute-export': file_export_manifest, - 'sync': file_download, + 'download': file_download, 'resume': file_resume, } file_commands_object = entry_point.commands.get('file') diff --git a/tests/app/utils/test_aggregated.py b/tests/app/utils/test_aggregated.py index 5617376e..9ddadd2f 100644 --- a/tests/app/utils/test_aggregated.py +++ b/tests/app/utils/test_aggregated.py @@ -21,7 +21,7 @@ def test_search_file_should_return_200(requests_mock, mocker): 'parent': 'parent-id', 'parent_path': 'folder1', 'restore_path': None, - 'archived': False, + 'status': 'ACTIVE', 'type': 'file', 'zone': 0, 'name': 'test-file', @@ -42,7 +42,7 @@ def test_search_file_should_return_200(requests_mock, mocker): 'parent': 'parent-id', 'parent_path': 'folder1', 'restore_path': None, - 'archived': False, + 'status': 'ACTIVE', 'type': 'file', 'zone': 0, 'name': 'test-file',