From ba7ed8e3772aa7de3f90087732a278f3f821359b Mon Sep 17 00:00:00 2001 From: zhiren Date: Wed, 27 Sep 2023 15:04:45 -0400 Subject: [PATCH 01/17] merge back with next preminor --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index f2b2b44d..a59ef366 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "app" -version = "2.7.3" +version = "2.8.0a0" description = "This service is designed to support pilot platform" authors = ["Indoc Systems"] From 7a05c068480fb54704c89732f8d00d66d4058ac3 Mon Sep 17 00:00:00 2001 From: zhiren Date: Wed, 27 Sep 2023 15:06:02 -0400 Subject: [PATCH 02/17] update version in user help page --- app/resources/custom_help.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/resources/custom_help.py b/app/resources/custom_help.py index 3b6412ef..a90ff648 100644 --- a/app/resources/custom_help.py +++ b/app/resources/custom_help.py @@ -6,7 +6,7 @@ class HelpPage: page = { 'update': { - 'version': '2.7.2', + 'version': '2.8.0a0', '1': 'Add new feature for folder merging', '2': 'Secure the config file', '3': 'Optimize logic, input and error message', From d65412ac10993aceb3c6fb175161ada7df3ff5c9 Mon Sep 17 00:00:00 2001 From: Color Zhan Date: Tue, 3 Oct 2023 10:08:50 -0400 Subject: [PATCH 03/17] PILOT-3589: use the item id for the presigned url api (#101) * use the item id for the presigned url api * bumpup version --------- Co-authored-by: zhiren --- app/services/file_manager/file_upload/upload_client.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/services/file_manager/file_upload/upload_client.py b/app/services/file_manager/file_upload/upload_client.py index 52ed7933..bea34f29 100644 --- a/app/services/file_manager/file_upload/upload_client.py +++ b/app/services/file_manager/file_upload/upload_client.py @@ -350,7 +350,7 @@ def upload_chunk(self, file_object: FileObject, chunk_number: int, chunk: str, e # request upload service to generate presigned url for the chunk params = { 'bucket': self.bucket, - 'key': file_object.object_path, + 'key': file_object.item_id, 'upload_id': file_object.resumable_id, 'chunk_number': chunk_number, } diff --git a/pyproject.toml b/pyproject.toml index a59ef366..d442ac23 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "app" -version = "2.8.0a0" +version = "2.8.0" description = "This service is designed to support pilot platform" authors = ["Indoc Systems"] From 3dcdd2e4c31bba12610e788ccac41b9b78cdbce6 Mon Sep 17 00:00:00 2001 From: QXgu Date: Fri, 13 Oct 2023 15:37:58 -0400 Subject: [PATCH 04/17] PILOT-3921: Update the ARCHIVED to TRASHED for metadata item status --- app/services/file_manager/file_upload/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/services/file_manager/file_upload/models.py b/app/services/file_manager/file_upload/models.py index fa0875e9..7db18404 100644 --- a/app/services/file_manager/file_upload/models.py +++ b/app/services/file_manager/file_upload/models.py @@ -29,13 +29,13 @@ class ItemStatus(str, Enum): Enum type for item status where: - REGISTERED means file is created by upload service but not complete yet. either in progress or fail. - ACTIVE means file uploading is complete. - - ARCHIVED means the file has been deleted + - TRASHED means the file has been deleted The status will be stored at metadata table. """ REGISTERED = 'REGISTERED' ACTIVE = 'ACTIVE' - ARCHIVED = 'ARCHIVED' + TRASHED = 'TRASHED' def __str__(self): return self.name From 4b889d20399d435a101f771f636d4f9f4aa54a67 Mon Sep 17 00:00:00 2001 From: QXgu Date: Tue, 17 Oct 2023 10:59:55 -0400 Subject: [PATCH 05/17] Update the unit test --- tests/app/utils/test_aggregated.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/app/utils/test_aggregated.py b/tests/app/utils/test_aggregated.py index 5617376e..9ddadd2f 100644 --- a/tests/app/utils/test_aggregated.py +++ b/tests/app/utils/test_aggregated.py @@ -21,7 +21,7 @@ def test_search_file_should_return_200(requests_mock, mocker): 'parent': 'parent-id', 'parent_path': 'folder1', 'restore_path': None, - 'archived': False, + 'status': 'ACTIVE', 'type': 'file', 'zone': 0, 'name': 'test-file', @@ -42,7 +42,7 @@ def test_search_file_should_return_200(requests_mock, mocker): 'parent': 'parent-id', 'parent_path': 'folder1', 'restore_path': None, - 'archived': False, + 'status': 'ACTIVE', 'type': 'file', 'zone': 0, 'name': 'test-file', From c5a0d9d673cf115fd0ba361ae12bbc6495446718 Mon Sep 17 00:00:00 2001 From: vadimsoltan Date: Thu, 19 Oct 2023 16:52:49 -0400 Subject: [PATCH 06/17] PILOT-3931: Added DELETED ItemStatus. --- app/services/file_manager/file_upload/models.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/services/file_manager/file_upload/models.py b/app/services/file_manager/file_upload/models.py index 7db18404..8d0b481b 100644 --- a/app/services/file_manager/file_upload/models.py +++ b/app/services/file_manager/file_upload/models.py @@ -29,13 +29,15 @@ class ItemStatus(str, Enum): Enum type for item status where: - REGISTERED means file is created by upload service but not complete yet. either in progress or fail. - ACTIVE means file uploading is complete. - - TRASHED means the file has been deleted + - TRASHED means the file has been moved to the trash bin. + - DELETED means the file has been permanently deleted. The status will be stored at metadata table. """ REGISTERED = 'REGISTERED' ACTIVE = 'ACTIVE' TRASHED = 'TRASHED' + DELETED = 'DELETED' def __str__(self): return self.name From 96eb81033ff07898f1485ded8429aef4596e5f4a Mon Sep 17 00:00:00 2001 From: zhiren Date: Tue, 7 Nov 2023 11:43:31 -0500 Subject: [PATCH 07/17] update the error message when user abort folder merging --- app/services/file_manager/file_upload/file_upload.py | 12 +++++++++--- app/services/output_manager/message_handler.py | 4 ++++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/app/services/file_manager/file_upload/file_upload.py b/app/services/file_manager/file_upload/file_upload.py index 93224751..9ad2b925 100644 --- a/app/services/file_manager/file_upload/file_upload.py +++ b/app/services/file_manager/file_upload/file_upload.py @@ -12,6 +12,7 @@ from typing import Tuple import click +from click.exceptions import Abort import app.services.logger_services.log_functions as logger import app.services.output_manager.message_handler as mhandler @@ -194,9 +195,14 @@ def simple_upload( # noqa: C901 elif len(duplicated_file) > 0: mhandler.SrvOutPutHandler.file_duplication_check_success() duplicate_warning_format = '\n'.join(duplicated_file) - click.confirm( - customized_error_msg(ECustomizedError.UPLOAD_SKIP_DUPLICATION) % (duplicate_warning_format), abort=True - ) + try: + click.confirm( + customized_error_msg(ECustomizedError.UPLOAD_SKIP_DUPLICATION) % (duplicate_warning_format), + abort=True, + ) + except Abort: + mhandler.SrvOutPutHandler.cancel_upload() + exit(1) # here is list of pre upload result. We decided to call pre upload api by batch pre_upload_infos = [] diff --git a/app/services/output_manager/message_handler.py b/app/services/output_manager/message_handler.py index a6aab665..e7fbfd24 100644 --- a/app/services/output_manager/message_handler.py +++ b/app/services/output_manager/message_handler.py @@ -191,6 +191,10 @@ def start_uploading(filename): """e.g. Start Uploading: ./test_file.""" logger.info(f'Starting upload of: {filename}') + @staticmethod + def cancel_upload(): + logger.warning('Upload cancelled.') + @staticmethod def start_requests(): """e.g. start requests.""" From ee426d5f1f93216e179665eb2c8ab5b370629088 Mon Sep 17 00:00:00 2001 From: zhiren Date: Tue, 7 Nov 2023 11:44:47 -0500 Subject: [PATCH 08/17] update command sync to download --- app/commands/file.py | 2 +- tests/app/commands/test_entry_point.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/commands/file.py b/app/commands/file.py index a77a0dc0..58871f25 100644 --- a/app/commands/file.py +++ b/app/commands/file.py @@ -349,7 +349,7 @@ def file_list(paths, zone, page, page_size, detached): srv_list.list_files_with_pagination(paths, zone, page, page_size) -@click.command(name='sync') +@click.command(name='download') @click.argument('paths', type=click.STRING, nargs=-1) @click.argument('output_path', type=click.Path(exists=True), nargs=1) @click.option( diff --git a/tests/app/commands/test_entry_point.py b/tests/app/commands/test_entry_point.py index b95ea560..2085de4b 100644 --- a/tests/app/commands/test_entry_point.py +++ b/tests/app/commands/test_entry_point.py @@ -60,7 +60,7 @@ def test_file_commands(user_login_true): 'upload': file_put, 'attribute-list': file_check_manifest, 'attribute-export': file_export_manifest, - 'sync': file_download, + 'download': file_download, 'resume': file_resume, } file_commands_object = entry_point.commands.get('file') From 8bacc7e02f4f7f3a1cd7ec113d1ca271db65e20f Mon Sep 17 00:00:00 2001 From: zhiren Date: Tue, 7 Nov 2023 11:45:06 -0500 Subject: [PATCH 09/17] bumpup the version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index d442ac23..3378cefa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "app" -version = "2.8.0" +version = "2.8.1" description = "This service is designed to support pilot platform" authors = ["Indoc Systems"] From f09613ab9ec2842fa0ec6dcb979aef9291a8c8aa Mon Sep 17 00:00:00 2001 From: zhiren Date: Thu, 9 Nov 2023 09:42:54 -0500 Subject: [PATCH 10/17] fixup the exit does not defined when abort folder merging. --- app/resources/custom_help.py | 4 +++- app/services/file_manager/file_upload/file_upload.py | 1 + pyproject.toml | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/app/resources/custom_help.py b/app/resources/custom_help.py index a90ff648..65b6aec3 100644 --- a/app/resources/custom_help.py +++ b/app/resources/custom_help.py @@ -2,11 +2,13 @@ # # Contact Indoc Systems for any questions regarding the use of this source code. +import pkg_resources + class HelpPage: page = { 'update': { - 'version': '2.8.0a0', + 'version': pkg_resources.get_distribution('app').version, '1': 'Add new feature for folder merging', '2': 'Secure the config file', '3': 'Optimize logic, input and error message', diff --git a/app/services/file_manager/file_upload/file_upload.py b/app/services/file_manager/file_upload/file_upload.py index 9ad2b925..d663eee0 100644 --- a/app/services/file_manager/file_upload/file_upload.py +++ b/app/services/file_manager/file_upload/file_upload.py @@ -6,6 +6,7 @@ import time import zipfile from multiprocessing.pool import ThreadPool +from sys import exit from typing import Any from typing import Dict from typing import List diff --git a/pyproject.toml b/pyproject.toml index 3378cefa..c6e565f2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "app" -version = "2.8.1" +version = "2.8.2" description = "This service is designed to support pilot platform" authors = ["Indoc Systems"] From e48eaf277d469074b3791a8834a052732c025e77 Mon Sep 17 00:00:00 2001 From: zhiren Date: Thu, 9 Nov 2023 17:36:11 -0500 Subject: [PATCH 11/17] remove the unnecessary item_type in the file search api --- app/utils/aggregated.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/utils/aggregated.py b/app/utils/aggregated.py index 725cb753..f706e559 100644 --- a/app/utils/aggregated.py +++ b/app/utils/aggregated.py @@ -34,7 +34,7 @@ def search_item(project_code, zone, folder_relative_path, item_type, container_t 'zone': zone, 'project_code': project_code, 'path': folder_relative_path, - 'item_type': item_type, + # 'item_type': item_type, 'container_type': container_type, } headers = {'Authorization': 'Bearer ' + token} From 23d28aaafe84b6e4be18e589ec8058c7b2e70117 Mon Sep 17 00:00:00 2001 From: zhiren Date: Thu, 9 Nov 2023 17:37:19 -0500 Subject: [PATCH 12/17] rever the changes --- app/utils/aggregated.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/utils/aggregated.py b/app/utils/aggregated.py index f706e559..725cb753 100644 --- a/app/utils/aggregated.py +++ b/app/utils/aggregated.py @@ -34,7 +34,7 @@ def search_item(project_code, zone, folder_relative_path, item_type, container_t 'zone': zone, 'project_code': project_code, 'path': folder_relative_path, - # 'item_type': item_type, + 'item_type': item_type, 'container_type': container_type, } headers = {'Authorization': 'Bearer ' + token} From 05b1772e12524040432bdcbdc6f80a71ec5581a7 Mon Sep 17 00:00:00 2001 From: zhiren Date: Fri, 17 Nov 2023 17:21:42 -0500 Subject: [PATCH 13/17] the cli version will require the poetry to install the root --- .github/workflows/build-and-publish.yml | 4 ++-- .github/workflows/run-tests.yml | 2 +- pyproject.toml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index 8ceb69c6..12fa811b 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -59,7 +59,7 @@ jobs: run: timeout 10s poetry run pip --version || rm -rf .venv - name: Install dependencies - run: poetry install --no-interaction --no-root + run: poetry install --no-interaction - name: Build default binary run: poetry run pyinstaller -F --distpath ./app/bundled_app/linux --specpath ./app/build/linux --workpath ./app/build/linux --paths=./.venv/lib/python3.9/site-packages ./app/pilotcli.py -n ${{ github.sha }} @@ -130,7 +130,7 @@ jobs: run: timeout 10s poetry run pip --version || rm -rf .venv - name: Install dependencies - run: poetry install --no-interaction --no-root + run: poetry install --no-interaction - name: Build binary run: poetry run pyinstaller -F --distpath ./app/bundled_app/macos --specpath ./app/build/macos --workpath ./app/build/macos --paths=./.venv/lib/python3.9/site-packages ./app/pilotcli.py -n ${{ github.sha }} diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 73ae9ac0..6dee81e8 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -41,7 +41,7 @@ jobs: run: timeout 10s poetry run pip --version || rm -rf .venv - name: Install dependencies - run: poetry install --no-interaction --no-root + run: poetry install --no-interaction - name: Run pre-commit hooks uses: pre-commit/action@v3.0.0 diff --git a/pyproject.toml b/pyproject.toml index c6e565f2..73b9a7cf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "app" -version = "2.8.2" +version = "2.8.3" description = "This service is designed to support pilot platform" authors = ["Indoc Systems"] From 40004a6b531ff35d31bb9fd40e82f631e9d8aa36 Mon Sep 17 00:00:00 2001 From: Vadym Moshynskyi Date: Mon, 20 Nov 2023 15:34:50 +0100 Subject: [PATCH 14/17] PILOT-3643: Fix issue with multiprocessing in pyinstaller (#109) --- app/pilotcli.py | 3 +++ pyproject.toml | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/app/pilotcli.py b/app/pilotcli.py index 8d882d8e..a2a25f12 100644 --- a/app/pilotcli.py +++ b/app/pilotcli.py @@ -2,6 +2,8 @@ # # Contact Indoc Systems for any questions regarding the use of this source code. +from multiprocessing import freeze_support + import click import requests @@ -38,4 +40,5 @@ def cli(): if __name__ == '__main__': + freeze_support() # Add support for multiprocessing after bundling with PyInstaller cli() diff --git a/pyproject.toml b/pyproject.toml index 73b9a7cf..d373d752 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "app" -version = "2.8.3" +version = "2.8.4" description = "This service is designed to support pilot platform" authors = ["Indoc Systems"] From b2db5f2360de5a62fb4b943819c9d49814d09550 Mon Sep 17 00:00:00 2001 From: Color Zhan Date: Wed, 22 Nov 2023 12:56:51 -0500 Subject: [PATCH 15/17] Pilot 4067: fixup the core zone upload message and abort error handler (#110) * fix up the message format when uploading to core. add proper handler when click recieve abort * bumpup version --------- Co-authored-by: zhiren --- app/commands/file.py | 20 +++++++++++++++---- app/resources/custom_error.py | 6 +++--- .../file_manager/file_upload/file_upload.py | 8 +++++++- app/utils/aggregated.py | 2 +- pyproject.toml | 2 +- 5 files changed, 28 insertions(+), 10 deletions(-) diff --git a/app/commands/file.py b/app/commands/file.py index 58871f25..c3bacfaf 100644 --- a/app/commands/file.py +++ b/app/commands/file.py @@ -4,8 +4,10 @@ import json import os +from sys import exit import click +from click.exceptions import Abort import app.services.output_manager.help_page as file_help import app.services.output_manager.message_handler as message_handler @@ -125,16 +127,26 @@ def file_put(**kwargs): # noqa: C901 zone = get_zone(zone) if zone else AppConfig.Env.green_zone.lower() toc = customized_error_msg(ECustomizedError.TOU_CONTENT).replace(' ', '...') - if zone.lower() == AppConfig.Env.core_zone.lower() and click.confirm(fit_terminal_width(toc), abort=True): - pass + try: + if zone.lower() == AppConfig.Env.core_zone.lower() and click.confirm(fit_terminal_width(toc), abort=True): + pass + except Abort: + message_handler.SrvOutPutHandler.cancel_upload() + exit(1) # check if user input at least one file/folder if len(paths) == 0: SrvErrorHandler.customized_handle(ECustomizedError.INVALID_PATHS, True) # check if the manifest file exists - if os.path.exists(output_path): - click.confirm(customized_error_msg(ECustomizedError.MANIFEST_OF_FOLDER_FILE_EXIST) % (output_path), abort=True) + try: + if os.path.exists(output_path): + click.confirm( + customized_error_msg(ECustomizedError.MANIFEST_OF_FOLDER_FILE_EXIST) % (output_path), abort=True + ) + except Abort: + message_handler.SrvOutPutHandler.cancel_upload() + exit(1) project_path = click.prompt('ProjectCode') if not project_path else project_path project_code, target_folder = identify_target_folder(project_path) diff --git a/app/resources/custom_error.py b/app/resources/custom_error.py index a8eff967..16a1c4f4 100644 --- a/app/resources/custom_error.py +++ b/app/resources/custom_error.py @@ -109,10 +109,10 @@ class Error: 'USER_NOT_FOUND': 'User not found.', 'CONTAINER_REGISTRY_OTHER': 'Encountered an error when interacting with container registry.', 'TOU_CONTENT': ( - 'You are about to transfer data directly to the PILOT Core! ' - 'In accordance with the PILOT Terms of Use, please confirm that you have made your best efforts ' + 'You are about to transfer data directly to the Pilot Core. ' + 'In accordance with the Pilot Terms of Use, please confirm that you have made your best efforts ' 'to pseudonymize or anonymize the data and that you have the legal authority to transfer and make this ' - 'data available for dissemination and use within the PILOT. If you need to process the data to remove ' + 'data available for dissemination and use within Pilot. If you need to process the data to remove ' 'sensitive identifiers, please cancel this transfer and upload the data to the Green Room to perform ' 'these actions.' ), diff --git a/app/services/file_manager/file_upload/file_upload.py b/app/services/file_manager/file_upload/file_upload.py index d663eee0..64c600e4 100644 --- a/app/services/file_manager/file_upload/file_upload.py +++ b/app/services/file_manager/file_upload/file_upload.py @@ -93,8 +93,14 @@ def assemble_path( # find the longest existing folder as parent folder # if user input a path that need to create some folders if not res.get('result'): + try: + click.confirm(customized_error_msg(ECustomizedError.CREATE_FOLDER_IF_NOT_EXIST), abort=True) + except Abort: + mhandler.SrvOutPutHandler.cancel_upload() + exit(1) + + # stop scaning and use the current folder as parent folder current_folder_node = folder_path - click.confirm(customized_error_msg(ECustomizedError.CREATE_FOLDER_IF_NOT_EXIST), abort=True) create_folder_flag = True break else: diff --git a/app/utils/aggregated.py b/app/utils/aggregated.py index 725cb753..a6d13cfe 100644 --- a/app/utils/aggregated.py +++ b/app/utils/aggregated.py @@ -60,7 +60,7 @@ def get_file_info_by_geid(geid: list): def fit_terminal_width(string_to_format): - string_to_format = string_to_format.split('...') + string_to_format = string_to_format.rsplit('...') current_len = 0 sentence = '' terminal_width = shutil.get_terminal_size().columns diff --git a/pyproject.toml b/pyproject.toml index d373d752..eab325d4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "app" -version = "2.8.4" +version = "2.8.5" description = "This service is designed to support pilot platform" authors = ["Indoc Systems"] From 6af654f62e281c85f672a4d311f5d04bc2a8fcdf Mon Sep 17 00:00:00 2001 From: zhiren Date: Thu, 23 Nov 2023 12:34:29 -0500 Subject: [PATCH 16/17] prepare the release branch --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index eab325d4..ce7a52f1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "app" -version = "2.8.5" +version = "2.9.0" description = "This service is designed to support pilot platform" authors = ["Indoc Systems"] From f377927a29005e01e0f9dcb7fb2ddd8d572f5216 Mon Sep 17 00:00:00 2001 From: zhiren Date: Thu, 23 Nov 2023 15:03:19 -0500 Subject: [PATCH 17/17] use correct version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index ce7a52f1..eab325d4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "app" -version = "2.9.0" +version = "2.8.5" description = "This service is designed to support pilot platform" authors = ["Indoc Systems"]