From 836d72ce41ab2ff87a6213d0b58b4ef29336ef4c Mon Sep 17 00:00:00 2001 From: zhiren Date: Thu, 21 Sep 2023 10:27:10 -0400 Subject: [PATCH 1/9] bumpup version for staging release --- app/resources/custom_help.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/resources/custom_help.py b/app/resources/custom_help.py index 7f6a27f5..1bdaf159 100644 --- a/app/resources/custom_help.py +++ b/app/resources/custom_help.py @@ -6,7 +6,7 @@ class HelpPage: page = { 'update': { - 'version': '2.7.0', + 'version': '2.7.1', '1': 'Add new feature for folder merging', '2': 'Secure the config file', '3': 'Optimize logic, input and error message', diff --git a/pyproject.toml b/pyproject.toml index 1ac9ae0c..16feba25 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "app" -version = "2.7.0" +version = "2.7.1" description = "This service is designed to support pilot platform" authors = ["Indoc Systems"] From 3b9fb5e9431b08e447139dbe91cb6cc2fd6890cd Mon Sep 17 00:00:00 2001 From: Color Zhan Date: Tue, 26 Sep 2023 13:39:47 -0400 Subject: [PATCH 2/9] Hotfix20230926: fixup the duplication check only apply for greenroom (#98) * hotfix the cli duplication check only apply for greenroom * bumpup version --------- Co-authored-by: zhiren --- app/services/file_manager/file_upload/upload_client.py | 8 +++++++- pyproject.toml | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/app/services/file_manager/file_upload/upload_client.py b/app/services/file_manager/file_upload/upload_client.py index cf3fb7a7..52ed7933 100644 --- a/app/services/file_manager/file_upload/upload_client.py +++ b/app/services/file_manager/file_upload/upload_client.py @@ -161,13 +161,19 @@ def check_upload_duplication(self, file_objects: List[FileObject]) -> Tuple[List """ headers = {'Authorization': 'Bearer ' + self.user.access_token, 'Session-ID': self.user.session_id} url = AppConfig.Connections.url_base + '/portal/v1/files/exists' + zone_int = 0 if self.zone == 'greenroom' else 1 # generate a list of locations for uploaded files to check duplication # at same time, generate a dict of mapping with object_path: FileObject locations = [x.object_path for x in file_objects] object_path_file_object_map = {x.object_path: x for x in file_objects} - payload = {'locations': locations, 'container_code': self.project_code, 'container_type': 'project', 'zone': 0} + payload = { + 'locations': locations, + 'container_code': self.project_code, + 'container_type': 'project', + 'zone': zone_int, + } response = resilient_session().post(url, json=payload, headers=headers) # pop the file object if the file has been uploaded diff --git a/pyproject.toml b/pyproject.toml index 16feba25..6234d349 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "app" -version = "2.7.1" +version = "2.7.2" description = "This service is designed to support pilot platform" authors = ["Indoc Systems"] From 2e694fa2ea76ed9825fd60345bde06aaab9a4d8b Mon Sep 17 00:00:00 2001 From: zhiren Date: Tue, 26 Sep 2023 14:19:49 -0400 Subject: [PATCH 3/9] bumpup version --- app/resources/custom_help.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/resources/custom_help.py b/app/resources/custom_help.py index 1bdaf159..3b6412ef 100644 --- a/app/resources/custom_help.py +++ b/app/resources/custom_help.py @@ -6,7 +6,7 @@ class HelpPage: page = { 'update': { - 'version': '2.7.1', + 'version': '2.7.2', '1': 'Add new feature for folder merging', '2': 'Secure the config file', '3': 'Optimize logic, input and error message', From db844751436803bab3a14d52abfa1ada97a2541e Mon Sep 17 00:00:00 2001 From: zhiren Date: Tue, 26 Sep 2023 16:52:59 -0400 Subject: [PATCH 4/9] update the error message when folder merging --- app/resources/custom_error.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/app/resources/custom_error.py b/app/resources/custom_error.py index e708703e..a8eff967 100644 --- a/app/resources/custom_error.py +++ b/app/resources/custom_error.py @@ -53,8 +53,11 @@ class Error: ), 'UPLOAD_CANCEL': 'Upload task was cancelled.', 'UPLOAD_FAIL': 'Upload task was failed. Please check the console output.', - 'UPLOAD_SKIP_DUPLICATION': 'Following files with the same ' - 'name already exist in the Project: \n%s.\nDo you want to skip uploading', + 'UPLOAD_SKIP_DUPLICATION': ( + '\nSome of the selected files cannot be uploaded.\n\n' + 'The following files already exist in the upload destination: \n%s\n' + 'Do you want to cancel the upload [N] or skip duplicates and continue uploading [y]?' + ), 'UPLOAD_ID_NOT_EXIST': ( 'The specified multipart upload does not exist. ' 'The upload ID may be invalid, or the upload may have been aborted or completed.' From 1be237f968dd0c81029e99dd16c025902a0b1825 Mon Sep 17 00:00:00 2001 From: zhiren Date: Tue, 26 Sep 2023 16:53:15 -0400 Subject: [PATCH 5/9] bumpup version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 6234d349..f2b2b44d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "app" -version = "2.7.2" +version = "2.7.3" description = "This service is designed to support pilot platform" authors = ["Indoc Systems"] From 0af981481681dca3d1c6d3ebb169083fa54f1b95 Mon Sep 17 00:00:00 2001 From: zhiren Date: Wed, 27 Sep 2023 09:12:56 -0400 Subject: [PATCH 6/9] update the warning message when all files are all duplicated --- app/services/file_manager/file_upload/file_upload.py | 1 + app/services/output_manager/message_handler.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/app/services/file_manager/file_upload/file_upload.py b/app/services/file_manager/file_upload/file_upload.py index aa8ae001..93224751 100644 --- a/app/services/file_manager/file_upload/file_upload.py +++ b/app/services/file_manager/file_upload/file_upload.py @@ -190,6 +190,7 @@ def simple_upload( # noqa: C901 if len(non_duplicate_file_objects) == 0: mhandler.SrvOutPutHandler.file_duplication_check_warning_with_all_same() + SrvErrorHandler.customized_handle(ECustomizedError.UPLOAD_CANCEL, if_exit=True) elif len(duplicated_file) > 0: mhandler.SrvOutPutHandler.file_duplication_check_success() duplicate_warning_format = '\n'.join(duplicated_file) diff --git a/app/services/output_manager/message_handler.py b/app/services/output_manager/message_handler.py index d5b179ac..a6aab665 100644 --- a/app/services/output_manager/message_handler.py +++ b/app/services/output_manager/message_handler.py @@ -154,7 +154,7 @@ def file_duplication_check_success(): @staticmethod def file_duplication_check_warning_with_all_same(): """e.g. file duplication check warning with all same.""" - return logger.warning('All files are the same, no need to upload.') + return logger.warning('\nAll files already exist in the upload destination.\n') @staticmethod def resume_check_success(): From 2b858c22ec904d56fd79eb92016ce7a1a4790c35 Mon Sep 17 00:00:00 2001 From: zhiren Date: Wed, 27 Sep 2023 09:39:22 -0400 Subject: [PATCH 7/9] fixup the test cases --- .../file_upload/test_file_upload.py | 49 +++++++++++++------ 1 file changed, 35 insertions(+), 14 deletions(-) diff --git a/tests/app/services/file_manager/file_upload/test_file_upload.py b/tests/app/services/file_manager/file_upload/test_file_upload.py index fcc1c726..c8eddaa4 100644 --- a/tests/app/services/file_manager/file_upload/test_file_upload.py +++ b/tests/app/services/file_manager/file_upload/test_file_upload.py @@ -109,7 +109,7 @@ def test_assemble_path_at_non_existing_folder(mocker): assert create_folder_flag is True -def test_file_upload_skip_empty_file(mocker, tmp_path): +def test_file_upload_skip_empty_file(mocker, tmp_path, capfd): file_name = 'test' upload_event = { 'file': file_name, @@ -120,8 +120,23 @@ def test_file_upload_skip_empty_file(mocker, tmp_path): mocker.patch('os.path.isdir', return_value=False) mocker.patch('app.services.file_manager.file_upload.models.FileObject.generate_meta', return_value=(0, 0)) - item_ids = simple_upload(upload_event, output_path=str(tmp_path / 'test')) - assert len(item_ids) == 0 + try: + simple_upload(upload_event, output_path=str(tmp_path / 'test')) + except SystemExit: + out, _ = capfd.readouterr() + + expect = ( + f'Starting upload of: {file_name}\n' + + 'Skip the file with 0 size: test\n' + + 'Checking for file duplication...\n' + + '\nAll files already exist in the upload destination.\n\n' + + customized_error_msg(ECustomizedError.UPLOAD_CANCEL) + + '\n' + ) + + assert out == expect + else: + AssertionError('SystemExit not raised') def test_dont_allow_tagging_when_folder_upload(mocker, capfd): @@ -242,17 +257,23 @@ def test_folder_merge_skip_with_all_duplication(mocker, mock_upload_client, capf return_value=([], dup_list), ) - item_ids = simple_upload(upload_event) - assert len(item_ids) == 0 - assert click_yes_mock.call_count == 0 - - out, _ = capfd.readouterr() - expect = ( - f'Starting upload of: {file_name}\n' - + 'Checking for file duplication...\n' - + 'All files are the same, no need to upload.\n' - ) - assert expect in out + try: + simple_upload(upload_event) + + except SystemExit: + assert click_yes_mock.call_count == 0 + + out, _ = capfd.readouterr() + expect = ( + f'Starting upload of: {file_name}\n' + + 'Checking for file duplication...\n' + + '\nAll files already exist in the upload destination.\n\n' + + customized_error_msg(ECustomizedError.UPLOAD_CANCEL) + + '\n' + ) + assert expect in out + else: + AssertionError('SystemExit not raised') def test_resume_upload(mocker): From ba7ed8e3772aa7de3f90087732a278f3f821359b Mon Sep 17 00:00:00 2001 From: zhiren Date: Wed, 27 Sep 2023 15:04:45 -0400 Subject: [PATCH 8/9] merge back with next preminor --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index f2b2b44d..a59ef366 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "app" -version = "2.7.3" +version = "2.8.0a0" description = "This service is designed to support pilot platform" authors = ["Indoc Systems"] From 7a05c068480fb54704c89732f8d00d66d4058ac3 Mon Sep 17 00:00:00 2001 From: zhiren Date: Wed, 27 Sep 2023 15:06:02 -0400 Subject: [PATCH 9/9] update version in user help page --- app/resources/custom_help.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/resources/custom_help.py b/app/resources/custom_help.py index 3b6412ef..a90ff648 100644 --- a/app/resources/custom_help.py +++ b/app/resources/custom_help.py @@ -6,7 +6,7 @@ class HelpPage: page = { 'update': { - 'version': '2.7.2', + 'version': '2.8.0a0', '1': 'Add new feature for folder merging', '2': 'Secure the config file', '3': 'Optimize logic, input and error message',