From 748e51d07fd295ec2c441ec8014b1407b769cc1b Mon Sep 17 00:00:00 2001 From: QXgu Date: Wed, 19 Apr 2023 22:30:14 -0400 Subject: [PATCH 01/13] PILOT-2678: Fix error message for dataset version download when version does not exist --- .../dataset_manager/dataset_download.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/app/services/dataset_manager/dataset_download.py b/app/services/dataset_manager/dataset_download.py index 0618803b..de36cb1d 100644 --- a/app/services/dataset_manager/dataset_download.py +++ b/app/services/dataset_manager/dataset_download.py @@ -43,15 +43,14 @@ def pre_dataset_version_download(self): 'Session-ID': self.session_id, } payload = {'version': self.version} - try: - response = requests.get(url, headers=headers, params=payload) - res = response.json() - code = res.get('code') - if code == 404: - SrvErrorHandler.customized_handle(ECustomizedError.VERSION_NOT_EXIST, True, self.version) - else: - return res - except Exception: + response = requests.get(url, headers=headers, params=payload) + res = response.json() + code = response.status_code + if code == 200: + return res + elif code == 404: + SrvErrorHandler.customized_handle(ECustomizedError.VERSION_NOT_EXIST, True, self.version) + else: SrvErrorHandler.default_handle(response.content, True) @require_valid_token() From e2087511fcd3ac172d0a1787f70db061f95b69f2 Mon Sep 17 00:00:00 2001 From: QXgu Date: Wed, 19 Apr 2023 23:11:56 -0400 Subject: [PATCH 02/13] PILOT-2676: Fix project file listing white page error --- app/services/file_manager/file_list.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/app/services/file_manager/file_list.py b/app/services/file_manager/file_list.py index f2193b05..5abf963c 100644 --- a/app/services/file_manager/file_list.py +++ b/app/services/file_manager/file_list.py @@ -67,9 +67,10 @@ def list_files_without_pagination(self, paths, zone, page, page_size): def list_files_with_pagination(self, paths, zone, page, page_size): while True: files = self.list_files(paths, zone, page, page_size) - if len(files) < page_size and page == 0: - break - elif len(files) < page_size and page != 0: + file_list = files.split('...')[:-1] if files != '' else [] + if len(file_list) < page_size and page == 0: + choice = ['exit'] + elif len(file_list) < page_size and page != 0: choice = ['previous page', 'exit'] elif page == 0: choice = ['next page', 'exit'] From fc6178b1b647cf67335d9730e26fe2f5ba72be03 Mon Sep 17 00:00:00 2001 From: QXgu Date: Thu, 20 Apr 2023 16:02:22 -0400 Subject: [PATCH 03/13] Add unit test --- tests/app/commands/test_file.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/app/commands/test_file.py b/tests/app/commands/test_file.py index a82fd154..8c96ba36 100644 --- a/tests/app/commands/test_file.py +++ b/tests/app/commands/test_file.py @@ -4,11 +4,13 @@ import click +from app.commands.file import file_list from app.commands.file import file_put from app.commands.file import file_resume from app.services.file_manager.file_upload.models import FileObject from app.services.output_manager.error_handler import ECustomizedError from app.services.output_manager.error_handler import customized_error_msg +from tests.conftest import decoded_token def test_file_upload_command_success_with_attribute(mocker, cli_runner): @@ -59,3 +61,15 @@ def test_resumable_upload_command_failed_with_file_not_exists(mocker, cli_runner result = cli_runner.invoke(file_resume, ['--resumable-manifest', 'test.json', '--thread', 1]) assert result.exit_code == 0 assert result.output == customized_error_msg(ECustomizedError.INVALID_RESUMABLE) + '\n' + + +def test_file_list_with_pagination(mocker, cli_runner): + mocker.patch( + 'app.services.user_authentication.token_manager.SrvTokenManager.decode_access_token', + return_value=decoded_token(), + ) + mock_list = mocker.patch( + 'app.services.file_manager.file_list.SrvFileList.list_files', return_value='...file1 ...file2...' + ) + cli_runner.invoke(file_list, ['testproject/admin', '-z', 'greenroom']) + mock_list.assert_called_once() From ccfe24305b722f241d57ca54ae901d461c7bf7fc Mon Sep 17 00:00:00 2001 From: QXgu Date: Thu, 20 Apr 2023 22:04:24 -0400 Subject: [PATCH 04/13] Add unit test --- tests/app/commands/test_dataset.py | 51 ++++++++++++++++++++++++++++++ tests/conftest.py | 1 + 2 files changed, 52 insertions(+) create mode 100644 tests/app/commands/test_dataset.py diff --git a/tests/app/commands/test_dataset.py b/tests/app/commands/test_dataset.py new file mode 100644 index 00000000..8554795a --- /dev/null +++ b/tests/app/commands/test_dataset.py @@ -0,0 +1,51 @@ +# Copyright (C) 2022-2023 Indoc Research +# +# Contact Indoc Research for any questions regarding the use of this source code. + +from app.commands.dataset import dataset_download +from app.configs.app_config import AppConfig + + +def test_file_upload_command_success_with_attribute(requests_mock, mocker, cli_runner, capsys): + mocker.patch('app.services.user_authentication.token_manager.SrvTokenManager.check_valid', return_value=0) + requests_mock.get( + 'http://bff_cli' + '/v1/dataset/testdataset', + json={ + 'code': 200, + 'error_msg': '', + 'result': { + 'general_info': { + 'id': 'fake-id', + 'source': '', + 'authors': ['test-admin', 'test-user'], + 'code': 'testdataset', + 'type': 'GENERAL', + 'modality': [], + 'collection_method': [], + 'license': '', + 'tags': ['cdsa'], + 'description': 'Description example.', + 'size': 25, + 'total_files': 1, + 'title': 'test dataset', + 'creator': 'test-admin', + 'project_id': 'project-id', + 'created_at': '2022-02-03T19:49:35', + 'updated_at': '2022-03-18T18:08:33', + }, + 'version_detail': [], + 'version_no': 0, + }, + }, + ) + + requests_mock.get( + AppConfig.Connections.url_dataset + '/fake-id/download/pre', + json={'error': 'version does not exist'}, + status_code=404, + ) + + result = cli_runner.invoke(dataset_download, ['testdataset', '.', '-v', '1.0']) + outputs = result.output.split('\n') + assert outputs[0] == 'Current dataset version: 1.0' + assert outputs[1] == 'Version not available: 1.0' diff --git a/tests/conftest.py b/tests/conftest.py index beeb3785..aaee5cc6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -20,6 +20,7 @@ def reset_singletons(): def mock_settings(monkeypatch): monkeypatch.setattr(AppConfig.Connections, 'url_authn', 'http://service_auth') monkeypatch.setattr(AppConfig.Connections, 'url_bff', 'http://bff_cli') + monkeypatch.setattr(AppConfig.Connections, 'url_dataset', 'http://url_dataset') monkeypatch.setattr(AppConfig.Connections, 'url_upload_greenroom', 'http://upload_gr') monkeypatch.setattr(AppConfig.Connections, 'url_upload_core', 'http://upload_core') monkeypatch.setattr(UserConfig, 'username', 'test-user') From 9ffc79321210bac2398760aaf045eba6953fe83e Mon Sep 17 00:00:00 2001 From: QXgu Date: Thu, 20 Apr 2023 22:05:29 -0400 Subject: [PATCH 05/13] Fix unit test name --- tests/app/commands/test_dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/app/commands/test_dataset.py b/tests/app/commands/test_dataset.py index 8554795a..bf043b9c 100644 --- a/tests/app/commands/test_dataset.py +++ b/tests/app/commands/test_dataset.py @@ -6,7 +6,7 @@ from app.configs.app_config import AppConfig -def test_file_upload_command_success_with_attribute(requests_mock, mocker, cli_runner, capsys): +def test_download_not_exited_dataset_version(requests_mock, mocker, cli_runner, capsys): mocker.patch('app.services.user_authentication.token_manager.SrvTokenManager.check_valid', return_value=0) requests_mock.get( 'http://bff_cli' + '/v1/dataset/testdataset', From 4687a643b38914aa80050d278dc48a48a311a086 Mon Sep 17 00:00:00 2001 From: QXgu Date: Fri, 21 Apr 2023 15:59:12 -0400 Subject: [PATCH 06/13] Update the unit tests --- tests/app/commands/test_file.py | 39 ++++++++++++++++++++++++++++----- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/tests/app/commands/test_file.py b/tests/app/commands/test_file.py index 8c96ba36..f9939b79 100644 --- a/tests/app/commands/test_file.py +++ b/tests/app/commands/test_file.py @@ -3,6 +3,7 @@ # Contact Indoc Research for any questions regarding the use of this source code. import click +import questionary from app.commands.file import file_list from app.commands.file import file_put @@ -63,13 +64,41 @@ def test_resumable_upload_command_failed_with_file_not_exists(mocker, cli_runner assert result.output == customized_error_msg(ECustomizedError.INVALID_RESUMABLE) + '\n' -def test_file_list_with_pagination(mocker, cli_runner): +def test_file_list_with_pagination(requests_mock, mocker, cli_runner): mocker.patch( 'app.services.user_authentication.token_manager.SrvTokenManager.decode_access_token', return_value=decoded_token(), ) - mock_list = mocker.patch( - 'app.services.file_manager.file_list.SrvFileList.list_files', return_value='...file1 ...file2...' + + mocker.patch('app.services.file_manager.file_list.search_item', return_value=None) + requests_mock.get( + 'http://bff_cli' + '/v1/testproject/files/query', + json={ + 'code': 200, + 'error_msg': '', + 'result': [{'type': 'file', 'name': 'file1'}, {'type': 'file', 'name': 'file2'}], + }, + ) + mocker.patch.object(questionary, 'select') + questionary.select.return_value.ask.return_value = 'exit' + result = cli_runner.invoke(file_list, ['testproject/admin', '-z', 'greenroom']) + outputs = result.output.split('\n') + assert outputs[0] == 'file1 file2 ' + + +def test_empty_file_list_with_pagination(requests_mock, mocker, cli_runner): + mocker.patch( + 'app.services.user_authentication.token_manager.SrvTokenManager.decode_access_token', + return_value=decoded_token(), + ) + + mocker.patch('app.services.file_manager.file_list.search_item', return_value=None) + requests_mock.get( + 'http://bff_cli' + '/v1/testproject/files/query', + json={'code': 200, 'error_msg': '', 'result': []}, ) - cli_runner.invoke(file_list, ['testproject/admin', '-z', 'greenroom']) - mock_list.assert_called_once() + mocker.patch.object(questionary, 'select') + questionary.select.return_value.ask.return_value = 'exit' + result = cli_runner.invoke(file_list, ['testproject/admin', '-z', 'greenroom']) + outputs = result.output.split('\n') + assert outputs[0] == ' ' From a62a8ffb196e674da5ebc724338b076581e49b43 Mon Sep 17 00:00:00 2001 From: QXgu Date: Mon, 24 Apr 2023 09:58:08 -0400 Subject: [PATCH 07/13] Update poetry version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 99a24127..4a942f44 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "app" -version = "2.2.3" +version = "2.2.4" description = "This service is designed to support pilot platform" authors = ["Indoc Research"] From e84a3647154e355c4b38a13106d4edcafa787b38 Mon Sep 17 00:00:00 2001 From: zhiren Date: Mon, 24 Apr 2023 10:37:44 -0400 Subject: [PATCH 08/13] bumpup to next preminor --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index e665545e..773ed5a9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "app" -version = "2.3.2" +version = "2.4.0a0" description = "This service is designed to support pilot platform" authors = ["Indoc Research"] From a3f76a5745df0a0b4f89dea40686ce6c217c4258 Mon Sep 17 00:00:00 2001 From: QXgu Date: Thu, 4 May 2023 10:23:29 -0400 Subject: [PATCH 09/13] Update the version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 99a24127..4a942f44 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "app" -version = "2.2.3" +version = "2.2.4" description = "This service is designed to support pilot platform" authors = ["Indoc Research"] From 097e7915bca2303ac1215b1bfe01e02fd750ee5c Mon Sep 17 00:00:00 2001 From: QXgu Date: Thu, 4 May 2023 10:29:14 -0400 Subject: [PATCH 10/13] Update the version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 4a942f44..9f1aaf98 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "app" -version = "2.2.4" +version = "2.4.0" description = "This service is designed to support pilot platform" authors = ["Indoc Research"] From 196c474af1256b4877d598adb996b7b92363c02c Mon Sep 17 00:00:00 2001 From: Color Zhan Date: Mon, 8 May 2023 09:30:12 -0400 Subject: [PATCH 11/13] fixup the core zone cannot resume the upload (#52) Co-authored-by: zhiren --- app/services/file_manager/file_upload/file_upload.py | 3 ++- app/services/file_manager/file_upload/upload_client.py | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/app/services/file_manager/file_upload/file_upload.py b/app/services/file_manager/file_upload/file_upload.py index bbfc8682..ee694a90 100644 --- a/app/services/file_manager/file_upload/file_upload.py +++ b/app/services/file_manager/file_upload/file_upload.py @@ -117,7 +117,7 @@ def simple_upload( # noqa: C901 tags = upload_event.get('tags') zone = upload_event.get('zone') # process_pipeline = upload_event.get('process_pipeline', None) - # upload_message = upload_event.get('upload_message') + upload_message = upload_event.get('upload_message') current_folder_node = upload_event.get('current_folder_node', '') parent_folder_id = upload_event.get('parent_folder_id', '') create_folder_flag = upload_event.get('create_folder_flag', False) @@ -155,6 +155,7 @@ def simple_upload( # noqa: C901 regular_file=regular_file, tags=tags, source_id=source_id, + upload_message=upload_message, ) # format the local path into object storage path for preupload diff --git a/app/services/file_manager/file_upload/upload_client.py b/app/services/file_manager/file_upload/upload_client.py index e189dd89..57a93ab1 100644 --- a/app/services/file_manager/file_upload/upload_client.py +++ b/app/services/file_manager/file_upload/upload_client.py @@ -224,6 +224,7 @@ def output_manifest(self, file_objects: List[FileObject], output_path: str) -> D 'parent_folder_id': self.parent_folder_id, 'current_folder_node': self.current_folder_node, 'tags': self.tags, + 'upload_message': self.upload_message, 'file_objects': {file_object.item_id: file_object.to_dict() for file_object in file_objects}, } From d801e8d11affd8e090e4ad142b4afbf6168441f8 Mon Sep 17 00:00:00 2001 From: zhiren Date: Mon, 15 May 2023 10:25:25 -0400 Subject: [PATCH 12/13] bumpup version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 9f1aaf98..59763c07 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "app" -version = "2.4.0" +version = "2.5.0a0" description = "This service is designed to support pilot platform" authors = ["Indoc Research"] From 132ec41b06f1736b840cec3d8820d76d8731d269 Mon Sep 17 00:00:00 2001 From: zhiren Date: Mon, 15 May 2023 13:06:22 -0400 Subject: [PATCH 13/13] bumpup version to 2.5.0 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 59763c07..2901987d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "app" -version = "2.5.0a0" +version = "2.5.0" description = "This service is designed to support pilot platform" authors = ["Indoc Research"]