diff --git a/app/services/dataset_manager/dataset_download.py b/app/services/dataset_manager/dataset_download.py index 0618803b..de36cb1d 100644 --- a/app/services/dataset_manager/dataset_download.py +++ b/app/services/dataset_manager/dataset_download.py @@ -43,15 +43,14 @@ def pre_dataset_version_download(self): 'Session-ID': self.session_id, } payload = {'version': self.version} - try: - response = requests.get(url, headers=headers, params=payload) - res = response.json() - code = res.get('code') - if code == 404: - SrvErrorHandler.customized_handle(ECustomizedError.VERSION_NOT_EXIST, True, self.version) - else: - return res - except Exception: + response = requests.get(url, headers=headers, params=payload) + res = response.json() + code = response.status_code + if code == 200: + return res + elif code == 404: + SrvErrorHandler.customized_handle(ECustomizedError.VERSION_NOT_EXIST, True, self.version) + else: SrvErrorHandler.default_handle(response.content, True) @require_valid_token() diff --git a/app/services/file_manager/file_list.py b/app/services/file_manager/file_list.py index f2193b05..5abf963c 100644 --- a/app/services/file_manager/file_list.py +++ b/app/services/file_manager/file_list.py @@ -67,9 +67,10 @@ def list_files_without_pagination(self, paths, zone, page, page_size): def list_files_with_pagination(self, paths, zone, page, page_size): while True: files = self.list_files(paths, zone, page, page_size) - if len(files) < page_size and page == 0: - break - elif len(files) < page_size and page != 0: + file_list = files.split('...')[:-1] if files != '' else [] + if len(file_list) < page_size and page == 0: + choice = ['exit'] + elif len(file_list) < page_size and page != 0: choice = ['previous page', 'exit'] elif page == 0: choice = ['next page', 'exit'] diff --git a/app/services/file_manager/file_upload/file_upload.py b/app/services/file_manager/file_upload/file_upload.py index bbfc8682..ee694a90 100644 --- a/app/services/file_manager/file_upload/file_upload.py +++ b/app/services/file_manager/file_upload/file_upload.py @@ -117,7 +117,7 @@ def simple_upload( # noqa: C901 tags = upload_event.get('tags') zone = upload_event.get('zone') # process_pipeline = upload_event.get('process_pipeline', None) - # upload_message = upload_event.get('upload_message') + upload_message = upload_event.get('upload_message') current_folder_node = upload_event.get('current_folder_node', '') parent_folder_id = upload_event.get('parent_folder_id', '') create_folder_flag = upload_event.get('create_folder_flag', False) @@ -155,6 +155,7 @@ def simple_upload( # noqa: C901 regular_file=regular_file, tags=tags, source_id=source_id, + upload_message=upload_message, ) # format the local path into object storage path for preupload diff --git a/app/services/file_manager/file_upload/upload_client.py b/app/services/file_manager/file_upload/upload_client.py index e189dd89..57a93ab1 100644 --- a/app/services/file_manager/file_upload/upload_client.py +++ b/app/services/file_manager/file_upload/upload_client.py @@ -224,6 +224,7 @@ def output_manifest(self, file_objects: List[FileObject], output_path: str) -> D 'parent_folder_id': self.parent_folder_id, 'current_folder_node': self.current_folder_node, 'tags': self.tags, + 'upload_message': self.upload_message, 'file_objects': {file_object.item_id: file_object.to_dict() for file_object in file_objects}, } diff --git a/pyproject.toml b/pyproject.toml index e665545e..2901987d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "app" -version = "2.3.2" +version = "2.5.0" description = "This service is designed to support pilot platform" authors = ["Indoc Research"] diff --git a/tests/app/commands/test_dataset.py b/tests/app/commands/test_dataset.py new file mode 100644 index 00000000..bf043b9c --- /dev/null +++ b/tests/app/commands/test_dataset.py @@ -0,0 +1,51 @@ +# Copyright (C) 2022-2023 Indoc Research +# +# Contact Indoc Research for any questions regarding the use of this source code. + +from app.commands.dataset import dataset_download +from app.configs.app_config import AppConfig + + +def test_download_not_exited_dataset_version(requests_mock, mocker, cli_runner, capsys): + mocker.patch('app.services.user_authentication.token_manager.SrvTokenManager.check_valid', return_value=0) + requests_mock.get( + 'http://bff_cli' + '/v1/dataset/testdataset', + json={ + 'code': 200, + 'error_msg': '', + 'result': { + 'general_info': { + 'id': 'fake-id', + 'source': '', + 'authors': ['test-admin', 'test-user'], + 'code': 'testdataset', + 'type': 'GENERAL', + 'modality': [], + 'collection_method': [], + 'license': '', + 'tags': ['cdsa'], + 'description': 'Description example.', + 'size': 25, + 'total_files': 1, + 'title': 'test dataset', + 'creator': 'test-admin', + 'project_id': 'project-id', + 'created_at': '2022-02-03T19:49:35', + 'updated_at': '2022-03-18T18:08:33', + }, + 'version_detail': [], + 'version_no': 0, + }, + }, + ) + + requests_mock.get( + AppConfig.Connections.url_dataset + '/fake-id/download/pre', + json={'error': 'version does not exist'}, + status_code=404, + ) + + result = cli_runner.invoke(dataset_download, ['testdataset', '.', '-v', '1.0']) + outputs = result.output.split('\n') + assert outputs[0] == 'Current dataset version: 1.0' + assert outputs[1] == 'Version not available: 1.0' diff --git a/tests/app/commands/test_file.py b/tests/app/commands/test_file.py index a82fd154..f9939b79 100644 --- a/tests/app/commands/test_file.py +++ b/tests/app/commands/test_file.py @@ -3,12 +3,15 @@ # Contact Indoc Research for any questions regarding the use of this source code. import click +import questionary +from app.commands.file import file_list from app.commands.file import file_put from app.commands.file import file_resume from app.services.file_manager.file_upload.models import FileObject from app.services.output_manager.error_handler import ECustomizedError from app.services.output_manager.error_handler import customized_error_msg +from tests.conftest import decoded_token def test_file_upload_command_success_with_attribute(mocker, cli_runner): @@ -59,3 +62,43 @@ def test_resumable_upload_command_failed_with_file_not_exists(mocker, cli_runner result = cli_runner.invoke(file_resume, ['--resumable-manifest', 'test.json', '--thread', 1]) assert result.exit_code == 0 assert result.output == customized_error_msg(ECustomizedError.INVALID_RESUMABLE) + '\n' + + +def test_file_list_with_pagination(requests_mock, mocker, cli_runner): + mocker.patch( + 'app.services.user_authentication.token_manager.SrvTokenManager.decode_access_token', + return_value=decoded_token(), + ) + + mocker.patch('app.services.file_manager.file_list.search_item', return_value=None) + requests_mock.get( + 'http://bff_cli' + '/v1/testproject/files/query', + json={ + 'code': 200, + 'error_msg': '', + 'result': [{'type': 'file', 'name': 'file1'}, {'type': 'file', 'name': 'file2'}], + }, + ) + mocker.patch.object(questionary, 'select') + questionary.select.return_value.ask.return_value = 'exit' + result = cli_runner.invoke(file_list, ['testproject/admin', '-z', 'greenroom']) + outputs = result.output.split('\n') + assert outputs[0] == 'file1 file2 ' + + +def test_empty_file_list_with_pagination(requests_mock, mocker, cli_runner): + mocker.patch( + 'app.services.user_authentication.token_manager.SrvTokenManager.decode_access_token', + return_value=decoded_token(), + ) + + mocker.patch('app.services.file_manager.file_list.search_item', return_value=None) + requests_mock.get( + 'http://bff_cli' + '/v1/testproject/files/query', + json={'code': 200, 'error_msg': '', 'result': []}, + ) + mocker.patch.object(questionary, 'select') + questionary.select.return_value.ask.return_value = 'exit' + result = cli_runner.invoke(file_list, ['testproject/admin', '-z', 'greenroom']) + outputs = result.output.split('\n') + assert outputs[0] == ' ' diff --git a/tests/conftest.py b/tests/conftest.py index beeb3785..aaee5cc6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -20,6 +20,7 @@ def reset_singletons(): def mock_settings(monkeypatch): monkeypatch.setattr(AppConfig.Connections, 'url_authn', 'http://service_auth') monkeypatch.setattr(AppConfig.Connections, 'url_bff', 'http://bff_cli') + monkeypatch.setattr(AppConfig.Connections, 'url_dataset', 'http://url_dataset') monkeypatch.setattr(AppConfig.Connections, 'url_upload_greenroom', 'http://upload_gr') monkeypatch.setattr(AppConfig.Connections, 'url_upload_core', 'http://upload_core') monkeypatch.setattr(UserConfig, 'username', 'test-user')