From 79b80af840b2db4a73d8c101992bf1f0deaab8da Mon Sep 17 00:00:00 2001 From: zhiren Date: Wed, 28 Jun 2023 11:08:23 -0400 Subject: [PATCH 1/3] fixup the resumable upload will not attach the file attribute --- app/commands/file.py | 9 ++++++++- app/services/file_manager/file_upload/file_upload.py | 1 + app/services/file_manager/file_upload/upload_client.py | 3 +++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/app/commands/file.py b/app/commands/file.py index acd53a0a..c5c73177 100644 --- a/app/commands/file.py +++ b/app/commands/file.py @@ -245,9 +245,16 @@ def file_resume(**kwargs): # noqa: C901 # are rather similar with the input validate_upload_event(resumable_manifest) - # print(resumable_manifest) resume_upload(resumable_manifest, thread) + # since only file upload can attach manifest, take the first file object + srv_manifest = SrvFileManifests() + item_id = next(iter(resumable_manifest.get('file_objects'))) + attribute = resumable_manifest.get('attributes') + zone = resumable_manifest.get('zone') + srv_manifest.attach_manifest(attribute, item_id, zone) if attribute else None + message_handler.SrvOutPutHandler.all_file_uploaded() + def validate_upload_event(event): """validate upload request, raise error when filed.""" diff --git a/app/services/file_manager/file_upload/file_upload.py b/app/services/file_manager/file_upload/file_upload.py index ee694a90..1e8814c0 100644 --- a/app/services/file_manager/file_upload/file_upload.py +++ b/app/services/file_manager/file_upload/file_upload.py @@ -156,6 +156,7 @@ def simple_upload( # noqa: C901 tags=tags, source_id=source_id, upload_message=upload_message, + attributes=attribute, ) # format the local path into object storage path for preupload diff --git a/app/services/file_manager/file_upload/upload_client.py b/app/services/file_manager/file_upload/upload_client.py index 57a93ab1..f4c42b88 100644 --- a/app/services/file_manager/file_upload/upload_client.py +++ b/app/services/file_manager/file_upload/upload_client.py @@ -55,6 +55,7 @@ def __init__( regular_file: str = True, tags: list = None, source_id: str = '', + attributes: dict = None, ): self.user = UserConfig() self.operator = self.user.username @@ -80,6 +81,7 @@ def __init__( # tags and souce_id are only allowed in file uplaod self.tags = tags self.source_id = source_id + self.attributes = attributes # the flag to indicate if all upload process finished # then the token refresh loop will end @@ -226,6 +228,7 @@ def output_manifest(self, file_objects: List[FileObject], output_path: str) -> D 'tags': self.tags, 'upload_message': self.upload_message, 'file_objects': {file_object.item_id: file_object.to_dict() for file_object in file_objects}, + 'attributes': self.attributes if self.attributes else {}, } with open(output_path, 'w') as f: From bc0223f528feb2f31908690b07d67050cf40bfe0 Mon Sep 17 00:00:00 2001 From: zhiren Date: Wed, 28 Jun 2023 11:34:03 -0400 Subject: [PATCH 2/3] add test cases --- tests/app/commands/test_file.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/tests/app/commands/test_file.py b/tests/app/commands/test_file.py index f9939b79..e3aa4721 100644 --- a/tests/app/commands/test_file.py +++ b/tests/app/commands/test_file.py @@ -49,11 +49,34 @@ def test_resumable_upload_command_success(mocker, cli_runner): # mock the open function mocked_open_data = mocker.mock_open(read_data='test') mocker.patch('builtins.open', mocked_open_data) - mocker.patch('json.load', return_value={'resumable_manifest': 'test.json', 'thread': 1}) + mocker.patch('json.load', return_value={'file_objects': {'test_item_id': {'file_name': 'test.json'}}, 'zone': 1}) mocker.patch('app.commands.file.resume_upload', return_value=None) + result = cli_runner.invoke(file_resume, ['--resumable-manifest', 'test.json', '--thread', 1]) + assert result.exit_code == 0 + + +def test_resumable_upload_command_with_file_attribute_success(mocker, cli_runner): + mocker.patch('os.path.exists', return_value=True) + # mock the open function + mocked_open_data = mocker.mock_open(read_data='test') + mocker.patch('builtins.open', mocked_open_data) + mocker.patch( + 'json.load', + return_value={ + 'file_objects': {'test_item_id': {'file_name': 'test.json'}}, + 'zone': 1, + 'attributes': {'M1': {'attr1': '1'}}, + }, + ) + mocker.patch('app.commands.file.resume_upload', return_value=None) + + attribute_fun_mock = mocker.patch( + 'app.services.file_manager.file_manifests.SrvFileManifests.attach_manifest', return_value=None + ) result = cli_runner.invoke(file_resume, ['--resumable-manifest', 'test.json', '--thread', 1]) assert result.exit_code == 0 + attribute_fun_mock.assert_called_once() def test_resumable_upload_command_failed_with_file_not_exists(mocker, cli_runner): From 12c707011bd8ed2dcef1b9394f69bb69daa51f3d Mon Sep 17 00:00:00 2001 From: zhiren Date: Wed, 28 Jun 2023 15:41:33 -0400 Subject: [PATCH 3/3] bumpup version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 2e6eed7f..b223a8bc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "app" -version = "2.5.2a0" +version = "2.5.2" description = "This service is designed to support pilot platform" authors = ["Indoc Research"]