diff --git a/.gitignore b/.gitignore index e33d6c8d..fbc35fb3 100644 --- a/.gitignore +++ b/.gitignore @@ -155,4 +155,5 @@ integration_tests # cli manifest data ./manifest.json manifest.json +resumable_upload_log.json test diff --git a/app/commands/file.py b/app/commands/file.py index 9dae8f01..2e821aa8 100644 --- a/app/commands/file.py +++ b/app/commands/file.py @@ -30,6 +30,7 @@ from app.utils.aggregated import get_file_info_by_geid from app.utils.aggregated import get_zone from app.utils.aggregated import identify_target_folder +from app.utils.aggregated import remove_the_output_file from app.utils.aggregated import search_item @@ -110,9 +111,9 @@ def cli(): @click.option( '--output-path', '-o', - default='./manifest.json', + default='./resumable_upload_log.json', required=False, - help='The output path for the manifest file of resumable upload.', + help='The output path for the manifest file of resumable upload log.', show_default=True, ) @doc(file_help.file_help_page(file_help.FileHELP.FILE_UPLOAD)) @@ -226,6 +227,8 @@ def file_put(**kwargs): # noqa: C901 srv_manifest.attach_manifest(attribute, item_ids[0], zone) if attribute else None message_handler.SrvOutPutHandler.all_file_uploaded() + remove_the_output_file(output_path) + @click.command(name='resume') @click.option( @@ -241,7 +244,7 @@ def file_put(**kwargs): # noqa: C901 '-r', default=None, required=True, - help='The manifest file for resumable upload', + help='The resumable upload log file', show_default=True, ) @doc(file_help.file_help_page(file_help.FileHELP.FILE_RESUME)) @@ -278,6 +281,8 @@ def file_resume(**kwargs): # noqa: C901 srv_manifest.attach_manifest(attribute, item_id, zone) if attribute else None message_handler.SrvOutPutHandler.all_file_uploaded() + remove_the_output_file(resumable_manifest_file) + def validate_upload_event(event): """validate upload request, raise error when filed.""" diff --git a/app/resources/custom_error.py b/app/resources/custom_error.py index ec566355..8f862a6b 100644 --- a/app/resources/custom_error.py +++ b/app/resources/custom_error.py @@ -63,7 +63,7 @@ class Error: 'The upload ID may be invalid, or the upload may have been aborted or completed.' ), 'MANIFEST_OF_FOLDER_FILE_EXIST': ( - 'The manifest file of folder %s already exist. ' 'Do you want to overwrite the existing manifest file?' + 'The manifest file of folder %s already exist. ' 'To continue and overwrite the resumable upload log, enter' ), 'INVALID_CHUNK_UPLOAD': ( '\nThe chunk number %d is not the same with previous etag.\n' diff --git a/app/resources/custom_help.py b/app/resources/custom_help.py index 47ed6f6c..ba86ddd9 100644 --- a/app/resources/custom_help.py +++ b/app/resources/custom_help.py @@ -36,7 +36,7 @@ class HelpPage: 'FILE_LIST': 'List files and folders inside a given Project/folder.', 'FILE_SYNC': 'Download files/folders from a given Project/folder/file in core zone.', 'FILE_UPLOAD': 'Upload files/folders to a given Project path.', - 'FILE_RESUME': 'Resume the upload process with given manifest file.', + 'FILE_RESUME': 'Resume the upload process with a resumable upload log.', 'FILE_Z': 'Target Zone (i.e., core/greenroom).', 'FILE_ATTRIBUTE_P': 'Project Code', 'FILE_ATTRIBUTE_N': 'Attribute Template Name', diff --git a/app/utils/aggregated.py b/app/utils/aggregated.py index f629c090..965f682b 100644 --- a/app/utils/aggregated.py +++ b/app/utils/aggregated.py @@ -12,6 +12,7 @@ import httpx import requests +import app.services.logger_services.log_functions as logger from app.configs.app_config import AppConfig from app.configs.config import ConfigClass from app.configs.user_config import UserConfig @@ -154,3 +155,13 @@ def batch_generator(iterable: List[Any], batch_size=1): max_size = len(iterable) for start_index in range(0, max_size, batch_size): yield iterable[start_index : min(start_index + batch_size, max_size)] + + +def remove_the_output_file(filepath: str) -> None: + """Remove the output file after each successful operation to avoid confusion.""" + try: + os.remove(filepath) + except FileNotFoundError: + pass + except OSError: + logger.warning(f'Unable to remove "{filepath}".') diff --git a/pyproject.toml b/pyproject.toml index ec5fba78..606b776e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "app" -version = "2.9.6" +version = "2.9.7" description = "This service is designed to support pilot platform" authors = ["Indoc Systems"] diff --git a/tests/app/commands/test_file.py b/tests/app/commands/test_file.py index b3f1df9f..3b6adb0e 100644 --- a/tests/app/commands/test_file.py +++ b/tests/app/commands/test_file.py @@ -62,6 +62,7 @@ def test_resumable_upload_command_success(mocker, cli_runner): mocker.patch('builtins.open', mocked_open_data) mocker.patch('json.load', return_value={'file_objects': {'test_item_id': {'file_name': 'test.json'}}, 'zone': 1}) mocker.patch('app.commands.file.resume_upload', return_value=None) + mocker.patch('os.remove', return_value=None) result = cli_runner.invoke(file_resume, ['--resumable-manifest', 'test.json', '--thread', 1]) assert result.exit_code == 0 @@ -85,6 +86,8 @@ def test_resumable_upload_command_with_file_attribute_success(mocker, cli_runner 'app.services.file_manager.file_manifests.SrvFileManifests.attach_manifest', return_value=None ) + mocker.patch('os.remove', return_value=None) + result = cli_runner.invoke(file_resume, ['--resumable-manifest', 'test.json', '--thread', 1]) assert result.exit_code == 0 attribute_fun_mock.assert_called_once() @@ -253,13 +256,13 @@ def test_download_file_metadata_file_duplicate_success(mocker, cli_runner): with runner.isolated_filesystem(): file_meta_client = FileMetaClient('zone', file_path, metadata_loc, metadata_loc, metadata_loc) # create all file to make duplicationn - makedirs(dirname(file_meta_client.general_location), exist_ok=True) + makedirs(dirname(file_meta_client.general_location), exist_ok=True, mode=0o0700) with open(file_meta_client.general_location, 'w') as f: f.write(file_meta_client.general_location) - makedirs(dirname(file_meta_client.attribute_location), exist_ok=True) + makedirs(dirname(file_meta_client.attribute_location), exist_ok=True, mode=0o0700) with open(file_meta_client.attribute_location, 'w') as f: f.write(file_meta_client.attribute_location) - makedirs(dirname(file_meta_client.tag_location), exist_ok=True) + makedirs(dirname(file_meta_client.tag_location), exist_ok=True, mode=0o0700) with open(file_meta_client.tag_location, 'w') as f: f.write(file_meta_client.tag_location) @@ -303,13 +306,13 @@ def test_download_file_metadata_file_duplicate_abort(mocker, cli_runner): with runner.isolated_filesystem(): file_meta_client = FileMetaClient('zone', file_path, metadata_loc, metadata_loc, metadata_loc) # create all file to make duplicationn - makedirs(dirname(file_meta_client.general_location), exist_ok=True) + makedirs(dirname(file_meta_client.general_location), exist_ok=True, mode=0o0700) with open(file_meta_client.general_location, 'w') as f: f.write(file_meta_client.general_location) - makedirs(dirname(file_meta_client.attribute_location), exist_ok=True) + makedirs(dirname(file_meta_client.attribute_location), exist_ok=True, mode=0o0700) with open(file_meta_client.attribute_location, 'w') as f: f.write(file_meta_client.attribute_location) - makedirs(dirname(file_meta_client.tag_location), exist_ok=True) + makedirs(dirname(file_meta_client.tag_location), exist_ok=True, mode=0o0700) with open(file_meta_client.tag_location, 'w') as f: f.write(file_meta_client.tag_location)