From 0d4cb0482aaac141928db44eb3302e68ea0765c5 Mon Sep 17 00:00:00 2001 From: zhiren Date: Thu, 13 Apr 2023 10:37:00 -0400 Subject: [PATCH 1/3] add back the time interval --- app/configs/app_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/configs/app_config.py b/app/configs/app_config.py index 12b2bf45..3d2ba3e8 100644 --- a/app/configs/app_config.py +++ b/app/configs/app_config.py @@ -13,7 +13,7 @@ class Env(object): msg_path = ConfigClass.custom_path user_config_file = f'{user_config_path}/config.ini' token_warn_need_refresh = 250 # refresh token if token is about to expire - token_refresh_interval = 10 # auto refresh token every 2 minutes + token_refresh_interval = 120 # auto refresh token every 2 minutes # NOTE: there is a limitation on minio that # the multipart number is 10000. so we set From db3df9a741ab1bfda1a0d4eb4cc01a8d11f8ff17 Mon Sep 17 00:00:00 2001 From: zhiren Date: Thu, 13 Apr 2023 15:41:38 -0400 Subject: [PATCH 2/3] add the constraint in file upload to block file size with 0 --- .../file_manager/file_upload/file_upload.py | 16 ++++++++++++---- .../file_manager/file_upload/upload_client.py | 4 +--- test | 0 3 files changed, 13 insertions(+), 7 deletions(-) delete mode 100644 test diff --git a/app/services/file_manager/file_upload/file_upload.py b/app/services/file_manager/file_upload/file_upload.py index f0f8b26f..86815d06 100644 --- a/app/services/file_manager/file_upload/file_upload.py +++ b/app/services/file_manager/file_upload/file_upload.py @@ -165,7 +165,14 @@ def simple_upload( # noqa: C901 # first remove the input path from the file path file_path_sub = file.replace(input_path + '/', '') object_path = os.path.join(target_folder, file_path_sub) - file_objects.append(FileObject(object_path, file)) + + # generate a placeholder for each file + file_object = FileObject(object_path, file) + # skip the file with 0 size + if file_object.total_size == 0: + logger.warning(f'Skip the file with 0 size: {file_object.file_name}') + else: + file_objects.append(FileObject(object_path, file)) # here add the batch of 500 per loop, the pre upload api cannot # process very large amount of file at same time. otherwise it will timeout @@ -190,10 +197,11 @@ def simple_upload( # noqa: C901 pool = ThreadPool(num_of_thread + 1) pool.apply_async(upload_client.upload_token_refresh) on_success_res = [] + + file_object: FileObject for file_object in pre_upload_infos: chunk_res = upload_client.stream_upload(file_object, pool) - # NOTE: if there is some racing error make the combine chunks - # out of thread pool. + # the on_success api will be called after all chunk uploaded res = pool.apply_async( upload_client.on_succeed, args=(file_object, tags, chunk_res), @@ -219,7 +227,7 @@ def simple_upload( # noqa: C901 upload_client.create_file_lineage(source_file) os.remove(file_batchs[0]) if os.path.isdir(input_path) and job_type == UploadType.AS_FILE else None - num_of_file = len(upload_file_path) + num_of_file = len(pre_upload_infos) logger.info(f'Upload Time: {time.time() - upload_start_time:.2f}s for {num_of_file:d} files') return [file_object.item_id for file_object in pre_upload_infos] diff --git a/app/services/file_manager/file_upload/upload_client.py b/app/services/file_manager/file_upload/upload_client.py index cbeaf7f1..8dac4b2f 100644 --- a/app/services/file_manager/file_upload/upload_client.py +++ b/app/services/file_manager/file_upload/upload_client.py @@ -364,9 +364,7 @@ def on_succeed(self, file_object: FileObject, tags: List[str], chunk_result: Lis """ # check if all the chunks have been uploaded - for res in chunk_result: - while res.get() is None: - time.sleep(1) + [res.wait() for res in chunk_result] for i in range(AppConfig.Env.resilient_retry): url = self.base_url + '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/v1/files' diff --git a/test b/test deleted file mode 100644 index e69de29b..00000000 From b18a24986fc537139c3847b1df11638dffb5415d Mon Sep 17 00:00:00 2001 From: zhiren Date: Thu, 13 Apr 2023 15:47:00 -0400 Subject: [PATCH 3/3] add the test to skip the empty file --- .gitignore | 1 + .../file_manager/file_upload/test_file_upload.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+) diff --git a/.gitignore b/.gitignore index 7bf999b0..e33d6c8d 100644 --- a/.gitignore +++ b/.gitignore @@ -155,3 +155,4 @@ integration_tests # cli manifest data ./manifest.json manifest.json +test diff --git a/tests/app/services/file_manager/file_upload/test_file_upload.py b/tests/app/services/file_manager/file_upload/test_file_upload.py index 5044f9c4..efb2a3e0 100644 --- a/tests/app/services/file_manager/file_upload/test_file_upload.py +++ b/tests/app/services/file_manager/file_upload/test_file_upload.py @@ -109,6 +109,21 @@ def test_assemble_path_at_non_existing_folder(mocker): assert create_folder_flag is True +def test_file_upload_skip_empty_file(mocker): + file_name = 'test' + upload_event = { + 'file': file_name, + 'project_code': 'test_project', + 'zone': 'greenroom', + } + + mocker.patch('os.path.isdir', return_value=False) + mocker.patch('app.services.file_manager.file_upload.models.FileObject.generate_meta', return_value=(0, 0)) + + item_ids = simple_upload(upload_event) + assert len(item_ids) == 0 + + def test_dont_allow_tagging_when_folder_upload(mocker, capfd): file_name = 'test' upload_event = {