From 29b19af31c982bbdcd52de6097160bd464e3faac Mon Sep 17 00:00:00 2001 From: zhiren Date: Tue, 18 Apr 2023 12:00:41 -0400 Subject: [PATCH 1/8] temporary disable the lineage in this stating release --- app/commands/file.py | 5 +++++ app/resources/custom_error.py | 1 + app/resources/custom_help.py | 8 +++++--- app/services/output_manager/error_handler.py | 1 + pyproject.toml | 2 +- 5 files changed, 13 insertions(+), 4 deletions(-) diff --git a/app/commands/file.py b/app/commands/file.py index ee007762..db943fef 100644 --- a/app/commands/file.py +++ b/app/commands/file.py @@ -120,6 +120,11 @@ def file_put(**kwargs): # noqa: C901 thread = kwargs.get('thread') output_path = kwargs.get('output_path') + # for 20230418 staging temporary disable the attribute + # since the backend is not ready yet + if source_file: + SrvErrorHandler.customized_handle(ECustomizedError.LINEAGE_FEATURE_NOT_READY, True) + user = UserConfig() # Check zone and upload-message zone = get_zone(zone) if zone else AppConfig.Env.green_zone.lower() diff --git a/app/resources/custom_error.py b/app/resources/custom_error.py index ed47b7be..6f44bd34 100644 --- a/app/resources/custom_error.py +++ b/app/resources/custom_error.py @@ -122,4 +122,5 @@ class Error: 'CONTAINER_REGISTRY_NO_URL': ( 'Container registry has not yet been configured. Related commands cannot be used at this time.' ), + 'LINEAGE_FEATURE_NOT_READY': 'Lineage is not support at v2.3.0', } diff --git a/app/resources/custom_help.py b/app/resources/custom_help.py index 938f5844..f3dd9f2c 100644 --- a/app/resources/custom_help.py +++ b/app/resources/custom_help.py @@ -6,9 +6,11 @@ class HelpPage: page = { 'update': { - 'version': '2.2.0', - '1': 'CLI supports to perform multi-threading upload for file/folders', - '2': 'CLI supports to perform resumable upload for single file', + 'version': '2.3.0', + '1': 'The logic of normal upload and resumble are splited. ' + 'add new command for resumable upload as `pilotcli file resume -r manifest.json`', + '2': 'The manifest file will be output for both file/folder upload', + '3': 'Optimize logic, input and error message', }, 'dataset': { 'DATASET_DOWNLOAD': 'Download a dataset or a particular version of a dataset.', diff --git a/app/services/output_manager/error_handler.py b/app/services/output_manager/error_handler.py index 4f9ea3f4..a0344839 100644 --- a/app/services/output_manager/error_handler.py +++ b/app/services/output_manager/error_handler.py @@ -81,6 +81,7 @@ class ECustomizedError(enum.Enum): CONTAINER_REGISTRY_NO_URL = 'CONTAINER_REGISTRY_NO_URL' CONFIG_NOT_FOUND = 'CONFIG_NOT_FOUND' CONFIG_EXIST = 'CONFIG_EXIST' + LINEAGE_FEATURE_NOT_READY = 'LINEAGE_FEATURE_NOT_READY' def customized_error_msg(customized_error: ECustomizedError): diff --git a/pyproject.toml b/pyproject.toml index 99a24127..18304491 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "app" -version = "2.2.3" +version = "2.3.0" description = "This service is designed to support pilot platform" authors = ["Indoc Research"] From 71f5b3e7224c2d2fbc10982b6fc099168b3aaeef Mon Sep 17 00:00:00 2001 From: zhiren Date: Tue, 18 Apr 2023 16:04:18 -0400 Subject: [PATCH 2/8] update the resumable upload as per batch operation --- .../file_manager/file_upload/file_upload.py | 48 ++++++++++++------- .../file_manager/file_upload/upload_client.py | 2 - .../output_manager/message_handler.py | 5 ++ 3 files changed, 37 insertions(+), 18 deletions(-) diff --git a/app/services/file_manager/file_upload/file_upload.py b/app/services/file_manager/file_upload/file_upload.py index f2a4cdd0..3b16b89c 100644 --- a/app/services/file_manager/file_upload/file_upload.py +++ b/app/services/file_manager/file_upload/file_upload.py @@ -256,28 +256,44 @@ def resume_upload( ) # check files in manifest if some of them are already uploaded - item_ids = [] + unfinished_items = [] all_files = manifest_json.get('file_objects') + item_ids = [] for item_id in all_files: item_ids.append(item_id) - items = get_file_info_by_geid(item_ids) - unfinished_items = [] - for x in items: - if x.get('result').get('status') == ItemStatus.REGISTERED: - file_info = all_files.get(x.get('result').get('id')) - unfinished_items.append( - FileObject( - file_info.get('object_path'), - file_info.get('local_path'), - file_info.get('resumable_id'), - file_info.get('job_id'), - file_info.get('item_id'), + # here add the batch of 500 per loop, the pre upload api cannot + # process very large amount of file at same time. otherwise it will timeout + num_of_batchs = math.ceil(len(all_files) / AppConfig.Env.upload_batch_size) + # here is list of pre upload result. We decided to call pre upload api by batch + for batch in range(0, num_of_batchs): + start_index = batch * AppConfig.Env.upload_batch_size + end_index = (batch + 1) * AppConfig.Env.upload_batch_size + file_batchs = item_ids[start_index:end_index] + items = get_file_info_by_geid(file_batchs) + + # get the detail of item to see if the file is already uploaded + unfinished_files = [] + for x in items: + if x.get('result').get('status') == ItemStatus.REGISTERED: + file_info = all_files.get(x.get('result').get('id')) + unfinished_files.append( + FileObject( + file_info.get('object_path'), + file_info.get('local_path'), + file_info.get('resumable_id'), + file_info.get('job_id'), + file_info.get('item_id'), + ) ) - ) - # then for the rest of the files, check if any chunks are already uploaded - unfinished_items = upload_client.resume_upload(unfinished_items) + # then for the rest of the files, check if any chunks are already uploaded + mhandler.SrvOutPutHandler.resume_check_in_progress() + if len(unfinished_files) > 0: + unfinished_items.extend(upload_client.resume_upload(unfinished_files)) + + mhandler.SrvOutPutHandler.resume_warning(len(unfinished_items)) + mhandler.SrvOutPutHandler.resume_check_success() # lastly, start resumable upload for the rest of the chunks # thread number +1 reserve one thread to refresh token diff --git a/app/services/file_manager/file_upload/upload_client.py b/app/services/file_manager/file_upload/upload_client.py index c77a9c36..e189dd89 100644 --- a/app/services/file_manager/file_upload/upload_client.py +++ b/app/services/file_manager/file_upload/upload_client.py @@ -114,7 +114,6 @@ def resume_upload(self, unfinished_file_objects: List[FileObject]) -> List[FileO - local_path(str): the local path of file. - chunk_info(dict): the mapping for chunks that already been uploaded. """ - mhandler.SrvOutPutHandler.resume_warning(len(unfinished_file_objects)) headers = {'Authorization': 'Bearer ' + self.user.access_token, 'Session-ID': self.user.session_id} url = AppConfig.Connections.url_bff + f'/v1/project/{self.project_code}/files/resumable' @@ -142,7 +141,6 @@ def resume_upload(self, unfinished_file_objects: List[FileObject]) -> List[FileO file_obj = rid_file_object_map.get(uploaded_info.get('resumable_id')) # update the chunk info file_obj.uploaded_chunks = uploaded_info.get('chunks_info') - mhandler.SrvOutPutHandler.resume_check_success() return unfinished_file_objects diff --git a/app/services/output_manager/message_handler.py b/app/services/output_manager/message_handler.py index 65217687..2ef1dd89 100644 --- a/app/services/output_manager/message_handler.py +++ b/app/services/output_manager/message_handler.py @@ -133,6 +133,11 @@ def resume_check_success(): """e.g. notify the resumable check succeed.""" return logger.info('Resumable upload check complete.') + @staticmethod + def resume_check_in_progress(): + """e.g. notify the resumable check succeed.""" + return logger.info('Resumable upload check in progress.') + @staticmethod def resume_warning(num_of_files: int): """e.g. notify the user if they comfirm the resumable upload.""" From bdb291c43960ecd6c307ad13934d686499daf812 Mon Sep 17 00:00:00 2001 From: zhiren Date: Wed, 19 Apr 2023 16:33:02 -0400 Subject: [PATCH 3/8] hotfix the folder upload with only one level will lose the folder structure --- app/services/file_manager/file_upload/file_upload.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/services/file_manager/file_upload/file_upload.py b/app/services/file_manager/file_upload/file_upload.py index 3b16b89c..b6e4374c 100644 --- a/app/services/file_manager/file_upload/file_upload.py +++ b/app/services/file_manager/file_upload/file_upload.py @@ -163,7 +163,7 @@ def simple_upload( # noqa: C901 input_path = os.path.dirname(input_path) for file in upload_file_path: # first remove the input path from the file path - file_path_sub = file.replace(input_path + '/', '') + file_path_sub = file.replace(input_path + '/', '') if input_path else file object_path = os.path.join(target_folder, file_path_sub) # generate a placeholder for each file @@ -174,6 +174,8 @@ def simple_upload( # noqa: C901 else: file_objects.append(FileObject(object_path, file)) + raise Exception('test') + # here add the batch of 500 per loop, the pre upload api cannot # process very large amount of file at same time. otherwise it will timeout num_of_batchs = math.ceil(len(file_objects) / AppConfig.Env.upload_batch_size) From 54d591d0095430320e42644bbba466f906414d59 Mon Sep 17 00:00:00 2001 From: zhiren Date: Wed, 19 Apr 2023 16:33:55 -0400 Subject: [PATCH 4/8] hotfix the folder upload with only one level will lose the folder structure --- app/services/file_manager/file_upload/file_upload.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/app/services/file_manager/file_upload/file_upload.py b/app/services/file_manager/file_upload/file_upload.py index b6e4374c..e7e08a13 100644 --- a/app/services/file_manager/file_upload/file_upload.py +++ b/app/services/file_manager/file_upload/file_upload.py @@ -174,8 +174,6 @@ def simple_upload( # noqa: C901 else: file_objects.append(FileObject(object_path, file)) - raise Exception('test') - # here add the batch of 500 per loop, the pre upload api cannot # process very large amount of file at same time. otherwise it will timeout num_of_batchs = math.ceil(len(file_objects) / AppConfig.Env.upload_batch_size) From ce213c39c65e3b3c3ee5ae3dcb7b491aebd05264 Mon Sep 17 00:00:00 2001 From: zhiren Date: Thu, 20 Apr 2023 10:24:01 -0400 Subject: [PATCH 5/8] remove the block for lineage operation --- app/commands/file.py | 5 ----- app/resources/custom_error.py | 1 - app/services/output_manager/error_handler.py | 1 - 3 files changed, 7 deletions(-) diff --git a/app/commands/file.py b/app/commands/file.py index db943fef..ee007762 100644 --- a/app/commands/file.py +++ b/app/commands/file.py @@ -120,11 +120,6 @@ def file_put(**kwargs): # noqa: C901 thread = kwargs.get('thread') output_path = kwargs.get('output_path') - # for 20230418 staging temporary disable the attribute - # since the backend is not ready yet - if source_file: - SrvErrorHandler.customized_handle(ECustomizedError.LINEAGE_FEATURE_NOT_READY, True) - user = UserConfig() # Check zone and upload-message zone = get_zone(zone) if zone else AppConfig.Env.green_zone.lower() diff --git a/app/resources/custom_error.py b/app/resources/custom_error.py index 6f44bd34..ed47b7be 100644 --- a/app/resources/custom_error.py +++ b/app/resources/custom_error.py @@ -122,5 +122,4 @@ class Error: 'CONTAINER_REGISTRY_NO_URL': ( 'Container registry has not yet been configured. Related commands cannot be used at this time.' ), - 'LINEAGE_FEATURE_NOT_READY': 'Lineage is not support at v2.3.0', } diff --git a/app/services/output_manager/error_handler.py b/app/services/output_manager/error_handler.py index a0344839..4f9ea3f4 100644 --- a/app/services/output_manager/error_handler.py +++ b/app/services/output_manager/error_handler.py @@ -81,7 +81,6 @@ class ECustomizedError(enum.Enum): CONTAINER_REGISTRY_NO_URL = 'CONTAINER_REGISTRY_NO_URL' CONFIG_NOT_FOUND = 'CONFIG_NOT_FOUND' CONFIG_EXIST = 'CONFIG_EXIST' - LINEAGE_FEATURE_NOT_READY = 'LINEAGE_FEATURE_NOT_READY' def customized_error_msg(customized_error: ECustomizedError): From 50c794701649a9cf5d4f56d06f619f501b302692 Mon Sep 17 00:00:00 2001 From: zhiren Date: Thu, 20 Apr 2023 10:37:14 -0400 Subject: [PATCH 6/8] bumpup version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 18304491..7d3638da 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "app" -version = "2.3.0" +version = "2.3.1" description = "This service is designed to support pilot platform" authors = ["Indoc Research"] From 80ac40a389b68d4c75d31416e31ea39fce7ec469 Mon Sep 17 00:00:00 2001 From: zhiren Date: Fri, 21 Apr 2023 12:00:51 -0400 Subject: [PATCH 7/8] fixup the preupload will recieve the source_id as None --- app/commands/file.py | 2 +- app/services/file_manager/file_upload/file_upload.py | 2 +- pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/commands/file.py b/app/commands/file.py index ee007762..acd53a0a 100644 --- a/app/commands/file.py +++ b/app/commands/file.py @@ -195,7 +195,7 @@ def file_put(**kwargs): # noqa: C901 'attribute': attribute, } if source_file: - upload_event['source_id'] = src_file_info.get('id') + upload_event['source_id'] = src_file_info.get('id', '') item_ids = simple_upload(upload_event, num_of_thread=thread, output_path=output_path) diff --git a/app/services/file_manager/file_upload/file_upload.py b/app/services/file_manager/file_upload/file_upload.py index e7e08a13..bbfc8682 100644 --- a/app/services/file_manager/file_upload/file_upload.py +++ b/app/services/file_manager/file_upload/file_upload.py @@ -123,7 +123,7 @@ def simple_upload( # noqa: C901 create_folder_flag = upload_event.get('create_folder_flag', False) compress_zip = upload_event.get('compress_zip', False) regular_file = upload_event.get('regular_file', True) - source_id = upload_event.get('source_id', None) + source_id = upload_event.get('source_id', '') attribute = upload_event.get('attribute') mhandler.SrvOutPutHandler.start_uploading(input_path) diff --git a/pyproject.toml b/pyproject.toml index 7d3638da..e665545e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "app" -version = "2.3.1" +version = "2.3.2" description = "This service is designed to support pilot platform" authors = ["Indoc Research"] From e84a3647154e355c4b38a13106d4edcafa787b38 Mon Sep 17 00:00:00 2001 From: zhiren Date: Mon, 24 Apr 2023 10:37:44 -0400 Subject: [PATCH 8/8] bumpup to next preminor --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index e665545e..773ed5a9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "app" -version = "2.3.2" +version = "2.4.0a0" description = "This service is designed to support pilot platform" authors = ["Indoc Research"]