From ca6e675eef4b9fee83c4dc946436066fc39e0bf9 Mon Sep 17 00:00:00 2001
From: zhiren <zzhan@indocresearch.org>
Date: Tue, 4 Apr 2023 12:18:45 -0400
Subject: [PATCH 01/62] add the folder resumable upload

---
 app/commands/file.py                          | 91 ++++++++++++-------
 app/resources/custom_error.py                 |  2 +-
 app/resources/custom_help.py                  |  1 +
 .../file_manager/file_upload/file_upload.py   | 75 +++++++++++++--
 .../file_manager/file_upload/models.py        | 18 ++++
 .../file_manager/file_upload/upload_client.py | 72 ++++++++++-----
 app/services/output_manager/help_page.py      |  1 +
 .../output_manager/message_handler.py         |  4 +-
 app/utils/aggregated.py                       |  3 +-
 9 files changed, 202 insertions(+), 65 deletions(-)

diff --git a/app/commands/file.py b/app/commands/file.py
index ceb65054..5d3ae93b 100644
--- a/app/commands/file.py
+++ b/app/commands/file.py
@@ -2,6 +2,8 @@
 #
 # Contact Indoc Research for any questions regarding the use of this source code.
 
+import json
+import os
 import re
 
 import click
@@ -14,6 +16,7 @@
 from app.services.file_manager.file_list import SrvFileList
 from app.services.file_manager.file_manifests import SrvFileManifests
 from app.services.file_manager.file_upload.file_upload import assemble_path
+from app.services.file_manager.file_upload.file_upload import resume_upload
 from app.services.file_manager.file_upload.file_upload import simple_upload
 from app.services.file_manager.file_upload.upload_validator import UploadEventValidator
 from app.services.output_manager.error_handler import ECustomizedError
@@ -103,27 +106,11 @@ def cli():
     show_default=True,
 )
 @click.option(
-    '--resumable-id',
-    '-rid',
-    default=None,
-    required=False,
-    help='The upload id to resume the failed upload job',
-    show_default=True,
-)
-@click.option(
-    '--job-id',
-    '-jid',
-    default=None,
+    '--output-path',
+    '-o',
+    default='./',
     required=False,
-    help='The job id to resume the failed upload job',
-    show_default=True,
-)
-@click.option(
-    '--item-id',
-    '-item',
-    default=None,
-    required=False,
-    help='The item id is required when resume an upload job',
+    help='The output path for the manifest file of resumable upload',
     show_default=True,
 )
 @doc(file_help.file_help_page(file_help.FileHELP.FILE_UPLOAD))
@@ -140,9 +127,7 @@ def file_put(**kwargs):  # noqa: C901
     zipping = kwargs.get('zip')
     attribute = kwargs.get('attribute')
     thread = kwargs.get('thread')
-    resumable_id = kwargs.get('resumable_id')
-    job_id = kwargs.get('job_id')
-    item_id = kwargs.get('item_id')
+    output_path = kwargs.get('output_path')
 
     user = UserConfig()
     # Check zone and upload-message
@@ -155,9 +140,9 @@ def file_put(**kwargs):  # noqa: C901
     # check if user input at least one file/folder
     if len(paths) == 0:
         SrvErrorHandler.customized_handle(ECustomizedError.INVALID_PATHS, True)
-    # check if resumable_id exist then job_id should also be inputed
-    if (resumable_id is None) != (job_id is None):
-        SrvErrorHandler.customized_handle(ECustomizedError.INVALID_RESUMABLE, True)
+    # # check if resumable_id exist then job_id should also be inputed
+    # if (resumable_id is None) != (job_id is None):
+    #     SrvErrorHandler.customized_handle(ECustomizedError.INVALID_RESUMABLE, True)
 
     project_path = click.prompt('ProjectCode') if not project_path else project_path
     project_code, target_folder = identify_target_folder(project_path)
@@ -184,9 +169,9 @@ def file_put(**kwargs):  # noqa: C901
                 SrvErrorHandler.customized_handle(ECustomizedError.INVALID_PIPELINENAME, True)
         if not upload_message:
             upload_message = AppConfig.Env.default_upload_message
+
     # Unique Paths
     paths = set(paths)
-
     # the loop will read all input path(folder or files)
     # and process them one by one
     for f in paths:
@@ -195,7 +180,6 @@ def file_put(**kwargs):  # noqa: C901
             target_folder,
             project_code,
             zone,
-            resumable_id,
             zipping,
         )
         upload_event = {
@@ -215,12 +199,58 @@ def file_put(**kwargs):  # noqa: C901
         if source_file:
             upload_event['valid_source'] = src_file_info
 
-        simple_upload(upload_event, num_of_thread=thread, resumable_id=resumable_id, job_id=job_id, item_id=item_id)
+        simple_upload(
+            upload_event, num_of_thread=thread, resumable_id=None, job_id=None, item_id=None, output_path=output_path
+        )
 
         srv_manifest.attach_manifest(attribute, result_file, zone) if attribute else None
         message_handler.SrvOutPutHandler.all_file_uploaded()
 
 
+@click.command(name='resume')
+@click.option(
+    '--thread',
+    '-td',
+    default=1,
+    required=False,
+    help='The number of thread for upload a file',
+    show_default=True,
+)
+@click.option(
+    '--resumable-file',
+    '-r',
+    default=None,
+    required=True,
+    help='The manifest file for resumable upload',
+    show_default=True,
+)
+@doc(file_help.file_help_page(file_help.FileHELP.FILE_UPLOAD))
+def file_resume(**kwargs):  # noqa: C901
+    """
+    Summary:
+        Resume upload file. Now split the logic of resumable upload and
+        normal file upload to make the code more clear.
+    Parameters:
+        - thread: The number of thread for upload a file
+        - resumable_file: The manifest file for resumable upload
+    """
+
+    thread = kwargs.get('thread')
+    resumable_manifest_file = kwargs.get('resumable_file')
+
+    # check if manifest file exist then read the manifest file as json
+    if not os.path.exists(resumable_manifest_file):
+        SrvErrorHandler.customized_handle(ECustomizedError.INVALID_RESUMABLE, True)
+
+    with open(resumable_manifest_file, 'r') as f:
+        resumable_manifest = json.load(f)
+        # use the same validator with upload. because resumable and normal upload
+        # are rather similar with the input
+        validate_upload_event(resumable_manifest)
+
+    resume_upload(resumable_manifest, thread)
+
+
 def validate_upload_event(event):
     """validate upload request, raise error when filed."""
     zone = event.get('zone')
@@ -357,12 +387,11 @@ def file_download(**kwargs):
     interactive = False if len(paths) > 1 else True
     # void_validate_zone('download', zone)
 
-    user = UserConfig()
     if len(paths) == 0:
         SrvErrorHandler.customized_handle(ECustomizedError.MISSING_PROJECT_CODE, interactive)
     # Query file information and collecting errors
     if geid:
-        item_res = get_file_info_by_geid(paths, user.access_token)
+        item_res = get_file_info_by_geid(paths)
     else:
         item_res = []
         for path in paths:
diff --git a/app/resources/custom_error.py b/app/resources/custom_error.py
index 9dcfabd7..c05291b0 100644
--- a/app/resources/custom_error.py
+++ b/app/resources/custom_error.py
@@ -42,7 +42,7 @@ class Error:
             'may only contain lowercase letters, numbers, and/or special characters of -_, .'
         ),
         'INVALID_PATHS': 'The input path is empty. Please select at least one file or folder to upload',
-        'INVALID_RESUMABLE': 'Both resumable_id and job_id should be specified when doing resumable upload',
+        'INVALID_RESUMABLE': 'The resumable manifest file is not exist.',
         'INVALID_FOLDERNAME': (
             'The input folder name is not valid. Please follow the rule:\n'
             ' - cannot contains special characters.\n'
diff --git a/app/resources/custom_help.py b/app/resources/custom_help.py
index de2c4bee..938f5844 100644
--- a/app/resources/custom_help.py
+++ b/app/resources/custom_help.py
@@ -32,6 +32,7 @@ class HelpPage:
             'FILE_LIST': 'List files and folders inside a given Project/folder.',
             'FILE_SYNC': 'Download files/folders from a given Project/folder/file in core zone.',
             'FILE_UPLOAD': 'Upload files/folders to a given Project path.',
+            'FILE_RESUME': 'Resume the upload process with given manifest file.',
             'FILE_Z': 'Target Zone (i.e., core/greenroom)  [default: greenroom]',
             'FILE_ATTRIBUTE_P': 'Project Code',
             'FILE_ATTRIBUTE_N': 'Attribute Template Name',
diff --git a/app/services/file_manager/file_upload/file_upload.py b/app/services/file_manager/file_upload/file_upload.py
index aad3818f..2bb35ead 100644
--- a/app/services/file_manager/file_upload/file_upload.py
+++ b/app/services/file_manager/file_upload/file_upload.py
@@ -7,6 +7,7 @@
 import time
 import zipfile
 from multiprocessing.pool import ThreadPool
+from typing import Any
 from typing import Dict
 from typing import Tuple
 
@@ -15,12 +16,14 @@
 import app.services.logger_services.log_functions as logger
 import app.services.output_manager.message_handler as mhandler
 from app.configs.app_config import AppConfig
+from app.services.file_manager.file_upload.models import FileObject
 from app.services.file_manager.file_upload.models import UploadType
 from app.services.file_manager.file_upload.upload_client import UploadClient
 from app.services.output_manager.error_handler import ECustomizedError
 from app.services.output_manager.error_handler import SrvErrorHandler
 from app.services.output_manager.error_handler import customized_error_msg
 from app.utils.aggregated import get_file_in_folder
+from app.utils.aggregated import get_file_info_by_geid
 from app.utils.aggregated import search_item
 
 
@@ -36,7 +39,7 @@ def compress_folder_to_zip(path):
 
 
 def assemble_path(
-    f: str, target_folder: str, project_code: str, zone: str, resumable_id: str, zipping: bool = False
+    f: str, target_folder: str, project_code: str, zone: str, zipping: bool = False
 ) -> Tuple[str, Dict, bool, str]:
     '''
     Summary:
@@ -54,7 +57,6 @@ def assemble_path(
          - target_folder(str): the folder on the platform
          - project_code(str): the unique identifier of project
          - zone(str): the zone label eg.greenroom/core
-         - resumable_id(str): the unique identifier of a upload process
          - zipping(bool): default False. The flag to indicate if upload as a zip
     Return:
          - current_file_path: the format file path on platform
@@ -73,7 +75,7 @@ def assemble_path(
     parent_folder = parent_folder.get('result')
     create_folder_flag = False
 
-    if len(current_file_path.split('/')) > 2 and not resumable_id:
+    if len(current_file_path.split('/')) > 2:
         sub_path = target_folder.split('/')
         for index in range(len(sub_path) - 1):
             folder_path = '/'.join(sub_path[0 : 2 + index])
@@ -87,8 +89,6 @@ def assemble_path(
                 break
             else:
                 parent_folder = res.get('result')
-    elif resumable_id:
-        mhandler.SrvOutPutHandler.resume_warning(resumable_id)
 
     # error check if the user dont have permission to see the folder
     # because the name folder will always be there if user has correct permission
@@ -106,6 +106,7 @@ def simple_upload(  # noqa: C901
     resumable_id: str = None,
     job_id: str = None,
     item_id: str = None,
+    output_path: str = None,
 ):
     upload_start_time = time.time()
     my_file = upload_event.get('file')
@@ -177,7 +178,7 @@ def simple_upload(  # noqa: C901
 
             # sending the pre upload request to generate
             # the placeholder in object storage
-            pre_upload_infos.extend(upload_client.pre_upload(file_batchs))
+            pre_upload_infos.extend(upload_client.pre_upload(file_batchs, output_path))
 
     # now loop over each file under the folder and start
     # the chunk upload
@@ -214,3 +215,65 @@ def simple_upload(  # noqa: C901
 
     num_of_file = len(upload_file_path)
     logger.info(f'Upload Time: {time.time() - upload_start_time:.2f}s for {num_of_file:d} files')
+
+
+def resume_upload(
+    manifest_json: Dict[str, Any],
+    num_of_thread: int = 1,
+):
+    """
+    Summary:
+        Resume upload from the manifest file
+    Parameters:
+        - manifest_json: the manifest json which store the upload information
+        - num_of_thread: the number of thread to upload the file
+    """
+    upload_start_time = time.time()
+
+    upload_client = UploadClient(
+        input_path=manifest_json.get('file'),
+        project_code=manifest_json.get('project_code'),
+        zone=manifest_json.get('zone'),
+        job_type='AS_FOLDER',
+        current_folder_node=manifest_json.get('current_folder_node', ''),
+        parent_folder_id=manifest_json.get('parent_folder_id', ''),
+        tags=manifest_json.get('tags'),
+    )
+
+    # check files in manifest if some of them are already uploaded
+    item_ids = [x.get('item_id') for x in manifest_json.get('file_objects')]
+    items = get_file_info_by_geid(item_ids)
+    unfinished_items = [x for x in items if x.get('status') == 'REGISTERED']  # update to enum later
+    # make them as FileObject
+    unfinished_items = [
+        FileObject(
+            x.get('resumable_id'), x.get('item_id'), x.get('job_id'), x.get('object_path'), x.get('local_paht'), []
+        )
+        for x in unfinished_items
+    ]
+
+    # then for the rest of the files, check if any chunks are already uploaded
+    unfinished_items = upload_client.resume_upload(unfinished_items)
+
+    # lastly, start resumable upload for the rest of the chunks
+    # thread number +1 reserve one thread to refresh token
+    # and remove the token decorator in functions
+
+    pool = ThreadPool(num_of_thread + 1)
+    pool.apply_async(upload_client.upload_token_refresh)
+    for file_object in unfinished_items:
+        upload_client.stream_upload(file_object, pool)
+        # NOTE: if there is some racing error make the combine chunks
+        # out of thread pool.
+        pool.apply_async(
+            upload_client.on_succeed,
+            args=(file_object, manifest_json.get('tags')),
+        )
+
+    upload_client.set_finish_upload()
+
+    pool.close()
+    pool.join()
+
+    num_of_file = len(unfinished_items)
+    logger.info(f'Upload Time: {time.time() - upload_start_time:.2f}s for {num_of_file:d} files')
diff --git a/app/services/file_manager/file_upload/models.py b/app/services/file_manager/file_upload/models.py
index 77676e92..918f04c7 100644
--- a/app/services/file_manager/file_upload/models.py
+++ b/app/services/file_manager/file_upload/models.py
@@ -73,3 +73,21 @@ def generate_meta(self, local_path: str) -> Tuple[int, int]:
         total_size = file_length_in_bytes
         total_chunks = math.ceil(total_size / AppConfig.Env.chunk_size)
         return total_size, total_chunks
+
+    def to_dict(self):
+        """
+        Summary:
+            The function is to convert the object to json format.
+        return:
+            - json format of the object.
+        """
+        return {
+            'resumable_id': self.resumable_id,
+            'job_id': self.job_id,
+            'item_id': self.item_id,
+            'object_path': self.object_path,
+            'local_path': self.local_path,
+            'total_size': self.total_size,
+            'total_chunks': self.total_chunks,
+            'uploaded_chunks': self.uploaded_chunks,
+        }
diff --git a/app/services/file_manager/file_upload/upload_client.py b/app/services/file_manager/file_upload/upload_client.py
index 7b0ad37d..3cd6b3fe 100644
--- a/app/services/file_manager/file_upload/upload_client.py
+++ b/app/services/file_manager/file_upload/upload_client.py
@@ -3,6 +3,7 @@
 # Contact Indoc Research for any questions regarding the use of this source code.
 
 import hashlib
+import json
 import math
 import os
 import time
@@ -103,15 +104,12 @@ def generate_meta(self, local_path: str) -> Tuple[int, int]:
         return total_size, total_chunks
 
     @require_valid_token()
-    def resume_upload(self, resumable_id: str, job_id: str, item_id: str, local_path: str) -> List[FileObject]:
+    def resume_upload(self, unfinished_file_objects: List[FileObject]) -> List[FileObject]:
         """
         Summary:
             The function is to check the uploaded chunks in object storage.
         Parameter:
-            - resumable_id(str): The unique id to indicate the multipart upload.
-            - job_id(str): The unique id to indicate the job id.
-            - item_id(str): The unique id for the item.
-            - local_path: the local path of interrupted file.
+            - unfinished_file_objects(List[FileObject]): the unfinished items that need to be resumed.
         return:
             - list of FileObject: the infomation retrieved from backend.
                 - resumable_id(str): the unique identifier for multipart upload.
@@ -119,20 +117,21 @@ def resume_upload(self, resumable_id: str, job_id: str, item_id: str, local_path
                 - local_path(str): the local path of file.
                 - chunk_info(dict): the mapping for chunks that already been uploaded.
         """
+        mhandler.SrvOutPutHandler.resume_warning(len(unfinished_file_objects))
 
         headers = {'Authorization': 'Bearer ' + self.user.access_token, 'Session-ID': self.user.session_id}
         url = AppConfig.Connections.url_bff + f'/v1/project/{self.project_code}/files/resumable'
-        file_name = os.path.basename(local_path)
-        object_path = os.path.join(self.current_folder_node, file_name)
+        rid_file_object_map = {x.resumable_id: x for x in unfinished_file_objects}
         payload = {
             'bucket': self.bucket,
             'zone': self.zone,
             'object_infos': [
                 {
-                    'object_path': object_path,
-                    'item_id': item_id,
-                    'resumable_id': resumable_id,
+                    'object_path': x.object_path,
+                    'item_id': x.item_id,
+                    'resumable_id': x.resumable_id,
                 }
+                for x in unfinished_file_objects
             ],
         }
 
@@ -144,27 +143,21 @@ def resume_upload(self, resumable_id: str, job_id: str, item_id: str, local_path
         uploaded_infos = response.json().get('result', [])
         file_objects = []
         for uploaded_info in uploaded_infos:
-            file_objects.append(
-                FileObject(
-                    uploaded_info.get('resumable_id'),
-                    job_id,
-                    item_id,
-                    uploaded_info.get('object_path'),
-                    local_path,  # TODO change it after folder manifest is setup
-                    uploaded_info.get('chunks_info'),
-                )
-            )
+            file_obj = rid_file_object_map.get(uploaded_info.get('resumable_id'))
+            # update the chunk info
+            file_obj.uploaded_chunks = uploaded_info.get('chunks_info')
         mhandler.SrvOutPutHandler.resume_check_success()
 
         return file_objects
 
     @require_valid_token()
-    def pre_upload(self, local_file_paths: List[str]) -> List[FileObject]:
+    def pre_upload(self, local_file_paths: List[str], output_path: str) -> List[FileObject]:
         """
         Summary:
             The function is to initiate all the multipart upload.
         Parameter:
             - local_file_paths(list of str): the local path of files to be uploaded.
+            - output_path(str): the output path of manifest.
         return:
             - list of FileObject: the infomation retrieved from backend.
                 - resumable_id(str): the unique identifier for multipart upload.
@@ -192,16 +185,21 @@ def pre_upload(self, local_file_paths: List[str]) -> List[FileObject]:
         response = resilient_session().post(url, json=payload, headers=headers, timeout=None)
         if response.status_code == 200:
             result = response.json().get('result')
-            res = []
+            file_objets = []
             for job in result:
                 object_path = job.get('target_names')[0]
                 resumable_id = job.get('payload').get('resumable_identifier')
                 item_id = job.get('payload').get('item_id')
                 job_id = job.get('job_id')
-                res.append(FileObject(resumable_id, job_id, item_id, object_path, file_mapping.get(object_path), {}))
+                file_objets.append(
+                    FileObject(resumable_id, job_id, item_id, object_path, file_mapping.get(object_path), {})
+                )
+
+            # then output manifest file to the output path
+            self.output_manifest(file_objets, output_path)
 
             mhandler.SrvOutPutHandler.preupload_success()
-            return res
+            return file_objets
         elif response.status_code == 403:
             SrvErrorHandler.customized_handle(ECustomizedError.PERMISSION_DENIED, self.regular_file)
         elif response.status_code == 401:
@@ -216,6 +214,32 @@ def pre_upload(self, local_file_paths: List[str]) -> List[FileObject]:
         else:
             SrvErrorHandler.default_handle(str(response.status_code) + ': ' + str(response.content), self.regular_file)
 
+    def output_manifest(self, file_objects: List[FileObject], output_path: str) -> None:
+        """
+        Summary:
+            The function is to output the manifest file.
+        Parameter:
+            - file_objects(list of FileObject): the file objects that contains correct
+                information for chunk uploading.
+        return:
+            - manifest_json(dict): the manifest file in json format.
+        """
+
+        manifest_json = {
+            'project_code': self.project_code,
+            'operator': self.operator,
+            'zone': self.zone,
+            'parent_folder_id': self.parent_folder_id,
+            'current_folder_node': self.current_folder_node,
+            'tags': self.tags,
+            'file_objects': {file_object.item_id: file_object.to_dict() for file_object in file_objects},
+        }
+
+        with open(output_path, 'w') as f:
+            json.dump(manifest_json, f)
+
+        return manifest_json
+
     def stream_upload(self, file_object: FileObject, pool: ThreadPool) -> None:
         """
         Summary:
diff --git a/app/services/output_manager/help_page.py b/app/services/output_manager/help_page.py
index 04aa0fa3..177e1753 100644
--- a/app/services/output_manager/help_page.py
+++ b/app/services/output_manager/help_page.py
@@ -66,6 +66,7 @@ class FileHELP(enum.Enum):
     FILE_LIST = 'USER_LOGOUT_CONFIRM'
     FILE_SYNC = 'USER_LOGIN_USERNAME'
     FILE_UPLOAD = 'USER_LOGIN_PASSWORD'
+    FILE_RESUME = 'FILE_RESUME'
     FILE_ATTRIBUTE_P = 'FILE_ATTRIBUTE_P'
     FILE_ATTRIBUTE_N = 'FILE_ATTRIBUTE_N'
     FILE_Z = 'FILE_Z'
diff --git a/app/services/output_manager/message_handler.py b/app/services/output_manager/message_handler.py
index 90ad68da..65217687 100644
--- a/app/services/output_manager/message_handler.py
+++ b/app/services/output_manager/message_handler.py
@@ -134,9 +134,9 @@ def resume_check_success():
         return logger.info('Resumable upload check complete.')
 
     @staticmethod
-    def resume_warning(resumable_id: str):
+    def resume_warning(num_of_files: int):
         """e.g. notify the user if they comfirm the resumable upload."""
-        return logger.warning(f'Resume the upload for {resumable_id}.')
+        return logger.warning(f'Resume the upload for {num_of_files} files.')
 
     @staticmethod
     def start_finalizing():
diff --git a/app/utils/aggregated.py b/app/utils/aggregated.py
index e929f3bc..281eee63 100644
--- a/app/utils/aggregated.py
+++ b/app/utils/aggregated.py
@@ -49,7 +49,8 @@ def search_item(project_code, zone, folder_relative_path, item_type, container_t
 
 
 @require_valid_token()
-def get_file_info_by_geid(geid: list, token):
+def get_file_info_by_geid(geid: list):
+    token = UserConfig().access_token
     payload = {'geid': geid}
     headers = {'Authorization': 'Bearer ' + token}
     url = AppConfig.Connections.url_bff + '/v1/query/geid'

From 798f8047075ed820cd1a119ad4f3ff326d596639 Mon Sep 17 00:00:00 2001
From: zhiren <zzhan@indocresearch.org>
Date: Tue, 4 Apr 2023 15:39:33 -0400
Subject: [PATCH 02/62] add the test case for resumable upload

---
 .../file_manager/file_upload/upload_client.py | 11 ++--
 test                                          |  0
 .../file_upload/test_upload_client.py         | 59 +++++++++++++++++++
 3 files changed, 65 insertions(+), 5 deletions(-)
 create mode 100644 test

diff --git a/app/services/file_manager/file_upload/upload_client.py b/app/services/file_manager/file_upload/upload_client.py
index 3cd6b3fe..41dcc768 100644
--- a/app/services/file_manager/file_upload/upload_client.py
+++ b/app/services/file_manager/file_upload/upload_client.py
@@ -8,6 +8,8 @@
 import os
 import time
 from multiprocessing.pool import ThreadPool
+from typing import Any
+from typing import Dict
 from typing import List
 from typing import Tuple
 
@@ -103,7 +105,7 @@ def generate_meta(self, local_path: str) -> Tuple[int, int]:
         total_chunks = math.ceil(total_size / self.chunk_size)
         return total_size, total_chunks
 
-    @require_valid_token()
+    # @require_valid_token()
     def resume_upload(self, unfinished_file_objects: List[FileObject]) -> List[FileObject]:
         """
         Summary:
@@ -141,16 +143,15 @@ def resume_upload(self, unfinished_file_objects: List[FileObject]) -> List[FileO
 
         # make the response into file objects
         uploaded_infos = response.json().get('result', [])
-        file_objects = []
         for uploaded_info in uploaded_infos:
             file_obj = rid_file_object_map.get(uploaded_info.get('resumable_id'))
             # update the chunk info
             file_obj.uploaded_chunks = uploaded_info.get('chunks_info')
         mhandler.SrvOutPutHandler.resume_check_success()
 
-        return file_objects
+        return unfinished_file_objects
 
-    @require_valid_token()
+    # @require_valid_token()
     def pre_upload(self, local_file_paths: List[str], output_path: str) -> List[FileObject]:
         """
         Summary:
@@ -214,7 +215,7 @@ def pre_upload(self, local_file_paths: List[str], output_path: str) -> List[File
         else:
             SrvErrorHandler.default_handle(str(response.status_code) + ': ' + str(response.content), self.regular_file)
 
-    def output_manifest(self, file_objects: List[FileObject], output_path: str) -> None:
+    def output_manifest(self, file_objects: List[FileObject], output_path: str) -> Dict[str, Any]:
         """
         Summary:
             The function is to output the manifest file.
diff --git a/test b/test
new file mode 100644
index 00000000..e69de29b
diff --git a/tests/app/services/file_manager/file_upload/test_upload_client.py b/tests/app/services/file_manager/file_upload/test_upload_client.py
index 1a68e681..a4c0ec6f 100644
--- a/tests/app/services/file_manager/file_upload/test_upload_client.py
+++ b/tests/app/services/file_manager/file_upload/test_upload_client.py
@@ -55,3 +55,62 @@ def test_token_refresh_auto(mocker):
 
     # make sure the token refresh function is called
     token_refresh_mock.assert_called_once()
+
+
+def test_resumable_pre_upload_success(httpx_mock, mocker):
+    upload_client = UploadClient('test', 'project_code', 'parent_folder_id')
+    mocker.patch('app.services.file_manager.file_upload.models.FileObject.generate_meta', return_value=(1, 1))
+    test_obj = FileObject('resumable_id', 'job_id', 'item_id', 'object/path', 'local_path', [])
+
+    url = AppConfig.Connections.url_bff + f'/v1/project/{upload_client.project_code}/files/resumable'
+    httpx_mock.add_response(
+        method='POST', url=url, json={'result': [{'resumable_id': 'resumable_id', 'chunks_info': ['chunks_info']}]}
+    )
+
+    res = upload_client.resume_upload([test_obj])
+
+    assert len(res) == 1
+    assert res[0].resumable_id == 'resumable_id'
+    assert res[0].uploaded_chunks == ['chunks_info']
+
+
+def test_resumable_pre_upload_failed_with_404(httpx_mock, mocker):
+    upload_client = UploadClient('test', 'project_code', 'parent_folder_id')
+    mocker.patch('app.services.file_manager.file_upload.models.FileObject.generate_meta', return_value=(1, 1))
+    test_obj = FileObject('resumable_id', 'job_id', 'item_id', 'object/path', 'local_path', [])
+
+    url = AppConfig.Connections.url_bff + f'/v1/project/{upload_client.project_code}/files/resumable'
+    httpx_mock.add_response(
+        method='POST',
+        url=url,
+        json={'result': [{'resumable_id': 'resumable_id', 'chunks_info': ['chunks_info']}]},
+        status_code=404,
+    )
+
+    try:
+        upload_client.resume_upload([test_obj])
+    except SystemExit:
+        pass
+    else:
+        AssertionError('SystemExit not raised')
+
+
+def test_output_manifest_success(mocker):
+    upload_client = UploadClient('test', 'project_code', 'parent_folder_id')
+    json_dump_mocker = mocker.patch('json.dump', return_value=None)
+    mocker.patch('app.services.file_manager.file_upload.models.FileObject.generate_meta', return_value=(1, 1))
+    test_obj = FileObject('resumable_id', 'job_id', 'item_id', 'object/path', 'local_path', [])
+
+    res = upload_client.output_manifest([test_obj], 'test')
+
+    assert res.get('project_code') == 'project_code'
+    assert res.get('parent_folder_id') == 'parent_folder_id'
+    assert len(res.get('file_objects')) == 1
+
+    file_item = res.get('file_objects').get('item_id')
+    assert file_item.get('resumable_id') == 'resumable_id'
+    assert file_item.get('local_path') == 'local_path'
+    assert file_item.get('object_path') == 'object/path'
+    assert file_item.get('item_id') == 'item_id'
+
+    json_dump_mocker.assert_called_once()

From 4337e64d91dbcbe73afb8ef18515a09c9d650275 Mon Sep 17 00:00:00 2001
From: zhiren <zzhan@indocresearch.org>
Date: Tue, 4 Apr 2023 16:08:08 -0400
Subject: [PATCH 03/62] add the test case for resume upload command

---
 poetry.lock                     | 1188 ++++++++++++++++++++++++-------
 pyproject.toml                  |    1 +
 tests/app/commands/test_file.py |   27 +
 3 files changed, 971 insertions(+), 245 deletions(-)
 create mode 100644 tests/app/commands/test_file.py

diff --git a/poetry.lock b/poetry.lock
index 18173271..8fef1458 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -1,3 +1,5 @@
+# This file is automatically @generated by Poetry and should not be changed by hand.
+
 [[package]]
 name = "aioboto3"
 version = "9.6.0"
@@ -5,13 +7,17 @@ description = "Async boto3 wrapper"
 category = "main"
 optional = false
 python-versions = ">=3.7,<4.0"
+files = [
+    {file = "aioboto3-9.6.0-py3-none-any.whl", hash = "sha256:a62de3203c7372b5f1247057b9705eeb26c45ee65b5d1fab91767e4f31fd7b47"},
+    {file = "aioboto3-9.6.0.tar.gz", hash = "sha256:abac5dcfa871627b7040e6586a69e3359e2dfc4e15dc66135969f2d26fbdcb3b"},
+]
 
 [package.dependencies]
 aiobotocore = {version = "2.3.0", extras = ["boto3"]}
 
 [package.extras]
-s3cse = ["cryptography (>=2.3.1)"]
 chalice = ["chalice (>=1.24.0)"]
+s3cse = ["cryptography (>=2.3.1)"]
 
 [[package]]
 name = "aiobotocore"
@@ -20,6 +26,9 @@ description = "Async client for aws services using botocore and aiohttp"
 category = "main"
 optional = false
 python-versions = ">=3.6"
+files = [
+    {file = "aiobotocore-2.3.0.tar.gz", hash = "sha256:fc3d3d6061410bc194d09ec545e30b4469d5f74770f937aca5f6aa45e62a1bfe"},
+]
 
 [package.dependencies]
 aiohttp = ">=3.3.1"
@@ -39,6 +48,95 @@ description = "Async http client/server framework (asyncio)"
 category = "main"
 optional = false
 python-versions = ">=3.6"
+files = [
+    {file = "aiohttp-3.8.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ba71c9b4dcbb16212f334126cc3d8beb6af377f6703d9dc2d9fb3874fd667ee9"},
+    {file = "aiohttp-3.8.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d24b8bb40d5c61ef2d9b6a8f4528c2f17f1c5d2d31fed62ec860f6006142e83e"},
+    {file = "aiohttp-3.8.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f88df3a83cf9df566f171adba39d5bd52814ac0b94778d2448652fc77f9eb491"},
+    {file = "aiohttp-3.8.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b97decbb3372d4b69e4d4c8117f44632551c692bb1361b356a02b97b69e18a62"},
+    {file = "aiohttp-3.8.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:309aa21c1d54b8ef0723181d430347d7452daaff93e8e2363db8e75c72c2fb2d"},
+    {file = "aiohttp-3.8.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ad5383a67514e8e76906a06741febd9126fc7c7ff0f599d6fcce3e82b80d026f"},
+    {file = "aiohttp-3.8.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:20acae4f268317bb975671e375493dbdbc67cddb5f6c71eebdb85b34444ac46b"},
+    {file = "aiohttp-3.8.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:05a3c31c6d7cd08c149e50dc7aa2568317f5844acd745621983380597f027a18"},
+    {file = "aiohttp-3.8.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d6f76310355e9fae637c3162936e9504b4767d5c52ca268331e2756e54fd4ca5"},
+    {file = "aiohttp-3.8.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:256deb4b29fe5e47893fa32e1de2d73c3afe7407738bd3c63829874661d4822d"},
+    {file = "aiohttp-3.8.3-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:5c59fcd80b9049b49acd29bd3598cada4afc8d8d69bd4160cd613246912535d7"},
+    {file = "aiohttp-3.8.3-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:059a91e88f2c00fe40aed9031b3606c3f311414f86a90d696dd982e7aec48142"},
+    {file = "aiohttp-3.8.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:2feebbb6074cdbd1ac276dbd737b40e890a1361b3cc30b74ac2f5e24aab41f7b"},
+    {file = "aiohttp-3.8.3-cp310-cp310-win32.whl", hash = "sha256:5bf651afd22d5f0c4be16cf39d0482ea494f5c88f03e75e5fef3a85177fecdeb"},
+    {file = "aiohttp-3.8.3-cp310-cp310-win_amd64.whl", hash = "sha256:653acc3880459f82a65e27bd6526e47ddf19e643457d36a2250b85b41a564715"},
+    {file = "aiohttp-3.8.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:86fc24e58ecb32aee09f864cb11bb91bc4c1086615001647dbfc4dc8c32f4008"},
+    {file = "aiohttp-3.8.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:75e14eac916f024305db517e00a9252714fce0abcb10ad327fb6dcdc0d060f1d"},
+    {file = "aiohttp-3.8.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d1fde0f44029e02d02d3993ad55ce93ead9bb9b15c6b7ccd580f90bd7e3de476"},
+    {file = "aiohttp-3.8.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4ab94426ddb1ecc6a0b601d832d5d9d421820989b8caa929114811369673235c"},
+    {file = "aiohttp-3.8.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:89d2e02167fa95172c017732ed7725bc8523c598757f08d13c5acca308e1a061"},
+    {file = "aiohttp-3.8.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:02f9a2c72fc95d59b881cf38a4b2be9381b9527f9d328771e90f72ac76f31ad8"},
+    {file = "aiohttp-3.8.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9c7149272fb5834fc186328e2c1fa01dda3e1fa940ce18fded6d412e8f2cf76d"},
+    {file = "aiohttp-3.8.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:512bd5ab136b8dc0ffe3fdf2dfb0c4b4f49c8577f6cae55dca862cd37a4564e2"},
+    {file = "aiohttp-3.8.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:7018ecc5fe97027214556afbc7c502fbd718d0740e87eb1217b17efd05b3d276"},
+    {file = "aiohttp-3.8.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:88c70ed9da9963d5496d38320160e8eb7e5f1886f9290475a881db12f351ab5d"},
+    {file = "aiohttp-3.8.3-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:da22885266bbfb3f78218dc40205fed2671909fbd0720aedba39b4515c038091"},
+    {file = "aiohttp-3.8.3-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:e65bc19919c910127c06759a63747ebe14f386cda573d95bcc62b427ca1afc73"},
+    {file = "aiohttp-3.8.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:08c78317e950e0762c2983f4dd58dc5e6c9ff75c8a0efeae299d363d439c8e34"},
+    {file = "aiohttp-3.8.3-cp311-cp311-win32.whl", hash = "sha256:45d88b016c849d74ebc6f2b6e8bc17cabf26e7e40c0661ddd8fae4c00f015697"},
+    {file = "aiohttp-3.8.3-cp311-cp311-win_amd64.whl", hash = "sha256:96372fc29471646b9b106ee918c8eeb4cca423fcbf9a34daa1b93767a88a2290"},
+    {file = "aiohttp-3.8.3-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:c971bf3786b5fad82ce5ad570dc6ee420f5b12527157929e830f51c55dc8af77"},
+    {file = "aiohttp-3.8.3-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ff25f48fc8e623d95eca0670b8cc1469a83783c924a602e0fbd47363bb54aaca"},
+    {file = "aiohttp-3.8.3-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e381581b37db1db7597b62a2e6b8b57c3deec95d93b6d6407c5b61ddc98aca6d"},
+    {file = "aiohttp-3.8.3-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:db19d60d846283ee275d0416e2a23493f4e6b6028825b51290ac05afc87a6f97"},
+    {file = "aiohttp-3.8.3-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:25892c92bee6d9449ffac82c2fe257f3a6f297792cdb18ad784737d61e7a9a85"},
+    {file = "aiohttp-3.8.3-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:398701865e7a9565d49189f6c90868efaca21be65c725fc87fc305906be915da"},
+    {file = "aiohttp-3.8.3-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:4a4fbc769ea9b6bd97f4ad0b430a6807f92f0e5eb020f1e42ece59f3ecfc4585"},
+    {file = "aiohttp-3.8.3-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:b29bfd650ed8e148f9c515474a6ef0ba1090b7a8faeee26b74a8ff3b33617502"},
+    {file = "aiohttp-3.8.3-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:1e56b9cafcd6531bab5d9b2e890bb4937f4165109fe98e2b98ef0dcfcb06ee9d"},
+    {file = "aiohttp-3.8.3-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:ec40170327d4a404b0d91855d41bfe1fe4b699222b2b93e3d833a27330a87a6d"},
+    {file = "aiohttp-3.8.3-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:2df5f139233060578d8c2c975128fb231a89ca0a462b35d4b5fcf7c501ebdbe1"},
+    {file = "aiohttp-3.8.3-cp36-cp36m-win32.whl", hash = "sha256:f973157ffeab5459eefe7b97a804987876dd0a55570b8fa56b4e1954bf11329b"},
+    {file = "aiohttp-3.8.3-cp36-cp36m-win_amd64.whl", hash = "sha256:437399385f2abcd634865705bdc180c8314124b98299d54fe1d4c8990f2f9494"},
+    {file = "aiohttp-3.8.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:09e28f572b21642128ef31f4e8372adb6888846f32fecb288c8b0457597ba61a"},
+    {file = "aiohttp-3.8.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6f3553510abdbec67c043ca85727396ceed1272eef029b050677046d3387be8d"},
+    {file = "aiohttp-3.8.3-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e168a7560b7c61342ae0412997b069753f27ac4862ec7867eff74f0fe4ea2ad9"},
+    {file = "aiohttp-3.8.3-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:db4c979b0b3e0fa7e9e69ecd11b2b3174c6963cebadeecfb7ad24532ffcdd11a"},
+    {file = "aiohttp-3.8.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e164e0a98e92d06da343d17d4e9c4da4654f4a4588a20d6c73548a29f176abe2"},
+    {file = "aiohttp-3.8.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e8a78079d9a39ca9ca99a8b0ac2fdc0c4d25fc80c8a8a82e5c8211509c523363"},
+    {file = "aiohttp-3.8.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:21b30885a63c3f4ff5b77a5d6caf008b037cb521a5f33eab445dc566f6d092cc"},
+    {file = "aiohttp-3.8.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:4b0f30372cef3fdc262f33d06e7b411cd59058ce9174ef159ad938c4a34a89da"},
+    {file = "aiohttp-3.8.3-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:8135fa153a20d82ffb64f70a1b5c2738684afa197839b34cc3e3c72fa88d302c"},
+    {file = "aiohttp-3.8.3-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:ad61a9639792fd790523ba072c0555cd6be5a0baf03a49a5dd8cfcf20d56df48"},
+    {file = "aiohttp-3.8.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:978b046ca728073070e9abc074b6299ebf3501e8dee5e26efacb13cec2b2dea0"},
+    {file = "aiohttp-3.8.3-cp37-cp37m-win32.whl", hash = "sha256:0d2c6d8c6872df4a6ec37d2ede71eff62395b9e337b4e18efd2177de883a5033"},
+    {file = "aiohttp-3.8.3-cp37-cp37m-win_amd64.whl", hash = "sha256:21d69797eb951f155026651f7e9362877334508d39c2fc37bd04ff55b2007091"},
+    {file = "aiohttp-3.8.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:2ca9af5f8f5812d475c5259393f52d712f6d5f0d7fdad9acdb1107dd9e3cb7eb"},
+    {file = "aiohttp-3.8.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d90043c1882067f1bd26196d5d2db9aa6d268def3293ed5fb317e13c9413ea4"},
+    {file = "aiohttp-3.8.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:d737fc67b9a970f3234754974531dc9afeea11c70791dcb7db53b0cf81b79784"},
+    {file = "aiohttp-3.8.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ebf909ea0a3fc9596e40d55d8000702a85e27fd578ff41a5500f68f20fd32e6c"},
+    {file = "aiohttp-3.8.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5835f258ca9f7c455493a57ee707b76d2d9634d84d5d7f62e77be984ea80b849"},
+    {file = "aiohttp-3.8.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:da37dcfbf4b7f45d80ee386a5f81122501ec75672f475da34784196690762f4b"},
+    {file = "aiohttp-3.8.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:87f44875f2804bc0511a69ce44a9595d5944837a62caecc8490bbdb0e18b1342"},
+    {file = "aiohttp-3.8.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:527b3b87b24844ea7865284aabfab08eb0faf599b385b03c2aa91fc6edd6e4b6"},
+    {file = "aiohttp-3.8.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:d5ba88df9aa5e2f806650fcbeedbe4f6e8736e92fc0e73b0400538fd25a4dd96"},
+    {file = "aiohttp-3.8.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:e7b8813be97cab8cb52b1375f41f8e6804f6507fe4660152e8ca5c48f0436017"},
+    {file = "aiohttp-3.8.3-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:2dea10edfa1a54098703cb7acaa665c07b4e7568472a47f4e64e6319d3821ccf"},
+    {file = "aiohttp-3.8.3-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:713d22cd9643ba9025d33c4af43943c7a1eb8547729228de18d3e02e278472b6"},
+    {file = "aiohttp-3.8.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:2d252771fc85e0cf8da0b823157962d70639e63cb9b578b1dec9868dd1f4f937"},
+    {file = "aiohttp-3.8.3-cp38-cp38-win32.whl", hash = "sha256:66bd5f950344fb2b3dbdd421aaa4e84f4411a1a13fca3aeb2bcbe667f80c9f76"},
+    {file = "aiohttp-3.8.3-cp38-cp38-win_amd64.whl", hash = "sha256:84b14f36e85295fe69c6b9789b51a0903b774046d5f7df538176516c3e422446"},
+    {file = "aiohttp-3.8.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:16c121ba0b1ec2b44b73e3a8a171c4f999b33929cd2397124a8c7fcfc8cd9e06"},
+    {file = "aiohttp-3.8.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8d6aaa4e7155afaf994d7924eb290abbe81a6905b303d8cb61310a2aba1c68ba"},
+    {file = "aiohttp-3.8.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:43046a319664a04b146f81b40e1545d4c8ac7b7dd04c47e40bf09f65f2437346"},
+    {file = "aiohttp-3.8.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:599418aaaf88a6d02a8c515e656f6faf3d10618d3dd95866eb4436520096c84b"},
+    {file = "aiohttp-3.8.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:92a2964319d359f494f16011e23434f6f8ef0434acd3cf154a6b7bec511e2fb7"},
+    {file = "aiohttp-3.8.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:73a4131962e6d91109bca6536416aa067cf6c4efb871975df734f8d2fd821b37"},
+    {file = "aiohttp-3.8.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:598adde339d2cf7d67beaccda3f2ce7c57b3b412702f29c946708f69cf8222aa"},
+    {file = "aiohttp-3.8.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:75880ed07be39beff1881d81e4a907cafb802f306efd6d2d15f2b3c69935f6fb"},
+    {file = "aiohttp-3.8.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:a0239da9fbafd9ff82fd67c16704a7d1bccf0d107a300e790587ad05547681c8"},
+    {file = "aiohttp-3.8.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:4e3a23ec214e95c9fe85a58470b660efe6534b83e6cbe38b3ed52b053d7cb6ad"},
+    {file = "aiohttp-3.8.3-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:47841407cc89a4b80b0c52276f3cc8138bbbfba4b179ee3acbd7d77ae33f7ac4"},
+    {file = "aiohttp-3.8.3-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:54d107c89a3ebcd13228278d68f1436d3f33f2dd2af5415e3feaeb1156e1a62c"},
+    {file = "aiohttp-3.8.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:c37c5cce780349d4d51739ae682dec63573847a2a8dcb44381b174c3d9c8d403"},
+    {file = "aiohttp-3.8.3-cp39-cp39-win32.whl", hash = "sha256:f178d2aadf0166be4df834c4953da2d7eef24719e8aec9a65289483eeea9d618"},
+    {file = "aiohttp-3.8.3-cp39-cp39-win_amd64.whl", hash = "sha256:88e5be56c231981428f4f506c68b6a46fa25c4123a2e86d156c58a8369d31ab7"},
+    {file = "aiohttp-3.8.3.tar.gz", hash = "sha256:3828fb41b7203176b82fe5d699e0d845435f2374750a44b480ea6b930f6be269"},
+]
 
 [package.dependencies]
 aiosignal = ">=1.1.2"
@@ -52,7 +150,7 @@ typing-extensions = {version = ">=3.7.4", markers = "python_version < \"3.8\""}
 yarl = ">=1.0,<2.0"
 
 [package.extras]
-speedups = ["aiodns", "brotli", "cchardet"]
+speedups = ["Brotli", "aiodns", "cchardet"]
 
 [[package]]
 name = "aioitertools"
@@ -61,6 +159,10 @@ description = "itertools and builtins for AsyncIO and mixed iterables"
 category = "main"
 optional = false
 python-versions = ">=3.6"
+files = [
+    {file = "aioitertools-0.11.0-py3-none-any.whl", hash = "sha256:04b95e3dab25b449def24d7df809411c10e62aab0cbe31a50ca4e68748c43394"},
+    {file = "aioitertools-0.11.0.tar.gz", hash = "sha256:42c68b8dd3a69c2bf7f2233bf7df4bb58b557bca5252ac02ed5187bbc67d6831"},
+]
 
 [package.dependencies]
 typing_extensions = {version = ">=4.0", markers = "python_version < \"3.10\""}
@@ -72,6 +174,10 @@ description = "asyncio (PEP 3156) Redis support"
 category = "main"
 optional = false
 python-versions = ">=3.6"
+files = [
+    {file = "aioredis-2.0.1-py3-none-any.whl", hash = "sha256:9ac0d0b3b485d293b8ca1987e6de8658d7dafcca1cddfcd1d506cae8cdebfdd6"},
+    {file = "aioredis-2.0.1.tar.gz", hash = "sha256:eaa51aaf993f2d71f54b70527c440437ba65340588afeb786cd87c55c89cd98e"},
+]
 
 [package.dependencies]
 async-timeout = "*"
@@ -87,6 +193,10 @@ description = "aiosignal: a list of registered asynchronous callbacks"
 category = "main"
 optional = false
 python-versions = ">=3.7"
+files = [
+    {file = "aiosignal-1.3.1-py3-none-any.whl", hash = "sha256:f8376fb07dd1e86a584e4fcdec80b36b7f81aac666ebc724e2c090300dd83b17"},
+    {file = "aiosignal-1.3.1.tar.gz", hash = "sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc"},
+]
 
 [package.dependencies]
 frozenlist = ">=1.1.0"
@@ -98,6 +208,10 @@ description = "Python graph (network) package"
 category = "main"
 optional = false
 python-versions = "*"
+files = [
+    {file = "altgraph-0.17.3-py2.py3-none-any.whl", hash = "sha256:c8ac1ca6772207179ed8003ce7687757c04b0b71536f81e2ac5755c6226458fe"},
+    {file = "altgraph-0.17.3.tar.gz", hash = "sha256:ad33358114df7c9416cdb8fa1eaa5852166c505118717021c6a8c7c7abbd03dd"},
+]
 
 [[package]]
 name = "anyio"
@@ -106,6 +220,10 @@ description = "High level compatibility layer for multiple asynchronous event lo
 category = "main"
 optional = false
 python-versions = ">=3.6.2"
+files = [
+    {file = "anyio-3.6.2-py3-none-any.whl", hash = "sha256:fbbe32bd270d2a2ef3ed1c5d45041250284e31fc0a4df4a5a6071842051a51e3"},
+    {file = "anyio-3.6.2.tar.gz", hash = "sha256:25ea0d673ae30af41a0c442f81cf3b38c7e79fdc7b60335a4c14e05eb0947421"},
+]
 
 [package.dependencies]
 idna = ">=2.8"
@@ -113,8 +231,8 @@ sniffio = ">=1.1"
 typing-extensions = {version = "*", markers = "python_version < \"3.8\""}
 
 [package.extras]
-doc = ["packaging", "sphinx-rtd-theme", "sphinx-autodoc-typehints (>=1.2.0)"]
-test = ["coverage[toml] (>=4.5)", "hypothesis (>=4.0)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "contextlib2", "uvloop (<0.15)", "mock (>=4)", "uvloop (>=0.15)"]
+doc = ["packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx-rtd-theme"]
+test = ["contextlib2", "coverage[toml] (>=4.5)", "hypothesis (>=4.0)", "mock (>=4)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "uvloop (<0.15)", "uvloop (>=0.15)"]
 trio = ["trio (>=0.16,<0.22)"]
 
 [[package]]
@@ -124,6 +242,10 @@ description = "Timeout context manager for asyncio programs"
 category = "main"
 optional = false
 python-versions = ">=3.6"
+files = [
+    {file = "async-timeout-4.0.2.tar.gz", hash = "sha256:2163e1640ddb52b7a8c80d0a67a08587e5d245cc9c553a74a847056bc2976b15"},
+    {file = "async_timeout-4.0.2-py3-none-any.whl", hash = "sha256:8ca1e4fcf50d07413d66d1a5e416e42cfdf5851c981d679a09851a6853383b3c"},
+]
 
 [package.dependencies]
 typing-extensions = {version = ">=3.6.5", markers = "python_version < \"3.8\""}
@@ -135,14 +257,21 @@ description = "Enhance the standard unittest package with features for testing a
 category = "main"
 optional = false
 python-versions = ">=3.5"
+files = [
+    {file = "asynctest-0.13.0-py3-none-any.whl", hash = "sha256:5da6118a7e6d6b54d83a8f7197769d046922a44d2a99c21382f0a6e4fadae676"},
+    {file = "asynctest-0.13.0.tar.gz", hash = "sha256:c27862842d15d83e6a34eb0b2866c323880eb3a75e4485b079ea11748fd77fac"},
+]
 
 [[package]]
 name = "atomicwrites"
 version = "1.4.1"
 description = "Atomic file writes."
-category = "dev"
+category = "main"
 optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
+files = [
+    {file = "atomicwrites-1.4.1.tar.gz", hash = "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11"},
+]
 
 [[package]]
 name = "attrs"
@@ -151,14 +280,17 @@ description = "Classes Without Boilerplate"
 category = "main"
 optional = false
 python-versions = ">=3.6"
+files = [
+    {file = "attrs-22.2.0-py3-none-any.whl", hash = "sha256:29e95c7f6778868dbd49170f98f8818f78f3dc5e0e37c0b1f474e3561b240836"},
+    {file = "attrs-22.2.0.tar.gz", hash = "sha256:c9227bfc2f01993c03f68db37d1d15c9690188323c067c641f1a35ca58185f99"},
+]
 
 [package.extras]
-cov = ["attrs", "coverage-enable-subprocess", "coverage[toml] (>=5.3)"]
-dev = ["attrs"]
-docs = ["furo", "sphinx", "myst-parser", "zope.interface", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier"]
-tests = ["attrs", "zope.interface"]
-tests-no-zope = ["hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist", "cloudpickle", "mypy (>=0.971,<0.990)", "pytest-mypy-plugins"]
-tests_no_zope = ["hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist", "cloudpickle", "mypy (>=0.971,<0.990)", "pytest-mypy-plugins"]
+cov = ["attrs[tests]", "coverage-enable-subprocess", "coverage[toml] (>=5.3)"]
+dev = ["attrs[docs,tests]"]
+docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope.interface"]
+tests = ["attrs[tests-no-zope]", "zope.interface"]
+tests-no-zope = ["cloudpickle", "cloudpickle", "hypothesis", "hypothesis", "mypy (>=0.971,<0.990)", "mypy (>=0.971,<0.990)", "pympler", "pympler", "pytest (>=4.3.0)", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-mypy-plugins", "pytest-xdist[psutil]", "pytest-xdist[psutil]"]
 
 [[package]]
 name = "boto3"
@@ -167,6 +299,10 @@ description = "The AWS SDK for Python"
 category = "main"
 optional = false
 python-versions = ">= 3.6"
+files = [
+    {file = "boto3-1.21.21-py3-none-any.whl", hash = "sha256:8fa32fcc8be38327bd667237223d71e5e4b2475f39d6882aca4dbad19fff8c29"},
+    {file = "boto3-1.21.21.tar.gz", hash = "sha256:6fa0622f308cfd1da758966fc98b52fbd74b80606d14586c8ad82c7a6c4f32d0"},
+]
 
 [package.dependencies]
 botocore = ">=1.24.21,<1.25.0"
@@ -183,6 +319,10 @@ description = "Low-level, data-driven core of boto 3."
 category = "main"
 optional = false
 python-versions = ">= 3.6"
+files = [
+    {file = "botocore-1.24.21-py3-none-any.whl", hash = "sha256:92daca8775e738a9db9b465d533019285f09d541e903233261299fd87c2f842c"},
+    {file = "botocore-1.24.21.tar.gz", hash = "sha256:7e976cfd0a61601e74624ef8f5246b40a01f2cce73a011ef29cf80a6e371d0fa"},
+]
 
 [package.dependencies]
 jmespath = ">=0.7.1,<2.0.0"
@@ -199,6 +339,10 @@ description = "Python package for providing Mozilla's CA Bundle."
 category = "main"
 optional = false
 python-versions = ">=3.6"
+files = [
+    {file = "certifi-2022.12.7-py3-none-any.whl", hash = "sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18"},
+    {file = "certifi-2022.12.7.tar.gz", hash = "sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3"},
+]
 
 [[package]]
 name = "cffi"
@@ -207,6 +351,72 @@ description = "Foreign Function Interface for Python calling C code."
 category = "main"
 optional = false
 python-versions = "*"
+files = [
+    {file = "cffi-1.15.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:a66d3508133af6e8548451b25058d5812812ec3798c886bf38ed24a98216fab2"},
+    {file = "cffi-1.15.1-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:470c103ae716238bbe698d67ad020e1db9d9dba34fa5a899b5e21577e6d52ed2"},
+    {file = "cffi-1.15.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:9ad5db27f9cabae298d151c85cf2bad1d359a1b9c686a275df03385758e2f914"},
+    {file = "cffi-1.15.1-cp27-cp27m-win32.whl", hash = "sha256:b3bbeb01c2b273cca1e1e0c5df57f12dce9a4dd331b4fa1635b8bec26350bde3"},
+    {file = "cffi-1.15.1-cp27-cp27m-win_amd64.whl", hash = "sha256:e00b098126fd45523dd056d2efba6c5a63b71ffe9f2bbe1a4fe1716e1d0c331e"},
+    {file = "cffi-1.15.1-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:d61f4695e6c866a23a21acab0509af1cdfd2c013cf256bbf5b6b5e2695827162"},
+    {file = "cffi-1.15.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:ed9cb427ba5504c1dc15ede7d516b84757c3e3d7868ccc85121d9310d27eed0b"},
+    {file = "cffi-1.15.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:39d39875251ca8f612b6f33e6b1195af86d1b3e60086068be9cc053aa4376e21"},
+    {file = "cffi-1.15.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:285d29981935eb726a4399badae8f0ffdff4f5050eaa6d0cfc3f64b857b77185"},
+    {file = "cffi-1.15.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3eb6971dcff08619f8d91607cfc726518b6fa2a9eba42856be181c6d0d9515fd"},
+    {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:21157295583fe8943475029ed5abdcf71eb3911894724e360acff1d61c1d54bc"},
+    {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5635bd9cb9731e6d4a1132a498dd34f764034a8ce60cef4f5319c0541159392f"},
+    {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2012c72d854c2d03e45d06ae57f40d78e5770d252f195b93f581acf3ba44496e"},
+    {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd86c085fae2efd48ac91dd7ccffcfc0571387fe1193d33b6394db7ef31fe2a4"},
+    {file = "cffi-1.15.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:fa6693661a4c91757f4412306191b6dc88c1703f780c8234035eac011922bc01"},
+    {file = "cffi-1.15.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:59c0b02d0a6c384d453fece7566d1c7e6b7bae4fc5874ef2ef46d56776d61c9e"},
+    {file = "cffi-1.15.1-cp310-cp310-win32.whl", hash = "sha256:cba9d6b9a7d64d4bd46167096fc9d2f835e25d7e4c121fb2ddfc6528fb0413b2"},
+    {file = "cffi-1.15.1-cp310-cp310-win_amd64.whl", hash = "sha256:ce4bcc037df4fc5e3d184794f27bdaab018943698f4ca31630bc7f84a7b69c6d"},
+    {file = "cffi-1.15.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3d08afd128ddaa624a48cf2b859afef385b720bb4b43df214f85616922e6a5ac"},
+    {file = "cffi-1.15.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3799aecf2e17cf585d977b780ce79ff0dc9b78d799fc694221ce814c2c19db83"},
+    {file = "cffi-1.15.1-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a591fe9e525846e4d154205572a029f653ada1a78b93697f3b5a8f1f2bc055b9"},
+    {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3548db281cd7d2561c9ad9984681c95f7b0e38881201e157833a2342c30d5e8c"},
+    {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:91fc98adde3d7881af9b59ed0294046f3806221863722ba7d8d120c575314325"},
+    {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:94411f22c3985acaec6f83c6df553f2dbe17b698cc7f8ae751ff2237d96b9e3c"},
+    {file = "cffi-1.15.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:03425bdae262c76aad70202debd780501fabeaca237cdfddc008987c0e0f59ef"},
+    {file = "cffi-1.15.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:cc4d65aeeaa04136a12677d3dd0b1c0c94dc43abac5860ab33cceb42b801c1e8"},
+    {file = "cffi-1.15.1-cp311-cp311-win32.whl", hash = "sha256:a0f100c8912c114ff53e1202d0078b425bee3649ae34d7b070e9697f93c5d52d"},
+    {file = "cffi-1.15.1-cp311-cp311-win_amd64.whl", hash = "sha256:04ed324bda3cda42b9b695d51bb7d54b680b9719cfab04227cdd1e04e5de3104"},
+    {file = "cffi-1.15.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50a74364d85fd319352182ef59c5c790484a336f6db772c1a9231f1c3ed0cbd7"},
+    {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e263d77ee3dd201c3a142934a086a4450861778baaeeb45db4591ef65550b0a6"},
+    {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cec7d9412a9102bdc577382c3929b337320c4c4c4849f2c5cdd14d7368c5562d"},
+    {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4289fc34b2f5316fbb762d75362931e351941fa95fa18789191b33fc4cf9504a"},
+    {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:173379135477dc8cac4bc58f45db08ab45d228b3363adb7af79436135d028405"},
+    {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:6975a3fac6bc83c4a65c9f9fcab9e47019a11d3d2cf7f3c0d03431bf145a941e"},
+    {file = "cffi-1.15.1-cp36-cp36m-win32.whl", hash = "sha256:2470043b93ff09bf8fb1d46d1cb756ce6132c54826661a32d4e4d132e1977adf"},
+    {file = "cffi-1.15.1-cp36-cp36m-win_amd64.whl", hash = "sha256:30d78fbc8ebf9c92c9b7823ee18eb92f2e6ef79b45ac84db507f52fbe3ec4497"},
+    {file = "cffi-1.15.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:198caafb44239b60e252492445da556afafc7d1e3ab7a1fb3f0584ef6d742375"},
+    {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5ef34d190326c3b1f822a5b7a45f6c4535e2f47ed06fec77d3d799c450b2651e"},
+    {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8102eaf27e1e448db915d08afa8b41d6c7ca7a04b7d73af6514df10a3e74bd82"},
+    {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5df2768244d19ab7f60546d0c7c63ce1581f7af8b5de3eb3004b9b6fc8a9f84b"},
+    {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a8c4917bd7ad33e8eb21e9a5bbba979b49d9a97acb3a803092cbc1133e20343c"},
+    {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e2642fe3142e4cc4af0799748233ad6da94c62a8bec3a6648bf8ee68b1c7426"},
+    {file = "cffi-1.15.1-cp37-cp37m-win32.whl", hash = "sha256:e229a521186c75c8ad9490854fd8bbdd9a0c9aa3a524326b55be83b54d4e0ad9"},
+    {file = "cffi-1.15.1-cp37-cp37m-win_amd64.whl", hash = "sha256:a0b71b1b8fbf2b96e41c4d990244165e2c9be83d54962a9a1d118fd8657d2045"},
+    {file = "cffi-1.15.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:320dab6e7cb2eacdf0e658569d2575c4dad258c0fcc794f46215e1e39f90f2c3"},
+    {file = "cffi-1.15.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e74c6b51a9ed6589199c787bf5f9875612ca4a8a0785fb2d4a84429badaf22a"},
+    {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a5c84c68147988265e60416b57fc83425a78058853509c1b0629c180094904a5"},
+    {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3b926aa83d1edb5aa5b427b4053dc420ec295a08e40911296b9eb1b6170f6cca"},
+    {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:87c450779d0914f2861b8526e035c5e6da0a3199d8f1add1a665e1cbc6fc6d02"},
+    {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f2c9f67e9821cad2e5f480bc8d83b8742896f1242dba247911072d4fa94c192"},
+    {file = "cffi-1.15.1-cp38-cp38-win32.whl", hash = "sha256:8b7ee99e510d7b66cdb6c593f21c043c248537a32e0bedf02e01e9553a172314"},
+    {file = "cffi-1.15.1-cp38-cp38-win_amd64.whl", hash = "sha256:00a9ed42e88df81ffae7a8ab6d9356b371399b91dbdf0c3cb1e84c03a13aceb5"},
+    {file = "cffi-1.15.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:54a2db7b78338edd780e7ef7f9f6c442500fb0d41a5a4ea24fff1c929d5af585"},
+    {file = "cffi-1.15.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:fcd131dd944808b5bdb38e6f5b53013c5aa4f334c5cad0c72742f6eba4b73db0"},
+    {file = "cffi-1.15.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7473e861101c9e72452f9bf8acb984947aa1661a7704553a9f6e4baa5ba64415"},
+    {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6c9a799e985904922a4d207a94eae35c78ebae90e128f0c4e521ce339396be9d"},
+    {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3bcde07039e586f91b45c88f8583ea7cf7a0770df3a1649627bf598332cb6984"},
+    {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:33ab79603146aace82c2427da5ca6e58f2b3f2fb5da893ceac0c42218a40be35"},
+    {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5d598b938678ebf3c67377cdd45e09d431369c3b1a5b331058c338e201f12b27"},
+    {file = "cffi-1.15.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db0fbb9c62743ce59a9ff687eb5f4afbe77e5e8403d6697f7446e5f609976f76"},
+    {file = "cffi-1.15.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:98d85c6a2bef81588d9227dde12db8a7f47f639f4a17c9ae08e773aa9c697bf3"},
+    {file = "cffi-1.15.1-cp39-cp39-win32.whl", hash = "sha256:40f4774f5a9d4f5e344f31a32b5096977b5d48560c5592e2f3d2c4374bd543ee"},
+    {file = "cffi-1.15.1-cp39-cp39-win_amd64.whl", hash = "sha256:70df4e3b545a17496c9b3f41f5115e69a4f2e77e94e1d2a8e1070bc0c38c8a3c"},
+    {file = "cffi-1.15.1.tar.gz", hash = "sha256:d400bfb9a37b1351253cb402671cea7e89bdecc294e8016a707f6d1d8ac934f9"},
+]
 
 [package.dependencies]
 pycparser = "*"
@@ -218,6 +428,10 @@ description = "Validate configuration and produce human readable error messages.
 category = "main"
 optional = false
 python-versions = ">=3.6.1"
+files = [
+    {file = "cfgv-3.3.1-py2.py3-none-any.whl", hash = "sha256:c6a0883f3917a037485059700b9e75da2464e6c27051014ad85ba6aaa5884426"},
+    {file = "cfgv-3.3.1.tar.gz", hash = "sha256:f5a830efb9ce7a445376bb66ec94c638a9787422f96264c98edc6bdeed8ab736"},
+]
 
 [[package]]
 name = "charset-normalizer"
@@ -226,9 +440,13 @@ description = "The Real First Universal Charset Detector. Open, modern and activ
 category = "main"
 optional = false
 python-versions = ">=3.6.0"
+files = [
+    {file = "charset-normalizer-2.1.1.tar.gz", hash = "sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845"},
+    {file = "charset_normalizer-2.1.1-py3-none-any.whl", hash = "sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f"},
+]
 
 [package.extras]
-unicode_backport = ["unicodedata2"]
+unicode-backport = ["unicodedata2"]
 
 [[package]]
 name = "click"
@@ -237,6 +455,10 @@ description = "Composable command line interface toolkit"
 category = "main"
 optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
+files = [
+    {file = "click-7.1.2-py2.py3-none-any.whl", hash = "sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc"},
+    {file = "click-7.1.2.tar.gz", hash = "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a"},
+]
 
 [[package]]
 name = "colorama"
@@ -245,6 +467,10 @@ description = "Cross-platform colored terminal text."
 category = "main"
 optional = false
 python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7"
+files = [
+    {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"},
+    {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"},
+]
 
 [[package]]
 name = "coverage"
@@ -253,6 +479,59 @@ description = "Code coverage measurement for Python"
 category = "dev"
 optional = false
 python-versions = ">=3.7"
+files = [
+    {file = "coverage-7.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3b946bbcd5a8231383450b195cfb58cb01cbe7f8949f5758566b881df4b33baf"},
+    {file = "coverage-7.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ec8e767f13be637d056f7e07e61d089e555f719b387a7070154ad80a0ff31801"},
+    {file = "coverage-7.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d4a5a5879a939cb84959d86869132b00176197ca561c664fc21478c1eee60d75"},
+    {file = "coverage-7.1.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b643cb30821e7570c0aaf54feaf0bfb630b79059f85741843e9dc23f33aaca2c"},
+    {file = "coverage-7.1.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:32df215215f3af2c1617a55dbdfb403b772d463d54d219985ac7cd3bf124cada"},
+    {file = "coverage-7.1.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:33d1ae9d4079e05ac4cc1ef9e20c648f5afabf1a92adfaf2ccf509c50b85717f"},
+    {file = "coverage-7.1.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:29571503c37f2ef2138a306d23e7270687c0efb9cab4bd8038d609b5c2393a3a"},
+    {file = "coverage-7.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:63ffd21aa133ff48c4dff7adcc46b7ec8b565491bfc371212122dd999812ea1c"},
+    {file = "coverage-7.1.0-cp310-cp310-win32.whl", hash = "sha256:4b14d5e09c656de5038a3f9bfe5228f53439282abcab87317c9f7f1acb280352"},
+    {file = "coverage-7.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:8361be1c2c073919500b6601220a6f2f98ea0b6d2fec5014c1d9cfa23dd07038"},
+    {file = "coverage-7.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:da9b41d4539eefd408c46725fb76ecba3a50a3367cafb7dea5f250d0653c1040"},
+    {file = "coverage-7.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c5b15ed7644ae4bee0ecf74fee95808dcc34ba6ace87e8dfbf5cb0dc20eab45a"},
+    {file = "coverage-7.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d12d076582507ea460ea2a89a8c85cb558f83406c8a41dd641d7be9a32e1274f"},
+    {file = "coverage-7.1.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e2617759031dae1bf183c16cef8fcfb3de7617f394c813fa5e8e46e9b82d4222"},
+    {file = "coverage-7.1.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c4e4881fa9e9667afcc742f0c244d9364d197490fbc91d12ac3b5de0bf2df146"},
+    {file = "coverage-7.1.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:9d58885215094ab4a86a6aef044e42994a2bd76a446dc59b352622655ba6621b"},
+    {file = "coverage-7.1.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:ffeeb38ee4a80a30a6877c5c4c359e5498eec095878f1581453202bfacc8fbc2"},
+    {file = "coverage-7.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3baf5f126f30781b5e93dbefcc8271cb2491647f8283f20ac54d12161dff080e"},
+    {file = "coverage-7.1.0-cp311-cp311-win32.whl", hash = "sha256:ded59300d6330be27bc6cf0b74b89ada58069ced87c48eaf9344e5e84b0072f7"},
+    {file = "coverage-7.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:6a43c7823cd7427b4ed763aa7fb63901ca8288591323b58c9cd6ec31ad910f3c"},
+    {file = "coverage-7.1.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:7a726d742816cb3a8973c8c9a97539c734b3a309345236cd533c4883dda05b8d"},
+    {file = "coverage-7.1.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bc7c85a150501286f8b56bd8ed3aa4093f4b88fb68c0843d21ff9656f0009d6a"},
+    {file = "coverage-7.1.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f5b4198d85a3755d27e64c52f8c95d6333119e49fd001ae5798dac872c95e0f8"},
+    {file = "coverage-7.1.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ddb726cb861c3117a553f940372a495fe1078249ff5f8a5478c0576c7be12050"},
+    {file = "coverage-7.1.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:51b236e764840a6df0661b67e50697aaa0e7d4124ca95e5058fa3d7cbc240b7c"},
+    {file = "coverage-7.1.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:7ee5c9bb51695f80878faaa5598040dd6c9e172ddcf490382e8aedb8ec3fec8d"},
+    {file = "coverage-7.1.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:c31b75ae466c053a98bf26843563b3b3517b8f37da4d47b1c582fdc703112bc3"},
+    {file = "coverage-7.1.0-cp37-cp37m-win32.whl", hash = "sha256:3b155caf3760408d1cb903b21e6a97ad4e2bdad43cbc265e3ce0afb8e0057e73"},
+    {file = "coverage-7.1.0-cp37-cp37m-win_amd64.whl", hash = "sha256:2a60d6513781e87047c3e630b33b4d1e89f39836dac6e069ffee28c4786715f5"},
+    {file = "coverage-7.1.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:f2cba5c6db29ce991029b5e4ac51eb36774458f0a3b8d3137241b32d1bb91f06"},
+    {file = "coverage-7.1.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:beeb129cacea34490ffd4d6153af70509aa3cda20fdda2ea1a2be870dfec8d52"},
+    {file = "coverage-7.1.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0c45948f613d5d18c9ec5eaa203ce06a653334cf1bd47c783a12d0dd4fd9c851"},
+    {file = "coverage-7.1.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef382417db92ba23dfb5864a3fc9be27ea4894e86620d342a116b243ade5d35d"},
+    {file = "coverage-7.1.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7c7c0d0827e853315c9bbd43c1162c006dd808dbbe297db7ae66cd17b07830f0"},
+    {file = "coverage-7.1.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:e5cdbb5cafcedea04924568d990e20ce7f1945a1dd54b560f879ee2d57226912"},
+    {file = "coverage-7.1.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:9817733f0d3ea91bea80de0f79ef971ae94f81ca52f9b66500c6a2fea8e4b4f8"},
+    {file = "coverage-7.1.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:218fe982371ac7387304153ecd51205f14e9d731b34fb0568181abaf7b443ba0"},
+    {file = "coverage-7.1.0-cp38-cp38-win32.whl", hash = "sha256:04481245ef966fbd24ae9b9e537ce899ae584d521dfbe78f89cad003c38ca2ab"},
+    {file = "coverage-7.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:8ae125d1134bf236acba8b83e74c603d1b30e207266121e76484562bc816344c"},
+    {file = "coverage-7.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:2bf1d5f2084c3932b56b962a683074a3692bce7cabd3aa023c987a2a8e7612f6"},
+    {file = "coverage-7.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:98b85dd86514d889a2e3dd22ab3c18c9d0019e696478391d86708b805f4ea0fa"},
+    {file = "coverage-7.1.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:38da2db80cc505a611938d8624801158e409928b136c8916cd2e203970dde4dc"},
+    {file = "coverage-7.1.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3164d31078fa9efe406e198aecd2a02d32a62fecbdef74f76dad6a46c7e48311"},
+    {file = "coverage-7.1.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:db61a79c07331e88b9a9974815c075fbd812bc9dbc4dc44b366b5368a2936063"},
+    {file = "coverage-7.1.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:9ccb092c9ede70b2517a57382a601619d20981f56f440eae7e4d7eaafd1d1d09"},
+    {file = "coverage-7.1.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:33ff26d0f6cc3ca8de13d14fde1ff8efe1456b53e3f0273e63cc8b3c84a063d8"},
+    {file = "coverage-7.1.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:d47dd659a4ee952e90dc56c97d78132573dc5c7b09d61b416a9deef4ebe01a0c"},
+    {file = "coverage-7.1.0-cp39-cp39-win32.whl", hash = "sha256:d248cd4a92065a4d4543b8331660121b31c4148dd00a691bfb7a5cdc7483cfa4"},
+    {file = "coverage-7.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:7ed681b0f8e8bcbbffa58ba26fcf5dbc8f79e7997595bf071ed5430d8c08d6f3"},
+    {file = "coverage-7.1.0-pp37.pp38.pp39-none-any.whl", hash = "sha256:755e89e32376c850f826c425ece2c35a4fc266c081490eb0a841e7c1cb0d3bda"},
+    {file = "coverage-7.1.0.tar.gz", hash = "sha256:10188fe543560ec4874f974b5305cd1a8bdcfa885ee00ea3a03733464c4ca265"},
+]
 
 [package.dependencies]
 tomli = {version = "*", optional = true, markers = "python_full_version <= \"3.11.0a6\" and extra == \"toml\""}
@@ -267,17 +546,41 @@ description = "cryptography is a package which provides cryptographic recipes an
 category = "main"
 optional = false
 python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*"
+files = [
+    {file = "cryptography-3.1.1-cp27-cp27m-macosx_10_10_x86_64.whl", hash = "sha256:65beb15e7f9c16e15934569d29fb4def74ea1469d8781f6b3507ab896d6d8719"},
+    {file = "cryptography-3.1.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:983c0c3de4cb9fcba68fd3f45ed846eb86a2a8b8d8bc5bb18364c4d00b3c61fe"},
+    {file = "cryptography-3.1.1-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:e97a3b627e3cb63c415a16245d6cef2139cca18bb1183d1b9375a1c14e83f3b3"},
+    {file = "cryptography-3.1.1-cp27-cp27m-win32.whl", hash = "sha256:cb179acdd4ae1e4a5a160d80b87841b3d0e0be84af46c7bb2cd7ece57a39c4ba"},
+    {file = "cryptography-3.1.1-cp27-cp27m-win_amd64.whl", hash = "sha256:b372026ebf32fe2523159f27d9f0e9f485092e43b00a5adacf732192a70ba118"},
+    {file = "cryptography-3.1.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:680da076cad81cdf5ffcac50c477b6790be81768d30f9da9e01960c4b18a66db"},
+    {file = "cryptography-3.1.1-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:5d52c72449bb02dd45a773a203196e6d4fae34e158769c896012401f33064396"},
+    {file = "cryptography-3.1.1-cp35-abi3-macosx_10_10_x86_64.whl", hash = "sha256:f0e099fc4cc697450c3dd4031791559692dd941a95254cb9aeded66a7aa8b9bc"},
+    {file = "cryptography-3.1.1-cp35-abi3-manylinux1_x86_64.whl", hash = "sha256:a7597ffc67987b37b12e09c029bd1dc43965f75d328076ae85721b84046e9ca7"},
+    {file = "cryptography-3.1.1-cp35-abi3-manylinux2010_x86_64.whl", hash = "sha256:4549b137d8cbe3c2eadfa56c0c858b78acbeff956bd461e40000b2164d9167c6"},
+    {file = "cryptography-3.1.1-cp35-abi3-manylinux2014_aarch64.whl", hash = "sha256:89aceb31cd5f9fc2449fe8cf3810797ca52b65f1489002d58fe190bfb265c536"},
+    {file = "cryptography-3.1.1-cp35-cp35m-win32.whl", hash = "sha256:559d622aef2a2dff98a892eef321433ba5bc55b2485220a8ca289c1ecc2bd54f"},
+    {file = "cryptography-3.1.1-cp35-cp35m-win_amd64.whl", hash = "sha256:451cdf60be4dafb6a3b78802006a020e6cd709c22d240f94f7a0696240a17154"},
+    {file = "cryptography-3.1.1-cp36-abi3-win32.whl", hash = "sha256:762bc5a0df03c51ee3f09c621e1cee64e3a079a2b5020de82f1613873d79ee70"},
+    {file = "cryptography-3.1.1-cp36-abi3-win_amd64.whl", hash = "sha256:b12e715c10a13ca1bd27fbceed9adc8c5ff640f8e1f7ea76416352de703523c8"},
+    {file = "cryptography-3.1.1-cp36-cp36m-win32.whl", hash = "sha256:21b47c59fcb1c36f1113f3709d37935368e34815ea1d7073862e92f810dc7499"},
+    {file = "cryptography-3.1.1-cp36-cp36m-win_amd64.whl", hash = "sha256:48ee615a779ffa749d7d50c291761dc921d93d7cf203dca2db663b4f193f0e49"},
+    {file = "cryptography-3.1.1-cp37-cp37m-win32.whl", hash = "sha256:b2bded09c578d19e08bd2c5bb8fed7f103e089752c9cf7ca7ca7de522326e921"},
+    {file = "cryptography-3.1.1-cp37-cp37m-win_amd64.whl", hash = "sha256:f99317a0fa2e49917689b8cf977510addcfaaab769b3f899b9c481bbd76730c2"},
+    {file = "cryptography-3.1.1-cp38-cp38-win32.whl", hash = "sha256:ab010e461bb6b444eaf7f8c813bb716be2d78ab786103f9608ffd37a4bd7d490"},
+    {file = "cryptography-3.1.1-cp38-cp38-win_amd64.whl", hash = "sha256:99d4984aabd4c7182050bca76176ce2dbc9fa9748afe583a7865c12954d714ba"},
+    {file = "cryptography-3.1.1.tar.gz", hash = "sha256:9d9fc6a16357965d282dd4ab6531013935425d0dc4950df2e0cf2a1b1ac1017d"},
+]
 
 [package.dependencies]
 cffi = ">=1.8,<1.11.3 || >1.11.3"
 six = ">=1.4.1"
 
 [package.extras]
-docs = ["sphinx (>=1.6.5,!=1.8.0,!=3.1.0,!=3.1.1)", "sphinx-rtd-theme"]
-docstest = ["doc8", "pyenchant (>=1.6.11)", "twine (>=1.12.0)", "sphinxcontrib-spelling (>=4.0.1)"]
+docs = ["sphinx (>=1.6.5,!=1.8.0,!=3.1.0,!=3.1.1)", "sphinx_rtd_theme"]
+docstest = ["doc8", "pyenchant (>=1.6.11)", "sphinxcontrib-spelling (>=4.0.1)", "twine (>=1.12.0)"]
 pep8test = ["black", "flake8", "flake8-import-order", "pep8-naming"]
 ssh = ["bcrypt (>=3.1.5)"]
-test = ["pytest (>=3.6.0,!=3.9.0,!=3.9.1,!=3.9.2)", "pretend", "iso8601", "pytz", "hypothesis (>=1.11.4,!=3.79.2)"]
+test = ["hypothesis (>=1.11.4,!=3.79.2)", "iso8601", "pretend", "pytest (>=3.6.0,!=3.9.0,!=3.9.1,!=3.9.2)", "pytz"]
 
 [[package]]
 name = "distlib"
@@ -286,6 +589,10 @@ description = "Distribution utilities"
 category = "main"
 optional = false
 python-versions = "*"
+files = [
+    {file = "distlib-0.3.6-py2.py3-none-any.whl", hash = "sha256:f35c4b692542ca110de7ef0bea44d73981caeb34ca0b9b6b2e6d7790dda8f80e"},
+    {file = "distlib-0.3.6.tar.gz", hash = "sha256:14bad2d9b04d3a36127ac97f30b12a19268f211063d8f8ee4f47108896e11b46"},
+]
 
 [[package]]
 name = "filelock"
@@ -294,10 +601,14 @@ description = "A platform independent file lock."
 category = "main"
 optional = false
 python-versions = ">=3.7"
+files = [
+    {file = "filelock-3.9.0-py3-none-any.whl", hash = "sha256:f58d535af89bb9ad5cd4df046f741f8553a418c01a7856bf0d173bbc9f6bd16d"},
+    {file = "filelock-3.9.0.tar.gz", hash = "sha256:7b319f24340b51f55a2bf7a12ac0755a9b03e718311dac567a0f4f7fabd2f5de"},
+]
 
 [package.extras]
-docs = ["furo (>=2022.12.7)", "sphinx-autodoc-typehints (>=1.19.5)", "sphinx (>=5.3)"]
-testing = ["covdefaults (>=2.2.2)", "coverage (>=7.0.1)", "pytest-cov (>=4)", "pytest-timeout (>=2.1)", "pytest (>=7.2)"]
+docs = ["furo (>=2022.12.7)", "sphinx (>=5.3)", "sphinx-autodoc-typehints (>=1.19.5)"]
+testing = ["covdefaults (>=2.2.2)", "coverage (>=7.0.1)", "pytest (>=7.2)", "pytest-cov (>=4)", "pytest-timeout (>=2.1)"]
 
 [[package]]
 name = "frozenlist"
@@ -306,6 +617,82 @@ description = "A list-like structure which implements collections.abc.MutableSeq
 category = "main"
 optional = false
 python-versions = ">=3.7"
+files = [
+    {file = "frozenlist-1.3.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ff8bf625fe85e119553b5383ba0fb6aa3d0ec2ae980295aaefa552374926b3f4"},
+    {file = "frozenlist-1.3.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:dfbac4c2dfcc082fcf8d942d1e49b6aa0766c19d3358bd86e2000bf0fa4a9cf0"},
+    {file = "frozenlist-1.3.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b1c63e8d377d039ac769cd0926558bb7068a1f7abb0f003e3717ee003ad85530"},
+    {file = "frozenlist-1.3.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7fdfc24dcfce5b48109867c13b4cb15e4660e7bd7661741a391f821f23dfdca7"},
+    {file = "frozenlist-1.3.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2c926450857408e42f0bbc295e84395722ce74bae69a3b2aa2a65fe22cb14b99"},
+    {file = "frozenlist-1.3.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1841e200fdafc3d51f974d9d377c079a0694a8f06de2e67b48150328d66d5483"},
+    {file = "frozenlist-1.3.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f470c92737afa7d4c3aacc001e335062d582053d4dbe73cda126f2d7031068dd"},
+    {file = "frozenlist-1.3.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:783263a4eaad7c49983fe4b2e7b53fa9770c136c270d2d4bbb6d2192bf4d9caf"},
+    {file = "frozenlist-1.3.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:924620eef691990dfb56dc4709f280f40baee568c794b5c1885800c3ecc69816"},
+    {file = "frozenlist-1.3.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:ae4dc05c465a08a866b7a1baf360747078b362e6a6dbeb0c57f234db0ef88ae0"},
+    {file = "frozenlist-1.3.3-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:bed331fe18f58d844d39ceb398b77d6ac0b010d571cba8267c2e7165806b00ce"},
+    {file = "frozenlist-1.3.3-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:02c9ac843e3390826a265e331105efeab489ffaf4dd86384595ee8ce6d35ae7f"},
+    {file = "frozenlist-1.3.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:9545a33965d0d377b0bc823dcabf26980e77f1b6a7caa368a365a9497fb09420"},
+    {file = "frozenlist-1.3.3-cp310-cp310-win32.whl", hash = "sha256:d5cd3ab21acbdb414bb6c31958d7b06b85eeb40f66463c264a9b343a4e238642"},
+    {file = "frozenlist-1.3.3-cp310-cp310-win_amd64.whl", hash = "sha256:b756072364347cb6aa5b60f9bc18e94b2f79632de3b0190253ad770c5df17db1"},
+    {file = "frozenlist-1.3.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:b4395e2f8d83fbe0c627b2b696acce67868793d7d9750e90e39592b3626691b7"},
+    {file = "frozenlist-1.3.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:14143ae966a6229350021384870458e4777d1eae4c28d1a7aa47f24d030e6678"},
+    {file = "frozenlist-1.3.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5d8860749e813a6f65bad8285a0520607c9500caa23fea6ee407e63debcdbef6"},
+    {file = "frozenlist-1.3.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:23d16d9f477bb55b6154654e0e74557040575d9d19fe78a161bd33d7d76808e8"},
+    {file = "frozenlist-1.3.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:eb82dbba47a8318e75f679690190c10a5e1f447fbf9df41cbc4c3afd726d88cb"},
+    {file = "frozenlist-1.3.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9309869032abb23d196cb4e4db574232abe8b8be1339026f489eeb34a4acfd91"},
+    {file = "frozenlist-1.3.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a97b4fe50b5890d36300820abd305694cb865ddb7885049587a5678215782a6b"},
+    {file = "frozenlist-1.3.3-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c188512b43542b1e91cadc3c6c915a82a5eb95929134faf7fd109f14f9892ce4"},
+    {file = "frozenlist-1.3.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:303e04d422e9b911a09ad499b0368dc551e8c3cd15293c99160c7f1f07b59a48"},
+    {file = "frozenlist-1.3.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:0771aed7f596c7d73444c847a1c16288937ef988dc04fb9f7be4b2aa91db609d"},
+    {file = "frozenlist-1.3.3-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:66080ec69883597e4d026f2f71a231a1ee9887835902dbe6b6467d5a89216cf6"},
+    {file = "frozenlist-1.3.3-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:41fe21dc74ad3a779c3d73a2786bdf622ea81234bdd4faf90b8b03cad0c2c0b4"},
+    {file = "frozenlist-1.3.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:f20380df709d91525e4bee04746ba612a4df0972c1b8f8e1e8af997e678c7b81"},
+    {file = "frozenlist-1.3.3-cp311-cp311-win32.whl", hash = "sha256:f30f1928162e189091cf4d9da2eac617bfe78ef907a761614ff577ef4edfb3c8"},
+    {file = "frozenlist-1.3.3-cp311-cp311-win_amd64.whl", hash = "sha256:a6394d7dadd3cfe3f4b3b186e54d5d8504d44f2d58dcc89d693698e8b7132b32"},
+    {file = "frozenlist-1.3.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8df3de3a9ab8325f94f646609a66cbeeede263910c5c0de0101079ad541af332"},
+    {file = "frozenlist-1.3.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0693c609e9742c66ba4870bcee1ad5ff35462d5ffec18710b4ac89337ff16e27"},
+    {file = "frozenlist-1.3.3-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cd4210baef299717db0a600d7a3cac81d46ef0e007f88c9335db79f8979c0d3d"},
+    {file = "frozenlist-1.3.3-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:394c9c242113bfb4b9aa36e2b80a05ffa163a30691c7b5a29eba82e937895d5e"},
+    {file = "frozenlist-1.3.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6327eb8e419f7d9c38f333cde41b9ae348bec26d840927332f17e887a8dcb70d"},
+    {file = "frozenlist-1.3.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2e24900aa13212e75e5b366cb9065e78bbf3893d4baab6052d1aca10d46d944c"},
+    {file = "frozenlist-1.3.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:3843f84a6c465a36559161e6c59dce2f2ac10943040c2fd021cfb70d58c4ad56"},
+    {file = "frozenlist-1.3.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:84610c1502b2461255b4c9b7d5e9c48052601a8957cd0aea6ec7a7a1e1fb9420"},
+    {file = "frozenlist-1.3.3-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:c21b9aa40e08e4f63a2f92ff3748e6b6c84d717d033c7b3438dd3123ee18f70e"},
+    {file = "frozenlist-1.3.3-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:efce6ae830831ab6a22b9b4091d411698145cb9b8fc869e1397ccf4b4b6455cb"},
+    {file = "frozenlist-1.3.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:40de71985e9042ca00b7953c4f41eabc3dc514a2d1ff534027f091bc74416401"},
+    {file = "frozenlist-1.3.3-cp37-cp37m-win32.whl", hash = "sha256:180c00c66bde6146a860cbb81b54ee0df350d2daf13ca85b275123bbf85de18a"},
+    {file = "frozenlist-1.3.3-cp37-cp37m-win_amd64.whl", hash = "sha256:9bbbcedd75acdfecf2159663b87f1bb5cfc80e7cd99f7ddd9d66eb98b14a8411"},
+    {file = "frozenlist-1.3.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:034a5c08d36649591be1cbb10e09da9f531034acfe29275fc5454a3b101ce41a"},
+    {file = "frozenlist-1.3.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:ba64dc2b3b7b158c6660d49cdb1d872d1d0bf4e42043ad8d5006099479a194e5"},
+    {file = "frozenlist-1.3.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:47df36a9fe24054b950bbc2db630d508cca3aa27ed0566c0baf661225e52c18e"},
+    {file = "frozenlist-1.3.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:008a054b75d77c995ea26629ab3a0c0d7281341f2fa7e1e85fa6153ae29ae99c"},
+    {file = "frozenlist-1.3.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:841ea19b43d438a80b4de62ac6ab21cfe6827bb8a9dc62b896acc88eaf9cecba"},
+    {file = "frozenlist-1.3.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e235688f42b36be2b6b06fc37ac2126a73b75fb8d6bc66dd632aa35286238703"},
+    {file = "frozenlist-1.3.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ca713d4af15bae6e5d79b15c10c8522859a9a89d3b361a50b817c98c2fb402a2"},
+    {file = "frozenlist-1.3.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9ac5995f2b408017b0be26d4a1d7c61bce106ff3d9e3324374d66b5964325448"},
+    {file = "frozenlist-1.3.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:a4ae8135b11652b08a8baf07631d3ebfe65a4c87909dbef5fa0cdde440444ee4"},
+    {file = "frozenlist-1.3.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:4ea42116ceb6bb16dbb7d526e242cb6747b08b7710d9782aa3d6732bd8d27649"},
+    {file = "frozenlist-1.3.3-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:810860bb4bdce7557bc0febb84bbd88198b9dbc2022d8eebe5b3590b2ad6c842"},
+    {file = "frozenlist-1.3.3-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:ee78feb9d293c323b59a6f2dd441b63339a30edf35abcb51187d2fc26e696d13"},
+    {file = "frozenlist-1.3.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:0af2e7c87d35b38732e810befb9d797a99279cbb85374d42ea61c1e9d23094b3"},
+    {file = "frozenlist-1.3.3-cp38-cp38-win32.whl", hash = "sha256:899c5e1928eec13fd6f6d8dc51be23f0d09c5281e40d9cf4273d188d9feeaf9b"},
+    {file = "frozenlist-1.3.3-cp38-cp38-win_amd64.whl", hash = "sha256:7f44e24fa70f6fbc74aeec3e971f60a14dde85da364aa87f15d1be94ae75aeef"},
+    {file = "frozenlist-1.3.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:2b07ae0c1edaa0a36339ec6cce700f51b14a3fc6545fdd32930d2c83917332cf"},
+    {file = "frozenlist-1.3.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:ebb86518203e12e96af765ee89034a1dbb0c3c65052d1b0c19bbbd6af8a145e1"},
+    {file = "frozenlist-1.3.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:5cf820485f1b4c91e0417ea0afd41ce5cf5965011b3c22c400f6d144296ccbc0"},
+    {file = "frozenlist-1.3.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5c11e43016b9024240212d2a65043b70ed8dfd3b52678a1271972702d990ac6d"},
+    {file = "frozenlist-1.3.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8fa3c6e3305aa1146b59a09b32b2e04074945ffcfb2f0931836d103a2c38f936"},
+    {file = "frozenlist-1.3.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:352bd4c8c72d508778cf05ab491f6ef36149f4d0cb3c56b1b4302852255d05d5"},
+    {file = "frozenlist-1.3.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:65a5e4d3aa679610ac6e3569e865425b23b372277f89b5ef06cf2cdaf1ebf22b"},
+    {file = "frozenlist-1.3.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b1e2c1185858d7e10ff045c496bbf90ae752c28b365fef2c09cf0fa309291669"},
+    {file = "frozenlist-1.3.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:f163d2fd041c630fed01bc48d28c3ed4a3b003c00acd396900e11ee5316b56bb"},
+    {file = "frozenlist-1.3.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:05cdb16d09a0832eedf770cb7bd1fe57d8cf4eaf5aced29c4e41e3f20b30a784"},
+    {file = "frozenlist-1.3.3-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:8bae29d60768bfa8fb92244b74502b18fae55a80eac13c88eb0b496d4268fd2d"},
+    {file = "frozenlist-1.3.3-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:eedab4c310c0299961ac285591acd53dc6723a1ebd90a57207c71f6e0c2153ab"},
+    {file = "frozenlist-1.3.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:3bbdf44855ed8f0fbcd102ef05ec3012d6a4fd7c7562403f76ce6a52aeffb2b1"},
+    {file = "frozenlist-1.3.3-cp39-cp39-win32.whl", hash = "sha256:efa568b885bca461f7c7b9e032655c0c143d305bf01c30caf6db2854a4532b38"},
+    {file = "frozenlist-1.3.3-cp39-cp39-win_amd64.whl", hash = "sha256:cfe33efc9cb900a4c46f91a5ceba26d6df370ffddd9ca386eb1d4f0ad97b9ea9"},
+    {file = "frozenlist-1.3.3.tar.gz", hash = "sha256:58bcc55721e8a90b88332d6cd441261ebb22342e238296bb330968952fbb3a6a"},
+]
 
 [[package]]
 name = "future"
@@ -314,6 +701,9 @@ description = "Clean single-source support for Python 3 and 2"
 category = "main"
 optional = false
 python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*"
+files = [
+    {file = "future-0.18.3.tar.gz", hash = "sha256:34a17436ed1e96697a86f9de3d15a3b0be01d8bc8de9c1dffd59fb8234ed5307"},
+]
 
 [[package]]
 name = "h11"
@@ -322,6 +712,10 @@ description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1"
 category = "main"
 optional = false
 python-versions = ">=3.6"
+files = [
+    {file = "h11-0.12.0-py3-none-any.whl", hash = "sha256:36a3cb8c0a032f56e2da7084577878a035d3b61d104230d4bd49c0c6b555a9c6"},
+    {file = "h11-0.12.0.tar.gz", hash = "sha256:47222cb6067e4a307d535814917cd98fd0a57b6788ce715755fa2b6c28b56042"},
+]
 
 [[package]]
 name = "httpcore"
@@ -330,6 +724,10 @@ description = "A minimal low-level HTTP client."
 category = "main"
 optional = false
 python-versions = ">=3.7"
+files = [
+    {file = "httpcore-0.15.0-py3-none-any.whl", hash = "sha256:1105b8b73c025f23ff7c36468e4432226cbb959176eab66864b8e31c4ee27fa6"},
+    {file = "httpcore-0.15.0.tar.gz", hash = "sha256:18b68ab86a3ccf3e7dc0f43598eaddcf472b602aba29f9aa6ab85fe2ada3980b"},
+]
 
 [package.dependencies]
 anyio = ">=3.0.0,<4.0.0"
@@ -348,6 +746,10 @@ description = "The next generation HTTP client."
 category = "main"
 optional = false
 python-versions = ">=3.7"
+files = [
+    {file = "httpx-0.23.0-py3-none-any.whl", hash = "sha256:42974f577483e1e932c3cdc3cd2303e883cbfba17fe228b0f63589764d7b9c4b"},
+    {file = "httpx-0.23.0.tar.gz", hash = "sha256:f28eac771ec9eb4866d3fb4ab65abd42d38c424739e80c08d8d20570de60b0ef"},
+]
 
 [package.dependencies]
 certifi = "*"
@@ -356,8 +758,8 @@ rfc3986 = {version = ">=1.3,<2", extras = ["idna2008"]}
 sniffio = "*"
 
 [package.extras]
-brotli = ["brotlicffi", "brotli"]
-cli = ["click (>=8.0.0,<9.0.0)", "rich (>=10,<13)", "pygments (>=2.0.0,<3.0.0)"]
+brotli = ["brotli", "brotlicffi"]
+cli = ["click (>=8.0.0,<9.0.0)", "pygments (>=2.0.0,<3.0.0)", "rich (>=10,<13)"]
 http2 = ["h2 (>=3,<5)"]
 socks = ["socksio (>=1.0.0,<2.0.0)"]
 
@@ -368,6 +770,10 @@ description = "File identification library for Python"
 category = "main"
 optional = false
 python-versions = ">=3.7"
+files = [
+    {file = "identify-2.5.15-py2.py3-none-any.whl", hash = "sha256:1f4b36c5f50f3f950864b2a047308743f064eaa6f6645da5e5c780d1c7125487"},
+    {file = "identify-2.5.15.tar.gz", hash = "sha256:c22aa206f47cc40486ecf585d27ad5f40adbfc494a3fa41dc3ed0499a23b123f"},
+]
 
 [package.extras]
 license = ["ukkonen"]
@@ -379,6 +785,10 @@ description = "Internationalized Domain Names in Applications (IDNA)"
 category = "main"
 optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
+files = [
+    {file = "idna-2.10-py2.py3-none-any.whl", hash = "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"},
+    {file = "idna-2.10.tar.gz", hash = "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6"},
+]
 
 [[package]]
 name = "importlib-metadata"
@@ -387,22 +797,30 @@ description = "Read metadata from Python packages"
 category = "main"
 optional = false
 python-versions = ">=3.6"
+files = [
+    {file = "importlib_metadata-4.2.0-py3-none-any.whl", hash = "sha256:057e92c15bc8d9e8109738a48db0ccb31b4d9d5cfbee5a8670879a30be66304b"},
+    {file = "importlib_metadata-4.2.0.tar.gz", hash = "sha256:b7e52a1f8dec14a75ea73e0891f3060099ca1d8e6a462a4dff11c3e119ea1b31"},
+]
 
 [package.dependencies]
 typing-extensions = {version = ">=3.6.4", markers = "python_version < \"3.8\""}
 zipp = ">=0.5"
 
 [package.extras]
-docs = ["sphinx", "jaraco.packaging (>=8.2)", "rst.linker (>=1.9)"]
-testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "packaging", "pep517", "pyfakefs", "flufl.flake8", "pytest-black (>=0.3.7)", "pytest-mypy", "importlib-resources (>=1.3)"]
+docs = ["jaraco.packaging (>=8.2)", "rst.linker (>=1.9)", "sphinx"]
+testing = ["flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pep517", "pyfakefs", "pytest (>=4.6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.0.1)", "pytest-flake8", "pytest-mypy"]
 
 [[package]]
 name = "iniconfig"
 version = "2.0.0"
 description = "brain-dead simple config-ini parsing"
-category = "dev"
+category = "main"
 optional = false
 python-versions = ">=3.7"
+files = [
+    {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"},
+    {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"},
+]
 
 [[package]]
 name = "jmespath"
@@ -411,6 +829,10 @@ description = "JSON Matching Expressions"
 category = "main"
 optional = false
 python-versions = ">=3.7"
+files = [
+    {file = "jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980"},
+    {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"},
+]
 
 [[package]]
 name = "macholib"
@@ -419,6 +841,10 @@ description = "Mach-O header analysis and editing"
 category = "main"
 optional = false
 python-versions = "*"
+files = [
+    {file = "macholib-1.16.2-py2.py3-none-any.whl", hash = "sha256:44c40f2cd7d6726af8fa6fe22549178d3a4dfecc35a9cd15ea916d9c83a688e0"},
+    {file = "macholib-1.16.2.tar.gz", hash = "sha256:557bbfa1bb255c20e9abafe7ed6cd8046b48d9525db2f9b77d3122a63a2a8bf8"},
+]
 
 [package.dependencies]
 altgraph = ">=0.17"
@@ -430,6 +856,10 @@ description = "MinIO Python SDK for Amazon S3 Compatible Cloud Storage"
 category = "main"
 optional = false
 python-versions = "*"
+files = [
+    {file = "minio-7.1.8-py3-none-any.whl", hash = "sha256:0feadaf4cfd8608ccaf17b092c799bbe4b9e0692f9c15f5e03c5ee21d85e8cdb"},
+    {file = "minio-7.1.8.tar.gz", hash = "sha256:c3fe5448ca281c88fe58f0486a73e0df6cdb05e8dbf72eb79e570e71125c1686"},
+]
 
 [package.dependencies]
 certifi = "*"
@@ -442,6 +872,82 @@ description = "multidict implementation"
 category = "main"
 optional = false
 python-versions = ">=3.7"
+files = [
+    {file = "multidict-6.0.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:0b1a97283e0c85772d613878028fec909f003993e1007eafa715b24b377cb9b8"},
+    {file = "multidict-6.0.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:eeb6dcc05e911516ae3d1f207d4b0520d07f54484c49dfc294d6e7d63b734171"},
+    {file = "multidict-6.0.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d6d635d5209b82a3492508cf5b365f3446afb65ae7ebd755e70e18f287b0adf7"},
+    {file = "multidict-6.0.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c048099e4c9e9d615545e2001d3d8a4380bd403e1a0578734e0d31703d1b0c0b"},
+    {file = "multidict-6.0.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ea20853c6dbbb53ed34cb4d080382169b6f4554d394015f1bef35e881bf83547"},
+    {file = "multidict-6.0.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:16d232d4e5396c2efbbf4f6d4df89bfa905eb0d4dc5b3549d872ab898451f569"},
+    {file = "multidict-6.0.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:36c63aaa167f6c6b04ef2c85704e93af16c11d20de1d133e39de6a0e84582a93"},
+    {file = "multidict-6.0.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:64bdf1086b6043bf519869678f5f2757f473dee970d7abf6da91ec00acb9cb98"},
+    {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:43644e38f42e3af682690876cff722d301ac585c5b9e1eacc013b7a3f7b696a0"},
+    {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:7582a1d1030e15422262de9f58711774e02fa80df0d1578995c76214f6954988"},
+    {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:ddff9c4e225a63a5afab9dd15590432c22e8057e1a9a13d28ed128ecf047bbdc"},
+    {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:ee2a1ece51b9b9e7752e742cfb661d2a29e7bcdba2d27e66e28a99f1890e4fa0"},
+    {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a2e4369eb3d47d2034032a26c7a80fcb21a2cb22e1173d761a162f11e562caa5"},
+    {file = "multidict-6.0.4-cp310-cp310-win32.whl", hash = "sha256:574b7eae1ab267e5f8285f0fe881f17efe4b98c39a40858247720935b893bba8"},
+    {file = "multidict-6.0.4-cp310-cp310-win_amd64.whl", hash = "sha256:4dcbb0906e38440fa3e325df2359ac6cb043df8e58c965bb45f4e406ecb162cc"},
+    {file = "multidict-6.0.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0dfad7a5a1e39c53ed00d2dd0c2e36aed4650936dc18fd9a1826a5ae1cad6f03"},
+    {file = "multidict-6.0.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:64da238a09d6039e3bd39bb3aee9c21a5e34f28bfa5aa22518581f910ff94af3"},
+    {file = "multidict-6.0.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ff959bee35038c4624250473988b24f846cbeb2c6639de3602c073f10410ceba"},
+    {file = "multidict-6.0.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:01a3a55bd90018c9c080fbb0b9f4891db37d148a0a18722b42f94694f8b6d4c9"},
+    {file = "multidict-6.0.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c5cb09abb18c1ea940fb99360ea0396f34d46566f157122c92dfa069d3e0e982"},
+    {file = "multidict-6.0.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:666daae833559deb2d609afa4490b85830ab0dfca811a98b70a205621a6109fe"},
+    {file = "multidict-6.0.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:11bdf3f5e1518b24530b8241529d2050014c884cf18b6fc69c0c2b30ca248710"},
+    {file = "multidict-6.0.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7d18748f2d30f94f498e852c67d61261c643b349b9d2a581131725595c45ec6c"},
+    {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:458f37be2d9e4c95e2d8866a851663cbc76e865b78395090786f6cd9b3bbf4f4"},
+    {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:b1a2eeedcead3a41694130495593a559a668f382eee0727352b9a41e1c45759a"},
+    {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:7d6ae9d593ef8641544d6263c7fa6408cc90370c8cb2bbb65f8d43e5b0351d9c"},
+    {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:5979b5632c3e3534e42ca6ff856bb24b2e3071b37861c2c727ce220d80eee9ed"},
+    {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:dcfe792765fab89c365123c81046ad4103fcabbc4f56d1c1997e6715e8015461"},
+    {file = "multidict-6.0.4-cp311-cp311-win32.whl", hash = "sha256:3601a3cece3819534b11d4efc1eb76047488fddd0c85a3948099d5da4d504636"},
+    {file = "multidict-6.0.4-cp311-cp311-win_amd64.whl", hash = "sha256:81a4f0b34bd92df3da93315c6a59034df95866014ac08535fc819f043bfd51f0"},
+    {file = "multidict-6.0.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:67040058f37a2a51ed8ea8f6b0e6ee5bd78ca67f169ce6122f3e2ec80dfe9b78"},
+    {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:853888594621e6604c978ce2a0444a1e6e70c8d253ab65ba11657659dcc9100f"},
+    {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:39ff62e7d0f26c248b15e364517a72932a611a9b75f35b45be078d81bdb86603"},
+    {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:af048912e045a2dc732847d33821a9d84ba553f5c5f028adbd364dd4765092ac"},
+    {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b1e8b901e607795ec06c9e42530788c45ac21ef3aaa11dbd0c69de543bfb79a9"},
+    {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:62501642008a8b9871ddfccbf83e4222cf8ac0d5aeedf73da36153ef2ec222d2"},
+    {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:99b76c052e9f1bc0721f7541e5e8c05db3941eb9ebe7b8553c625ef88d6eefde"},
+    {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:509eac6cf09c794aa27bcacfd4d62c885cce62bef7b2c3e8b2e49d365b5003fe"},
+    {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:21a12c4eb6ddc9952c415f24eef97e3e55ba3af61f67c7bc388dcdec1404a067"},
+    {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:5cad9430ab3e2e4fa4a2ef4450f548768400a2ac635841bc2a56a2052cdbeb87"},
+    {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ab55edc2e84460694295f401215f4a58597f8f7c9466faec545093045476327d"},
+    {file = "multidict-6.0.4-cp37-cp37m-win32.whl", hash = "sha256:5a4dcf02b908c3b8b17a45fb0f15b695bf117a67b76b7ad18b73cf8e92608775"},
+    {file = "multidict-6.0.4-cp37-cp37m-win_amd64.whl", hash = "sha256:6ed5f161328b7df384d71b07317f4d8656434e34591f20552c7bcef27b0ab88e"},
+    {file = "multidict-6.0.4-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:5fc1b16f586f049820c5c5b17bb4ee7583092fa0d1c4e28b5239181ff9532e0c"},
+    {file = "multidict-6.0.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1502e24330eb681bdaa3eb70d6358e818e8e8f908a22a1851dfd4e15bc2f8161"},
+    {file = "multidict-6.0.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:b692f419760c0e65d060959df05f2a531945af31fda0c8a3b3195d4efd06de11"},
+    {file = "multidict-6.0.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:45e1ecb0379bfaab5eef059f50115b54571acfbe422a14f668fc8c27ba410e7e"},
+    {file = "multidict-6.0.4-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ddd3915998d93fbcd2566ddf9cf62cdb35c9e093075f862935573d265cf8f65d"},
+    {file = "multidict-6.0.4-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:59d43b61c59d82f2effb39a93c48b845efe23a3852d201ed2d24ba830d0b4cf2"},
+    {file = "multidict-6.0.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cc8e1d0c705233c5dd0c5e6460fbad7827d5d36f310a0fadfd45cc3029762258"},
+    {file = "multidict-6.0.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d6aa0418fcc838522256761b3415822626f866758ee0bc6632c9486b179d0b52"},
+    {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6748717bb10339c4760c1e63da040f5f29f5ed6e59d76daee30305894069a660"},
+    {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:4d1a3d7ef5e96b1c9e92f973e43aa5e5b96c659c9bc3124acbbd81b0b9c8a951"},
+    {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4372381634485bec7e46718edc71528024fcdc6f835baefe517b34a33c731d60"},
+    {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:fc35cb4676846ef752816d5be2193a1e8367b4c1397b74a565a9d0389c433a1d"},
+    {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4b9d9e4e2b37daddb5c23ea33a3417901fa7c7b3dee2d855f63ee67a0b21e5b1"},
+    {file = "multidict-6.0.4-cp38-cp38-win32.whl", hash = "sha256:e41b7e2b59679edfa309e8db64fdf22399eec4b0b24694e1b2104fb789207779"},
+    {file = "multidict-6.0.4-cp38-cp38-win_amd64.whl", hash = "sha256:d6c254ba6e45d8e72739281ebc46ea5eb5f101234f3ce171f0e9f5cc86991480"},
+    {file = "multidict-6.0.4-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:16ab77bbeb596e14212e7bab8429f24c1579234a3a462105cda4a66904998664"},
+    {file = "multidict-6.0.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:bc779e9e6f7fda81b3f9aa58e3a6091d49ad528b11ed19f6621408806204ad35"},
+    {file = "multidict-6.0.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4ceef517eca3e03c1cceb22030a3e39cb399ac86bff4e426d4fc6ae49052cc60"},
+    {file = "multidict-6.0.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:281af09f488903fde97923c7744bb001a9b23b039a909460d0f14edc7bf59706"},
+    {file = "multidict-6.0.4-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:52f2dffc8acaba9a2f27174c41c9e57f60b907bb9f096b36b1a1f3be71c6284d"},
+    {file = "multidict-6.0.4-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b41156839806aecb3641f3208c0dafd3ac7775b9c4c422d82ee2a45c34ba81ca"},
+    {file = "multidict-6.0.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d5e3fc56f88cc98ef8139255cf8cd63eb2c586531e43310ff859d6bb3a6b51f1"},
+    {file = "multidict-6.0.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8316a77808c501004802f9beebde51c9f857054a0c871bd6da8280e718444449"},
+    {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:f70b98cd94886b49d91170ef23ec5c0e8ebb6f242d734ed7ed677b24d50c82cf"},
+    {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bf6774e60d67a9efe02b3616fee22441d86fab4c6d335f9d2051d19d90a40063"},
+    {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:e69924bfcdda39b722ef4d9aa762b2dd38e4632b3641b1d9a57ca9cd18f2f83a"},
+    {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:6b181d8c23da913d4ff585afd1155a0e1194c0b50c54fcfe286f70cdaf2b7176"},
+    {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:52509b5be062d9eafc8170e53026fbc54cf3b32759a23d07fd935fb04fc22d95"},
+    {file = "multidict-6.0.4-cp39-cp39-win32.whl", hash = "sha256:27c523fbfbdfd19c6867af7346332b62b586eed663887392cff78d614f9ec313"},
+    {file = "multidict-6.0.4-cp39-cp39-win_amd64.whl", hash = "sha256:33029f5734336aa0d4c0384525da0387ef89148dc7191aae00ca5fb23d7aafc2"},
+    {file = "multidict-6.0.4.tar.gz", hash = "sha256:3666906492efb76453c0e7b97f2cf459b0682e7402c0489a95484965dbc1da49"},
+]
 
 [[package]]
 name = "nodeenv"
@@ -450,14 +956,25 @@ description = "Node.js virtual environment builder"
 category = "main"
 optional = false
 python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*"
+files = [
+    {file = "nodeenv-1.7.0-py2.py3-none-any.whl", hash = "sha256:27083a7b96a25f2f5e1d8cb4b6317ee8aeda3bdd121394e5ac54e498028a042e"},
+    {file = "nodeenv-1.7.0.tar.gz", hash = "sha256:e0e7f7dfb85fc5394c6fe1e8fa98131a2473e04311a45afb6508f7cf1836fa2b"},
+]
+
+[package.dependencies]
+setuptools = "*"
 
 [[package]]
 name = "packaging"
 version = "23.0"
 description = "Core utilities for Python packages"
-category = "dev"
+category = "main"
 optional = false
 python-versions = ">=3.7"
+files = [
+    {file = "packaging-23.0-py3-none-any.whl", hash = "sha256:714ac14496c3e68c99c29b00845f7a2b85f3bb6f1078fd9f72fd20f0570002b2"},
+    {file = "packaging-23.0.tar.gz", hash = "sha256:b6ad297f8907de0fa2fe1ccbd26fdaf387f5f47c7275fedf8cce89f99446cf97"},
+]
 
 [[package]]
 name = "pefile"
@@ -466,6 +983,9 @@ description = "Python PE parsing module"
 category = "main"
 optional = false
 python-versions = ">=3.6.0"
+files = [
+    {file = "pefile-2022.5.30.tar.gz", hash = "sha256:a5488a3dd1fd021ce33f969780b88fe0f7eebb76eb20996d7318f307612a045b"},
+]
 
 [package.dependencies]
 future = "*"
@@ -477,6 +997,10 @@ description = "Generates entity ID and connects with Vault (secret engine) to re
 category = "main"
 optional = false
 python-versions = ">=3.6"
+files = [
+    {file = "pilot-platform-common-0.1.3.tar.gz", hash = "sha256:c9b3adbe9654d1c8291a74e9ee742dcca3cd0e598b8d0146a05888950864850d"},
+    {file = "pilot_platform_common-0.1.3-py3-none-any.whl", hash = "sha256:1873f3fa062ec9a34910bc5a2f9ead8020a1188a2567d4cc1c850c2206164e7b"},
+]
 
 [package.dependencies]
 aioboto3 = "9.6.0"
@@ -494,21 +1018,29 @@ description = "A small Python package for determining appropriate platform-speci
 category = "main"
 optional = false
 python-versions = ">=3.7"
+files = [
+    {file = "platformdirs-2.6.2-py3-none-any.whl", hash = "sha256:83c8f6d04389165de7c9b6f0c682439697887bca0aa2f1c87ef1826be3584490"},
+    {file = "platformdirs-2.6.2.tar.gz", hash = "sha256:e1fea1fe471b9ff8332e229df3cb7de4f53eeea4998d3b6bfff542115e998bd2"},
+]
 
 [package.dependencies]
 typing-extensions = {version = ">=4.4", markers = "python_version < \"3.8\""}
 
 [package.extras]
-docs = ["furo (>=2022.12.7)", "proselint (>=0.13)", "sphinx-autodoc-typehints (>=1.19.5)", "sphinx (>=5.3)"]
-test = ["appdirs (==1.4.4)", "covdefaults (>=2.2.2)", "pytest-cov (>=4)", "pytest-mock (>=3.10)", "pytest (>=7.2)"]
+docs = ["furo (>=2022.12.7)", "proselint (>=0.13)", "sphinx (>=5.3)", "sphinx-autodoc-typehints (>=1.19.5)"]
+test = ["appdirs (==1.4.4)", "covdefaults (>=2.2.2)", "pytest (>=7.2)", "pytest-cov (>=4)", "pytest-mock (>=3.10)"]
 
 [[package]]
 name = "pluggy"
 version = "1.0.0"
 description = "plugin and hook calling mechanisms for python"
-category = "dev"
+category = "main"
 optional = false
 python-versions = ">=3.6"
+files = [
+    {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"},
+    {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"},
+]
 
 [package.dependencies]
 importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""}
@@ -524,6 +1056,10 @@ description = "A framework for managing and maintaining multi-language pre-commi
 category = "main"
 optional = false
 python-versions = ">=3.7"
+files = [
+    {file = "pre_commit-2.21.0-py2.py3-none-any.whl", hash = "sha256:e2f91727039fc39a92f58a588a25b87f936de6567eed4f0e673e0507edc75bad"},
+    {file = "pre_commit-2.21.0.tar.gz", hash = "sha256:31ef31af7e474a8d8995027fefdfcf509b5c913ff31f2015b4ec4beb26a6f658"},
+]
 
 [package.dependencies]
 cfgv = ">=2.0.0"
@@ -540,6 +1076,10 @@ description = "Library for building powerful interactive command lines in Python
 category = "main"
 optional = false
 python-versions = ">=3.6.2"
+files = [
+    {file = "prompt_toolkit-3.0.36-py3-none-any.whl", hash = "sha256:aa64ad242a462c5ff0363a7b9cfe696c20d55d9fc60c11fd8e632d064804d305"},
+    {file = "prompt_toolkit-3.0.36.tar.gz", hash = "sha256:3e163f254bef5a03b146397d7c1963bd3e2812f0964bb9a24e6ec761fd28db63"},
+]
 
 [package.dependencies]
 wcwidth = "*"
@@ -548,9 +1088,13 @@ wcwidth = "*"
 name = "py"
 version = "1.11.0"
 description = "library with cross-python path, ini-parsing, io, code, log facilities"
-category = "dev"
+category = "main"
 optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
+files = [
+    {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"},
+    {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"},
+]
 
 [[package]]
 name = "pycparser"
@@ -559,6 +1103,10 @@ description = "C parser in Python"
 category = "main"
 optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
+files = [
+    {file = "pycparser-2.21-py2.py3-none-any.whl", hash = "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9"},
+    {file = "pycparser-2.21.tar.gz", hash = "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"},
+]
 
 [[package]]
 name = "pydantic"
@@ -567,6 +1115,44 @@ description = "Data validation and settings management using python type hints"
 category = "main"
 optional = false
 python-versions = ">=3.7"
+files = [
+    {file = "pydantic-1.10.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b5635de53e6686fe7a44b5cf25fcc419a0d5e5c1a1efe73d49d48fe7586db854"},
+    {file = "pydantic-1.10.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6dc1cc241440ed7ca9ab59d9929075445da6b7c94ced281b3dd4cfe6c8cff817"},
+    {file = "pydantic-1.10.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:51bdeb10d2db0f288e71d49c9cefa609bca271720ecd0c58009bd7504a0c464c"},
+    {file = "pydantic-1.10.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:78cec42b95dbb500a1f7120bdf95c401f6abb616bbe8785ef09887306792e66e"},
+    {file = "pydantic-1.10.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:8775d4ef5e7299a2f4699501077a0defdaac5b6c4321173bcb0f3c496fbadf85"},
+    {file = "pydantic-1.10.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:572066051eeac73d23f95ba9a71349c42a3e05999d0ee1572b7860235b850cc6"},
+    {file = "pydantic-1.10.4-cp310-cp310-win_amd64.whl", hash = "sha256:7feb6a2d401f4d6863050f58325b8d99c1e56f4512d98b11ac64ad1751dc647d"},
+    {file = "pydantic-1.10.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:39f4a73e5342b25c2959529f07f026ef58147249f9b7431e1ba8414a36761f53"},
+    {file = "pydantic-1.10.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:983e720704431a6573d626b00662eb78a07148c9115129f9b4351091ec95ecc3"},
+    {file = "pydantic-1.10.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75d52162fe6b2b55964fbb0af2ee58e99791a3138588c482572bb6087953113a"},
+    {file = "pydantic-1.10.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fdf8d759ef326962b4678d89e275ffc55b7ce59d917d9f72233762061fd04a2d"},
+    {file = "pydantic-1.10.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:05a81b006be15655b2a1bae5faa4280cf7c81d0e09fcb49b342ebf826abe5a72"},
+    {file = "pydantic-1.10.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d88c4c0e5c5dfd05092a4b271282ef0588e5f4aaf345778056fc5259ba098857"},
+    {file = "pydantic-1.10.4-cp311-cp311-win_amd64.whl", hash = "sha256:6a05a9db1ef5be0fe63e988f9617ca2551013f55000289c671f71ec16f4985e3"},
+    {file = "pydantic-1.10.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:887ca463c3bc47103c123bc06919c86720e80e1214aab79e9b779cda0ff92a00"},
+    {file = "pydantic-1.10.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fdf88ab63c3ee282c76d652fc86518aacb737ff35796023fae56a65ced1a5978"},
+    {file = "pydantic-1.10.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a48f1953c4a1d9bd0b5167ac50da9a79f6072c63c4cef4cf2a3736994903583e"},
+    {file = "pydantic-1.10.4-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:a9f2de23bec87ff306aef658384b02aa7c32389766af3c5dee9ce33e80222dfa"},
+    {file = "pydantic-1.10.4-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:cd8702c5142afda03dc2b1ee6bc358b62b3735b2cce53fc77b31ca9f728e4bc8"},
+    {file = "pydantic-1.10.4-cp37-cp37m-win_amd64.whl", hash = "sha256:6e7124d6855b2780611d9f5e1e145e86667eaa3bd9459192c8dc1a097f5e9903"},
+    {file = "pydantic-1.10.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0b53e1d41e97063d51a02821b80538053ee4608b9a181c1005441f1673c55423"},
+    {file = "pydantic-1.10.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:55b1625899acd33229c4352ce0ae54038529b412bd51c4915349b49ca575258f"},
+    {file = "pydantic-1.10.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:301d626a59edbe5dfb48fcae245896379a450d04baeed50ef40d8199f2733b06"},
+    {file = "pydantic-1.10.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b6f9d649892a6f54a39ed56b8dfd5e08b5f3be5f893da430bed76975f3735d15"},
+    {file = "pydantic-1.10.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:d7b5a3821225f5c43496c324b0d6875fde910a1c2933d726a743ce328fbb2a8c"},
+    {file = "pydantic-1.10.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:f2f7eb6273dd12472d7f218e1fef6f7c7c2f00ac2e1ecde4db8824c457300416"},
+    {file = "pydantic-1.10.4-cp38-cp38-win_amd64.whl", hash = "sha256:4b05697738e7d2040696b0a66d9f0a10bec0efa1883ca75ee9e55baf511909d6"},
+    {file = "pydantic-1.10.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a9a6747cac06c2beb466064dda999a13176b23535e4c496c9d48e6406f92d42d"},
+    {file = "pydantic-1.10.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:eb992a1ef739cc7b543576337bebfc62c0e6567434e522e97291b251a41dad7f"},
+    {file = "pydantic-1.10.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:990406d226dea0e8f25f643b370224771878142155b879784ce89f633541a024"},
+    {file = "pydantic-1.10.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e82a6d37a95e0b1b42b82ab340ada3963aea1317fd7f888bb6b9dfbf4fff57c"},
+    {file = "pydantic-1.10.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9193d4f4ee8feca58bc56c8306bcb820f5c7905fd919e0750acdeeeef0615b28"},
+    {file = "pydantic-1.10.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:2b3ce5f16deb45c472dde1a0ee05619298c864a20cded09c4edd820e1454129f"},
+    {file = "pydantic-1.10.4-cp39-cp39-win_amd64.whl", hash = "sha256:9cbdc268a62d9a98c56e2452d6c41c0263d64a2009aac69246486f01b4f594c4"},
+    {file = "pydantic-1.10.4-py3-none-any.whl", hash = "sha256:4948f264678c703f3877d1c8877c4e3b2e12e549c57795107f08cf70c6ec7774"},
+    {file = "pydantic-1.10.4.tar.gz", hash = "sha256:b9a3859f24eb4e097502a3be1fb4b2abb79b6103dd9e2e0edb70613a4459a648"},
+]
 
 [package.dependencies]
 typing-extensions = ">=4.2.0"
@@ -582,6 +1168,20 @@ description = "PyInstaller bundles a Python application and all its dependencies
 category = "main"
 optional = false
 python-versions = "<3.12,>=3.7"
+files = [
+    {file = "pyinstaller-5.7.0-py3-none-macosx_10_13_universal2.whl", hash = "sha256:b967ae71ab7b05e18608dbb4518da5afa54f0835927cb7a5ce52ab8fffed03b6"},
+    {file = "pyinstaller-5.7.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:3180b9bf22263380adc5e2ee051b7c21463292877215bbe70c9155dc76f4b966"},
+    {file = "pyinstaller-5.7.0-py3-none-manylinux2014_i686.whl", hash = "sha256:0f80e2403e76630ad3392c71f09c1a4284e8d8a8a99fb55ff3a0aba0e06300ed"},
+    {file = "pyinstaller-5.7.0-py3-none-manylinux2014_ppc64le.whl", hash = "sha256:2c1dd9d11cfc48bab61eeb06de69a3d1ad742bbb2ef14716965ca0333dd43a5b"},
+    {file = "pyinstaller-5.7.0-py3-none-manylinux2014_s390x.whl", hash = "sha256:dfc12e92fe10ae645dd0dd1fcfa4cd7677b2e96119e3cd4980d742e09bb78925"},
+    {file = "pyinstaller-5.7.0-py3-none-manylinux2014_x86_64.whl", hash = "sha256:f35f06d48faea0ad738429c009941059beebaa306e9d9ead95f1df4b441de2aa"},
+    {file = "pyinstaller-5.7.0-py3-none-musllinux_1_1_aarch64.whl", hash = "sha256:28a8a0da656493aa32d9665e2f6f84775da0f23174859ed8facaa4226fe77a17"},
+    {file = "pyinstaller-5.7.0-py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:1ac3f09b838710c43e34b0a7ad003bd168a754b0b786c561b47baf1af9104354"},
+    {file = "pyinstaller-5.7.0-py3-none-win32.whl", hash = "sha256:9cdb8ee8622ee8d2c6cd67f001b610019d4371a8bf3f7850562640ce786894d7"},
+    {file = "pyinstaller-5.7.0-py3-none-win_amd64.whl", hash = "sha256:9b47c10fbefac6f6493266f8b1689109b2b14efa9142dbd2cd7549226a4568b7"},
+    {file = "pyinstaller-5.7.0-py3-none-win_arm64.whl", hash = "sha256:3e51e18a16dec0414079762843cf892a5d70749ad56ca7b3c7b5f8367dc50b1e"},
+    {file = "pyinstaller-5.7.0.tar.gz", hash = "sha256:0e5953937d35f0b37543cc6915dacaf3239bcbdf3fd3ecbb7866645468a16775"},
+]
 
 [package.dependencies]
 altgraph = "*"
@@ -590,10 +1190,11 @@ macholib = {version = ">=1.8", markers = "sys_platform == \"darwin\""}
 pefile = {version = ">=2022.5.30", markers = "sys_platform == \"win32\""}
 pyinstaller-hooks-contrib = ">=2021.4"
 pywin32-ctypes = {version = ">=0.2.0", markers = "sys_platform == \"win32\""}
+setuptools = ">=42.0.0"
 
 [package.extras]
 encryption = ["tinyaes (>=1.0.0)"]
-hook_testing = ["pytest (>=2.7.3)", "execnet (>=1.5.0)", "psutil"]
+hook-testing = ["execnet (>=1.5.0)", "psutil", "pytest (>=2.7.3)"]
 
 [[package]]
 name = "pyinstaller-hooks-contrib"
@@ -602,6 +1203,10 @@ description = "Community maintained hooks for PyInstaller"
 category = "main"
 optional = false
 python-versions = ">=3.7"
+files = [
+    {file = "pyinstaller-hooks-contrib-2022.15.tar.gz", hash = "sha256:73fd4051dc1620f3ae9643291cd9e2f47bfed582ade2eb05e3247ecab4a4f5f3"},
+    {file = "pyinstaller_hooks_contrib-2022.15-py2.py3-none-any.whl", hash = "sha256:55c1def8066d0279d06cd67eea30c12ffcdb961a5edeeaf361adac0164baef30"},
+]
 
 [[package]]
 name = "pyjwt"
@@ -610,6 +1215,10 @@ description = "JSON Web Token implementation in Python"
 category = "main"
 optional = false
 python-versions = "*"
+files = [
+    {file = "PyJWT-1.7.1-py2.py3-none-any.whl", hash = "sha256:5c6eca3c2940464d106b99ba83b00c6add741c9becaec087fb7ccdefea71350e"},
+    {file = "PyJWT-1.7.1.tar.gz", hash = "sha256:8d59a976fb773f3e6a39c85636357c4f0e242707394cadadd9814f5cbaa20e96"},
+]
 
 [package.extras]
 crypto = ["cryptography (>=1.4)"]
@@ -623,14 +1232,22 @@ description = "Pure Python library for saving and loading PNG images"
 category = "main"
 optional = false
 python-versions = "*"
+files = [
+    {file = "pypng-0.20220715.0-py3-none-any.whl", hash = "sha256:4a43e969b8f5aaafb2a415536c1a8ec7e341cd6a3f957fd5b5f32a4cfeed902c"},
+    {file = "pypng-0.20220715.0.tar.gz", hash = "sha256:739c433ba96f078315de54c0db975aee537cbc3e1d0ae4ed9aab0ca1e427e2c1"},
+]
 
 [[package]]
 name = "pytest"
 version = "6.2.5"
 description = "pytest: simple powerful testing with Python"
-category = "dev"
+category = "main"
 optional = false
 python-versions = ">=3.6"
+files = [
+    {file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"},
+    {file = "pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"},
+]
 
 [package.dependencies]
 atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""}
@@ -646,6 +1263,22 @@ toml = "*"
 [package.extras]
 testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"]
 
+[[package]]
+name = "pytest-click"
+version = "1.1.0"
+description = "Pytest plugin for Click"
+category = "main"
+optional = false
+python-versions = "*"
+files = [
+    {file = "pytest_click-1.1.0-py3-none-any.whl", hash = "sha256:eade4742c2f02c345e78a32534a43e8db04acf98d415090539dacc880b7cd0e9"},
+    {file = "pytest_click-1.1.0.tar.gz", hash = "sha256:fdd9f6721f877dda021e7c5dc73e70aecd37e5ed23ec6820f8a7b3fd7b4f8d30"},
+]
+
+[package.dependencies]
+click = ">=6.0"
+pytest = ">=5.0"
+
 [[package]]
 name = "pytest-cov"
 version = "3.0.0"
@@ -653,13 +1286,17 @@ description = "Pytest plugin for measuring coverage."
 category = "dev"
 optional = false
 python-versions = ">=3.6"
+files = [
+    {file = "pytest-cov-3.0.0.tar.gz", hash = "sha256:e7f0f5b1617d2210a2cabc266dfe2f4c75a8d32fb89eafb7ad9d06f6d076d470"},
+    {file = "pytest_cov-3.0.0-py3-none-any.whl", hash = "sha256:578d5d15ac4a25e5f961c938b85a05b09fdaae9deef3bb6de9a6e766622ca7a6"},
+]
 
 [package.dependencies]
 coverage = {version = ">=5.2.1", extras = ["toml"]}
 pytest = ">=4.6"
 
 [package.extras]
-testing = ["fields", "hunter", "process-tests", "six", "pytest-xdist", "virtualenv"]
+testing = ["fields", "hunter", "process-tests", "pytest-xdist", "six", "virtualenv"]
 
 [[package]]
 name = "pytest-httpx"
@@ -668,6 +1305,10 @@ description = "Send responses to httpx."
 category = "dev"
 optional = false
 python-versions = ">=3.7"
+files = [
+    {file = "pytest_httpx-0.21.3-py3-none-any.whl", hash = "sha256:50b52b910f6f6cfb0aa65039d6f5bedb6ae3a0c02a98c4a7187543fe437c428a"},
+    {file = "pytest_httpx-0.21.3.tar.gz", hash = "sha256:edcb62baceffbd57753c1a7afc4656b0e71e91c7a512e143c0adbac762d979c1"},
+]
 
 [package.dependencies]
 httpx = ">=0.23.0,<0.24.0"
@@ -683,12 +1324,16 @@ description = "Thin-wrapper around the mock package for easier use with pytest"
 category = "dev"
 optional = false
 python-versions = ">=3.7"
+files = [
+    {file = "pytest-mock-3.10.0.tar.gz", hash = "sha256:fbbdb085ef7c252a326fd8cdcac0aa3b1333d8811f131bdcc701002e1be7ed4f"},
+    {file = "pytest_mock-3.10.0-py3-none-any.whl", hash = "sha256:f4c973eeae0282963eb293eb173ce91b091a79c1334455acfac9ddee8a1c784b"},
+]
 
 [package.dependencies]
 pytest = ">=5.0"
 
 [package.extras]
-dev = ["pre-commit", "tox", "pytest-asyncio"]
+dev = ["pre-commit", "pytest-asyncio", "tox"]
 
 [[package]]
 name = "python-dateutil"
@@ -697,6 +1342,10 @@ description = "Extensions to the standard Python datetime module"
 category = "main"
 optional = false
 python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7"
+files = [
+    {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"},
+    {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"},
+]
 
 [package.dependencies]
 six = ">=1.5"
@@ -708,6 +1357,10 @@ description = "Read key-value pairs from a .env file and set them as environment
 category = "main"
 optional = false
 python-versions = ">=3.5"
+files = [
+    {file = "python-dotenv-0.19.1.tar.gz", hash = "sha256:14f8185cc8d494662683e6914addcb7e95374771e707601dfc70166946b4c4b8"},
+    {file = "python_dotenv-0.19.1-py2.py3-none-any.whl", hash = "sha256:bbd3da593fc49c249397cbfbcc449cf36cb02e75afc8157fcc6a81df6fb7750a"},
+]
 
 [package.extras]
 cli = ["click (>=5.0)"]
@@ -719,6 +1372,10 @@ description = "A python library adding a json log formatter"
 category = "main"
 optional = false
 python-versions = ">=3.5"
+files = [
+    {file = "python-json-logger-2.0.2.tar.gz", hash = "sha256:202a4f29901a4b8002a6d1b958407eeb2dd1d83c18b18b816f5b64476dde9096"},
+    {file = "python_json_logger-2.0.2-py3-none-any.whl", hash = "sha256:99310d148f054e858cd5f4258794ed6777e7ad2c3fd7e1c1b527f1cba4d08420"},
+]
 
 [[package]]
 name = "pywin32-ctypes"
@@ -727,6 +1384,10 @@ description = ""
 category = "main"
 optional = false
 python-versions = "*"
+files = [
+    {file = "pywin32-ctypes-0.2.0.tar.gz", hash = "sha256:24ffc3b341d457d48e8922352130cf2644024a4ff09762a2261fd34c36ee5942"},
+    {file = "pywin32_ctypes-0.2.0-py2.py3-none-any.whl", hash = "sha256:9dc2d991b3479cc2df15930958b674a48a227d5361d413827a4cfd0b5876fc98"},
+]
 
 [[package]]
 name = "pyyaml"
@@ -735,6 +1396,48 @@ description = "YAML parser and emitter for Python"
 category = "main"
 optional = false
 python-versions = ">=3.6"
+files = [
+    {file = "PyYAML-6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53"},
+    {file = "PyYAML-6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c"},
+    {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc"},
+    {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b"},
+    {file = "PyYAML-6.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5"},
+    {file = "PyYAML-6.0-cp310-cp310-win32.whl", hash = "sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513"},
+    {file = "PyYAML-6.0-cp310-cp310-win_amd64.whl", hash = "sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a"},
+    {file = "PyYAML-6.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d4b0ba9512519522b118090257be113b9468d804b19d63c71dbcf4a48fa32358"},
+    {file = "PyYAML-6.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:81957921f441d50af23654aa6c5e5eaf9b06aba7f0a19c18a538dc7ef291c5a1"},
+    {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:afa17f5bc4d1b10afd4466fd3a44dc0e245382deca5b3c353d8b757f9e3ecb8d"},
+    {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dbad0e9d368bb989f4515da330b88a057617d16b6a8245084f1b05400f24609f"},
+    {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:432557aa2c09802be39460360ddffd48156e30721f5e8d917f01d31694216782"},
+    {file = "PyYAML-6.0-cp311-cp311-win32.whl", hash = "sha256:bfaef573a63ba8923503d27530362590ff4f576c626d86a9fed95822a8255fd7"},
+    {file = "PyYAML-6.0-cp311-cp311-win_amd64.whl", hash = "sha256:01b45c0191e6d66c470b6cf1b9531a771a83c1c4208272ead47a3ae4f2f603bf"},
+    {file = "PyYAML-6.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86"},
+    {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f"},
+    {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92"},
+    {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4"},
+    {file = "PyYAML-6.0-cp36-cp36m-win32.whl", hash = "sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293"},
+    {file = "PyYAML-6.0-cp36-cp36m-win_amd64.whl", hash = "sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57"},
+    {file = "PyYAML-6.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c"},
+    {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0"},
+    {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4"},
+    {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9"},
+    {file = "PyYAML-6.0-cp37-cp37m-win32.whl", hash = "sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737"},
+    {file = "PyYAML-6.0-cp37-cp37m-win_amd64.whl", hash = "sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d"},
+    {file = "PyYAML-6.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b"},
+    {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba"},
+    {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34"},
+    {file = "PyYAML-6.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287"},
+    {file = "PyYAML-6.0-cp38-cp38-win32.whl", hash = "sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78"},
+    {file = "PyYAML-6.0-cp38-cp38-win_amd64.whl", hash = "sha256:1e4747bc279b4f613a09eb64bba2ba602d8a6664c6ce6396a4d0cd413a50ce07"},
+    {file = "PyYAML-6.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b"},
+    {file = "PyYAML-6.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174"},
+    {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803"},
+    {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3"},
+    {file = "PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0"},
+    {file = "PyYAML-6.0-cp39-cp39-win32.whl", hash = "sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb"},
+    {file = "PyYAML-6.0-cp39-cp39-win_amd64.whl", hash = "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c"},
+    {file = "PyYAML-6.0.tar.gz", hash = "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2"},
+]
 
 [[package]]
 name = "qrcode"
@@ -743,6 +1446,10 @@ description = "QR Code image generator"
 category = "main"
 optional = false
 python-versions = ">=3.7"
+files = [
+    {file = "qrcode-7.4.2-py3-none-any.whl", hash = "sha256:581dca7a029bcb2deef5d01068e39093e80ef00b4a61098a2182eac59d01643a"},
+    {file = "qrcode-7.4.2.tar.gz", hash = "sha256:9dd969454827e127dbd93696b20747239e6d540e082937c90f14ac95b30f5845"},
+]
 
 [package.dependencies]
 colorama = {version = "*", markers = "platform_system == \"Windows\""}
@@ -750,9 +1457,9 @@ pypng = "*"
 typing-extensions = "*"
 
 [package.extras]
-all = ["zest.releaser", "tox", "pytest", "pytest-cov", "pillow (>=9.1.0)"]
-dev = ["tox", "pytest", "pytest-cov"]
-maintainer = ["zest.releaser"]
+all = ["pillow (>=9.1.0)", "pytest", "pytest-cov", "tox", "zest.releaser[recommended]"]
+dev = ["pytest", "pytest-cov", "tox"]
+maintainer = ["zest.releaser[recommended]"]
 pil = ["pillow (>=9.1.0)"]
 test = ["coverage", "pytest"]
 
@@ -763,12 +1470,16 @@ description = "Python library to build pretty command line user prompts ⭐️"
 category = "main"
 optional = false
 python-versions = ">=3.6,<4.0"
+files = [
+    {file = "questionary-1.10.0-py3-none-any.whl", hash = "sha256:fecfcc8cca110fda9d561cb83f1e97ecbb93c613ff857f655818839dac74ce90"},
+    {file = "questionary-1.10.0.tar.gz", hash = "sha256:600d3aefecce26d48d97eee936fdb66e4bc27f934c3ab6dd1e292c4f43946d90"},
+]
 
 [package.dependencies]
 prompt_toolkit = ">=2.0,<4.0"
 
 [package.extras]
-docs = ["Sphinx (>=3.3,<4.0)", "sphinx-rtd-theme (>=0.5.0,<0.6.0)", "sphinx-autobuild (>=2020.9.1,<2021.0.0)", "sphinx-copybutton (>=0.3.1,<0.4.0)", "sphinx-autodoc-typehints (>=1.11.1,<2.0.0)"]
+docs = ["Sphinx (>=3.3,<4.0)", "sphinx-autobuild (>=2020.9.1,<2021.0.0)", "sphinx-autodoc-typehints (>=1.11.1,<2.0.0)", "sphinx-copybutton (>=0.3.1,<0.4.0)", "sphinx-rtd-theme (>=0.5.0,<0.6.0)"]
 
 [[package]]
 name = "requests"
@@ -777,6 +1488,10 @@ description = "Python HTTP for Humans."
 category = "main"
 optional = false
 python-versions = ">=3.7, <4"
+files = [
+    {file = "requests-2.28.2-py3-none-any.whl", hash = "sha256:64299f4909223da747622c030b781c0d7811e359c37124b4bd368fb8c6518baa"},
+    {file = "requests-2.28.2.tar.gz", hash = "sha256:98b1b2782e3c6c4904938b84c0eb932721069dfdb9134313beff7c83c2df24bf"},
+]
 
 [package.dependencies]
 certifi = ">=2017.4.17"
@@ -786,7 +1501,7 @@ urllib3 = ">=1.21.1,<1.27"
 
 [package.extras]
 socks = ["PySocks (>=1.5.6,!=1.5.7)"]
-use_chardet_on_py3 = ["chardet (>=3.0.2,<6)"]
+use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"]
 
 [[package]]
 name = "requests-mock"
@@ -795,6 +1510,10 @@ description = "Mock out responses from the requests package"
 category = "dev"
 optional = false
 python-versions = "*"
+files = [
+    {file = "requests-mock-1.10.0.tar.gz", hash = "sha256:59c9c32419a9fb1ae83ec242d98e889c45bd7d7a65d48375cc243ec08441658b"},
+    {file = "requests_mock-1.10.0-py2.py3-none-any.whl", hash = "sha256:2fdbb637ad17ee15c06f33d31169e71bf9fe2bdb7bc9da26185be0dd8d842699"},
+]
 
 [package.dependencies]
 requests = ">=2.3,<3"
@@ -802,7 +1521,7 @@ six = "*"
 
 [package.extras]
 fixture = ["fixtures"]
-test = ["fixtures", "mock", "purl", "pytest", "sphinx", "testrepository (>=0.0.18)", "testtools", "requests-futures"]
+test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "testrepository (>=0.0.18)", "testtools"]
 
 [[package]]
 name = "rfc3986"
@@ -811,6 +1530,10 @@ description = "Validating URI References per RFC 3986"
 category = "main"
 optional = false
 python-versions = "*"
+files = [
+    {file = "rfc3986-1.5.0-py2.py3-none-any.whl", hash = "sha256:a86d6e1f5b1dc238b218b012df0aa79409667bb209e58da56d0b94704e712a97"},
+    {file = "rfc3986-1.5.0.tar.gz", hash = "sha256:270aaf10d87d0d4e095063c65bf3ddbc6ee3d0b226328ce21e036f946e421835"},
+]
 
 [package.dependencies]
 idna = {version = "*", optional = true, markers = "extra == \"idna2008\""}
@@ -825,6 +1548,10 @@ description = "An Amazon S3 Transfer Manager"
 category = "main"
 optional = false
 python-versions = ">= 3.6"
+files = [
+    {file = "s3transfer-0.5.2-py3-none-any.whl", hash = "sha256:7a6f4c4d1fdb9a2b640244008e142cbc2cd3ae34b386584ef044dd0f27101971"},
+    {file = "s3transfer-0.5.2.tar.gz", hash = "sha256:95c58c194ce657a5f4fb0b9e60a84968c808888aed628cd98ab8771fe1db98ed"},
+]
 
 [package.dependencies]
 botocore = ">=1.12.36,<2.0a.0"
@@ -832,6 +1559,23 @@ botocore = ">=1.12.36,<2.0a.0"
 [package.extras]
 crt = ["botocore[crt] (>=1.20.29,<2.0a.0)"]
 
+[[package]]
+name = "setuptools"
+version = "67.6.1"
+description = "Easily download, build, install, upgrade, and uninstall Python packages"
+category = "main"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "setuptools-67.6.1-py3-none-any.whl", hash = "sha256:e728ca814a823bf7bf60162daf9db95b93d532948c4c0bea762ce62f60189078"},
+    {file = "setuptools-67.6.1.tar.gz", hash = "sha256:257de92a9d50a60b8e22abfcbb771571fde0dbf3ec234463212027a4eeecbe9a"},
+]
+
+[package.extras]
+docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"]
+testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8 (<5)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"]
+testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"]
+
 [[package]]
 name = "six"
 version = "1.16.0"
@@ -839,6 +1583,10 @@ description = "Python 2 and 3 compatibility utilities"
 category = "main"
 optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*"
+files = [
+    {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"},
+    {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"},
+]
 
 [[package]]
 name = "sniffio"
@@ -847,14 +1595,22 @@ description = "Sniff out which async library your code is running under"
 category = "main"
 optional = false
 python-versions = ">=3.7"
+files = [
+    {file = "sniffio-1.3.0-py3-none-any.whl", hash = "sha256:eecefdce1e5bbfb7ad2eeaabf7c1eeb404d7757c379bd1f7e5cce9d8bf425384"},
+    {file = "sniffio-1.3.0.tar.gz", hash = "sha256:e60305c5e5d314f5389259b7f22aaa33d8f7dee49763119234af3755c55b9101"},
+]
 
 [[package]]
 name = "toml"
 version = "0.10.2"
 description = "Python Library for Tom's Obvious, Minimal Language"
-category = "dev"
+category = "main"
 optional = false
 python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*"
+files = [
+    {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"},
+    {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"},
+]
 
 [[package]]
 name = "tomli"
@@ -863,6 +1619,10 @@ description = "A lil' TOML parser"
 category = "dev"
 optional = false
 python-versions = ">=3.7"
+files = [
+    {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"},
+    {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"},
+]
 
 [[package]]
 name = "tqdm"
@@ -871,6 +1631,10 @@ description = "Fast, Extensible Progress Meter"
 category = "main"
 optional = false
 python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.7"
+files = [
+    {file = "tqdm-4.56.0-py2.py3-none-any.whl", hash = "sha256:4621f6823bab46a9cc33d48105753ccbea671b68bab2c50a9f0be23d4065cb5a"},
+    {file = "tqdm-4.56.0.tar.gz", hash = "sha256:fe3d08dd00a526850568d542ff9de9bbc2a09a791da3c334f3213d8d0bbbca65"},
+]
 
 [package.extras]
 dev = ["py-make (>=0.1.0)", "twine", "wheel"]
@@ -883,6 +1647,10 @@ description = "Backported and Experimental Type Hints for Python 3.7+"
 category = "main"
 optional = false
 python-versions = ">=3.7"
+files = [
+    {file = "typing_extensions-4.4.0-py3-none-any.whl", hash = "sha256:16fa4864408f655d35ec496218b85f79b3437c829e93320c7c9215ccfd92489e"},
+    {file = "typing_extensions-4.4.0.tar.gz", hash = "sha256:1511434bb92bf8dd198c12b1cc812e800d4181cfcb867674e0f8279cc93087aa"},
+]
 
 [[package]]
 name = "urllib3"
@@ -891,10 +1659,14 @@ description = "HTTP library with thread-safe connection pooling, file post, and
 category = "main"
 optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*"
+files = [
+    {file = "urllib3-1.26.14-py2.py3-none-any.whl", hash = "sha256:75edcdc2f7d85b137124a6c3c9fc3933cdeaa12ecb9a6a959f22797a0feca7e1"},
+    {file = "urllib3-1.26.14.tar.gz", hash = "sha256:076907bf8fd355cde77728471316625a4d2f7e713c125f51953bb5b3eecf4f72"},
+]
 
 [package.extras]
-brotli = ["brotlicffi (>=0.8.0)", "brotli (>=1.0.9)", "brotlipy (>=0.6.0)"]
-secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "certifi", "urllib3-secure-extra", "ipaddress"]
+brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"]
+secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"]
 socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"]
 
 [[package]]
@@ -904,6 +1676,10 @@ description = "Virtual Python Environment builder"
 category = "main"
 optional = false
 python-versions = ">=3.6"
+files = [
+    {file = "virtualenv-20.16.2-py2.py3-none-any.whl", hash = "sha256:635b272a8e2f77cb051946f46c60a54ace3cb5e25568228bd6b57fc70eca9ff3"},
+    {file = "virtualenv-20.16.2.tar.gz", hash = "sha256:0ef5be6d07181946891f5abc8047fda8bc2f0b4b9bf222c64e6e8963baee76db"},
+]
 
 [package.dependencies]
 distlib = ">=0.3.1,<1"
@@ -922,6 +1698,10 @@ description = "Measures the displayed width of unicode strings in a terminal"
 category = "main"
 optional = false
 python-versions = "*"
+files = [
+    {file = "wcwidth-0.2.6-py2.py3-none-any.whl", hash = "sha256:795b138f6875577cd91bba52baf9e445cd5118fd32723b460e30a0af30ea230e"},
+    {file = "wcwidth-0.2.6.tar.gz", hash = "sha256:a5220780a404dbe3353789870978e472cfe477761f06ee55077256e509b156d0"},
+]
 
 [[package]]
 name = "wrapt"
@@ -930,208 +1710,7 @@ description = "Module for decorators, wrappers and monkey patching."
 category = "main"
 optional = false
 python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7"
-
-[[package]]
-name = "xmltodict"
-version = "0.13.0"
-description = "Makes working with XML feel like you are working with JSON"
-category = "main"
-optional = false
-python-versions = ">=3.4"
-
-[[package]]
-name = "yarl"
-version = "1.8.2"
-description = "Yet another URL library"
-category = "main"
-optional = false
-python-versions = ">=3.7"
-
-[package.dependencies]
-idna = ">=2.0"
-multidict = ">=4.0"
-typing-extensions = {version = ">=3.7.4", markers = "python_version < \"3.8\""}
-
-[[package]]
-name = "zipp"
-version = "3.11.0"
-description = "Backport of pathlib-compatible object wrapper for zip files"
-category = "main"
-optional = false
-python-versions = ">=3.7"
-
-[package.extras]
-docs = ["sphinx (>=3.5)", "jaraco.packaging (>=9)", "rst.linker (>=1.9)", "furo", "jaraco.tidelift (>=1.4)"]
-testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "flake8 (<5)", "pytest-cov", "pytest-enabler (>=1.3)", "jaraco.itertools", "func-timeout", "jaraco.functools", "more-itertools", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)", "pytest-flake8"]
-
-[metadata]
-lock-version = "1.1"
-python-versions = ">=3.7,<3.11"
-content-hash = "8fe27d4e764394baa6fbeb701d7ee9e556f6d1b5b43cf4a04427c125972f819d"
-
-[metadata.files]
-aioboto3 = []
-aiobotocore = []
-aiohttp = []
-aioitertools = []
-aioredis = [
-    {file = "aioredis-2.0.1-py3-none-any.whl", hash = "sha256:9ac0d0b3b485d293b8ca1987e6de8658d7dafcca1cddfcd1d506cae8cdebfdd6"},
-    {file = "aioredis-2.0.1.tar.gz", hash = "sha256:eaa51aaf993f2d71f54b70527c440437ba65340588afeb786cd87c55c89cd98e"},
-]
-aiosignal = []
-altgraph = []
-anyio = []
-async-timeout = [
-    {file = "async-timeout-4.0.2.tar.gz", hash = "sha256:2163e1640ddb52b7a8c80d0a67a08587e5d245cc9c553a74a847056bc2976b15"},
-    {file = "async_timeout-4.0.2-py3-none-any.whl", hash = "sha256:8ca1e4fcf50d07413d66d1a5e416e42cfdf5851c981d679a09851a6853383b3c"},
-]
-asynctest = []
-atomicwrites = []
-attrs = []
-boto3 = []
-botocore = [
-    {file = "botocore-1.24.21-py3-none-any.whl", hash = "sha256:92daca8775e738a9db9b465d533019285f09d541e903233261299fd87c2f842c"},
-    {file = "botocore-1.24.21.tar.gz", hash = "sha256:7e976cfd0a61601e74624ef8f5246b40a01f2cce73a011ef29cf80a6e371d0fa"},
-]
-certifi = []
-cffi = []
-cfgv = []
-charset-normalizer = []
-click = [
-    {file = "click-7.1.2-py2.py3-none-any.whl", hash = "sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc"},
-    {file = "click-7.1.2.tar.gz", hash = "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a"},
-]
-colorama = []
-coverage = []
-cryptography = []
-distlib = []
-filelock = []
-frozenlist = []
-future = []
-h11 = [
-    {file = "h11-0.12.0-py3-none-any.whl", hash = "sha256:36a3cb8c0a032f56e2da7084577878a035d3b61d104230d4bd49c0c6b555a9c6"},
-    {file = "h11-0.12.0.tar.gz", hash = "sha256:47222cb6067e4a307d535814917cd98fd0a57b6788ce715755fa2b6c28b56042"},
-]
-httpcore = []
-httpx = []
-identify = []
-idna = [
-    {file = "idna-2.10-py2.py3-none-any.whl", hash = "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"},
-    {file = "idna-2.10.tar.gz", hash = "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6"},
-]
-importlib-metadata = [
-    {file = "importlib_metadata-4.2.0-py3-none-any.whl", hash = "sha256:057e92c15bc8d9e8109738a48db0ccb31b4d9d5cfbee5a8670879a30be66304b"},
-    {file = "importlib_metadata-4.2.0.tar.gz", hash = "sha256:b7e52a1f8dec14a75ea73e0891f3060099ca1d8e6a462a4dff11c3e119ea1b31"},
-]
-iniconfig = []
-jmespath = []
-macholib = []
-minio = []
-multidict = []
-nodeenv = []
-packaging = []
-pefile = []
-pilot-platform-common = []
-platformdirs = []
-pluggy = [
-    {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"},
-    {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"},
-]
-pre-commit = []
-prompt-toolkit = []
-py = [
-    {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"},
-    {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"},
-]
-pycparser = []
-pydantic = []
-pyinstaller = []
-pyinstaller-hooks-contrib = []
-pyjwt = []
-pypng = []
-pytest = [
-    {file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"},
-    {file = "pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"},
-]
-pytest-cov = [
-    {file = "pytest-cov-3.0.0.tar.gz", hash = "sha256:e7f0f5b1617d2210a2cabc266dfe2f4c75a8d32fb89eafb7ad9d06f6d076d470"},
-    {file = "pytest_cov-3.0.0-py3-none-any.whl", hash = "sha256:578d5d15ac4a25e5f961c938b85a05b09fdaae9deef3bb6de9a6e766622ca7a6"},
-]
-pytest-httpx = []
-pytest-mock = []
-python-dateutil = [
-    {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"},
-    {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"},
-]
-python-dotenv = [
-    {file = "python-dotenv-0.19.1.tar.gz", hash = "sha256:14f8185cc8d494662683e6914addcb7e95374771e707601dfc70166946b4c4b8"},
-    {file = "python_dotenv-0.19.1-py2.py3-none-any.whl", hash = "sha256:bbd3da593fc49c249397cbfbcc449cf36cb02e75afc8157fcc6a81df6fb7750a"},
-]
-python-json-logger = []
-pywin32-ctypes = []
-pyyaml = [
-    {file = "PyYAML-6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53"},
-    {file = "PyYAML-6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c"},
-    {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc"},
-    {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b"},
-    {file = "PyYAML-6.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5"},
-    {file = "PyYAML-6.0-cp310-cp310-win32.whl", hash = "sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513"},
-    {file = "PyYAML-6.0-cp310-cp310-win_amd64.whl", hash = "sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a"},
-    {file = "PyYAML-6.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86"},
-    {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f"},
-    {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92"},
-    {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4"},
-    {file = "PyYAML-6.0-cp36-cp36m-win32.whl", hash = "sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293"},
-    {file = "PyYAML-6.0-cp36-cp36m-win_amd64.whl", hash = "sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57"},
-    {file = "PyYAML-6.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c"},
-    {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0"},
-    {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4"},
-    {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9"},
-    {file = "PyYAML-6.0-cp37-cp37m-win32.whl", hash = "sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737"},
-    {file = "PyYAML-6.0-cp37-cp37m-win_amd64.whl", hash = "sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d"},
-    {file = "PyYAML-6.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b"},
-    {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba"},
-    {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34"},
-    {file = "PyYAML-6.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287"},
-    {file = "PyYAML-6.0-cp38-cp38-win32.whl", hash = "sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78"},
-    {file = "PyYAML-6.0-cp38-cp38-win_amd64.whl", hash = "sha256:1e4747bc279b4f613a09eb64bba2ba602d8a6664c6ce6396a4d0cd413a50ce07"},
-    {file = "PyYAML-6.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b"},
-    {file = "PyYAML-6.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174"},
-    {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803"},
-    {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3"},
-    {file = "PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0"},
-    {file = "PyYAML-6.0-cp39-cp39-win32.whl", hash = "sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb"},
-    {file = "PyYAML-6.0-cp39-cp39-win_amd64.whl", hash = "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c"},
-    {file = "PyYAML-6.0.tar.gz", hash = "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2"},
-]
-qrcode = []
-questionary = []
-requests = []
-requests-mock = []
-rfc3986 = [
-    {file = "rfc3986-1.5.0-py2.py3-none-any.whl", hash = "sha256:a86d6e1f5b1dc238b218b012df0aa79409667bb209e58da56d0b94704e712a97"},
-    {file = "rfc3986-1.5.0.tar.gz", hash = "sha256:270aaf10d87d0d4e095063c65bf3ddbc6ee3d0b226328ce21e036f946e421835"},
-]
-s3transfer = []
-six = [
-    {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"},
-    {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"},
-]
-sniffio = []
-toml = [
-    {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"},
-    {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"},
-]
-tomli = [
-    {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"},
-    {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"},
-]
-tqdm = []
-typing-extensions = []
-urllib3 = []
-virtualenv = []
-wcwidth = []
-wrapt = [
+files = [
     {file = "wrapt-1.14.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:1b376b3f4896e7930f1f772ac4b064ac12598d1c38d04907e696cc4d794b43d3"},
     {file = "wrapt-1.14.1-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:903500616422a40a98a5a3c4ff4ed9d0066f3b4c951fa286018ecdf0750194ef"},
     {file = "wrapt-1.14.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:5a9a0d155deafd9448baff28c08e150d9b24ff010e899311ddd63c45c2445e28"},
@@ -1197,6 +1776,125 @@ wrapt = [
     {file = "wrapt-1.14.1-cp39-cp39-win_amd64.whl", hash = "sha256:dee60e1de1898bde3b238f18340eec6148986da0455d8ba7848d50470a7a32fb"},
     {file = "wrapt-1.14.1.tar.gz", hash = "sha256:380a85cf89e0e69b7cfbe2ea9f765f004ff419f34194018a6827ac0e3edfed4d"},
 ]
-xmltodict = []
-yarl = []
-zipp = []
+
+[[package]]
+name = "xmltodict"
+version = "0.13.0"
+description = "Makes working with XML feel like you are working with JSON"
+category = "main"
+optional = false
+python-versions = ">=3.4"
+files = [
+    {file = "xmltodict-0.13.0-py2.py3-none-any.whl", hash = "sha256:aa89e8fd76320154a40d19a0df04a4695fb9dc5ba977cbb68ab3e4eb225e7852"},
+    {file = "xmltodict-0.13.0.tar.gz", hash = "sha256:341595a488e3e01a85a9d8911d8912fd922ede5fecc4dce437eb4b6c8d037e56"},
+]
+
+[[package]]
+name = "yarl"
+version = "1.8.2"
+description = "Yet another URL library"
+category = "main"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "yarl-1.8.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:bb81f753c815f6b8e2ddd2eef3c855cf7da193b82396ac013c661aaa6cc6b0a5"},
+    {file = "yarl-1.8.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:47d49ac96156f0928f002e2424299b2c91d9db73e08c4cd6742923a086f1c863"},
+    {file = "yarl-1.8.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3fc056e35fa6fba63248d93ff6e672c096f95f7836938241ebc8260e062832fe"},
+    {file = "yarl-1.8.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:58a3c13d1c3005dbbac5c9f0d3210b60220a65a999b1833aa46bd6677c69b08e"},
+    {file = "yarl-1.8.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:10b08293cda921157f1e7c2790999d903b3fd28cd5c208cf8826b3b508026996"},
+    {file = "yarl-1.8.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:de986979bbd87272fe557e0a8fcb66fd40ae2ddfe28a8b1ce4eae22681728fef"},
+    {file = "yarl-1.8.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c4fcfa71e2c6a3cb568cf81aadc12768b9995323186a10827beccf5fa23d4f8"},
+    {file = "yarl-1.8.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae4d7ff1049f36accde9e1ef7301912a751e5bae0a9d142459646114c70ecba6"},
+    {file = "yarl-1.8.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:bf071f797aec5b96abfc735ab97da9fd8f8768b43ce2abd85356a3127909d146"},
+    {file = "yarl-1.8.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:74dece2bfc60f0f70907c34b857ee98f2c6dd0f75185db133770cd67300d505f"},
+    {file = "yarl-1.8.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:df60a94d332158b444301c7f569659c926168e4d4aad2cfbf4bce0e8fb8be826"},
+    {file = "yarl-1.8.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:63243b21c6e28ec2375f932a10ce7eda65139b5b854c0f6b82ed945ba526bff3"},
+    {file = "yarl-1.8.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:cfa2bbca929aa742b5084fd4663dd4b87c191c844326fcb21c3afd2d11497f80"},
+    {file = "yarl-1.8.2-cp310-cp310-win32.whl", hash = "sha256:b05df9ea7496df11b710081bd90ecc3a3db6adb4fee36f6a411e7bc91a18aa42"},
+    {file = "yarl-1.8.2-cp310-cp310-win_amd64.whl", hash = "sha256:24ad1d10c9db1953291f56b5fe76203977f1ed05f82d09ec97acb623a7976574"},
+    {file = "yarl-1.8.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:2a1fca9588f360036242f379bfea2b8b44cae2721859b1c56d033adfd5893634"},
+    {file = "yarl-1.8.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f37db05c6051eff17bc832914fe46869f8849de5b92dc4a3466cd63095d23dfd"},
+    {file = "yarl-1.8.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:77e913b846a6b9c5f767b14dc1e759e5aff05502fe73079f6f4176359d832581"},
+    {file = "yarl-1.8.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0978f29222e649c351b173da2b9b4665ad1feb8d1daa9d971eb90df08702668a"},
+    {file = "yarl-1.8.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:388a45dc77198b2460eac0aca1efd6a7c09e976ee768b0d5109173e521a19daf"},
+    {file = "yarl-1.8.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2305517e332a862ef75be8fad3606ea10108662bc6fe08509d5ca99503ac2aee"},
+    {file = "yarl-1.8.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42430ff511571940d51e75cf42f1e4dbdded477e71c1b7a17f4da76c1da8ea76"},
+    {file = "yarl-1.8.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3150078118f62371375e1e69b13b48288e44f6691c1069340081c3fd12c94d5b"},
+    {file = "yarl-1.8.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:c15163b6125db87c8f53c98baa5e785782078fbd2dbeaa04c6141935eb6dab7a"},
+    {file = "yarl-1.8.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:4d04acba75c72e6eb90745447d69f84e6c9056390f7a9724605ca9c56b4afcc6"},
+    {file = "yarl-1.8.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:e7fd20d6576c10306dea2d6a5765f46f0ac5d6f53436217913e952d19237efc4"},
+    {file = "yarl-1.8.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:75c16b2a900b3536dfc7014905a128a2bea8fb01f9ee26d2d7d8db0a08e7cb2c"},
+    {file = "yarl-1.8.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:6d88056a04860a98341a0cf53e950e3ac9f4e51d1b6f61a53b0609df342cc8b2"},
+    {file = "yarl-1.8.2-cp311-cp311-win32.whl", hash = "sha256:fb742dcdd5eec9f26b61224c23baea46c9055cf16f62475e11b9b15dfd5c117b"},
+    {file = "yarl-1.8.2-cp311-cp311-win_amd64.whl", hash = "sha256:8c46d3d89902c393a1d1e243ac847e0442d0196bbd81aecc94fcebbc2fd5857c"},
+    {file = "yarl-1.8.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:ceff9722e0df2e0a9e8a79c610842004fa54e5b309fe6d218e47cd52f791d7ef"},
+    {file = "yarl-1.8.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3f6b4aca43b602ba0f1459de647af954769919c4714706be36af670a5f44c9c1"},
+    {file = "yarl-1.8.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1684a9bd9077e922300ecd48003ddae7a7474e0412bea38d4631443a91d61077"},
+    {file = "yarl-1.8.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ebb78745273e51b9832ef90c0898501006670d6e059f2cdb0e999494eb1450c2"},
+    {file = "yarl-1.8.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3adeef150d528ded2a8e734ebf9ae2e658f4c49bf413f5f157a470e17a4a2e89"},
+    {file = "yarl-1.8.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:57a7c87927a468e5a1dc60c17caf9597161d66457a34273ab1760219953f7f4c"},
+    {file = "yarl-1.8.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:efff27bd8cbe1f9bd127e7894942ccc20c857aa8b5a0327874f30201e5ce83d0"},
+    {file = "yarl-1.8.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:a783cd344113cb88c5ff7ca32f1f16532a6f2142185147822187913eb989f739"},
+    {file = "yarl-1.8.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:705227dccbe96ab02c7cb2c43e1228e2826e7ead880bb19ec94ef279e9555b5b"},
+    {file = "yarl-1.8.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:34c09b43bd538bf6c4b891ecce94b6fa4f1f10663a8d4ca589a079a5018f6ed7"},
+    {file = "yarl-1.8.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:a48f4f7fea9a51098b02209d90297ac324241bf37ff6be6d2b0149ab2bd51b37"},
+    {file = "yarl-1.8.2-cp37-cp37m-win32.whl", hash = "sha256:0414fd91ce0b763d4eadb4456795b307a71524dbacd015c657bb2a39db2eab89"},
+    {file = "yarl-1.8.2-cp37-cp37m-win_amd64.whl", hash = "sha256:d881d152ae0007809c2c02e22aa534e702f12071e6b285e90945aa3c376463c5"},
+    {file = "yarl-1.8.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:5df5e3d04101c1e5c3b1d69710b0574171cc02fddc4b23d1b2813e75f35a30b1"},
+    {file = "yarl-1.8.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:7a66c506ec67eb3159eea5096acd05f5e788ceec7b96087d30c7d2865a243918"},
+    {file = "yarl-1.8.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:2b4fa2606adf392051d990c3b3877d768771adc3faf2e117b9de7eb977741229"},
+    {file = "yarl-1.8.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1e21fb44e1eff06dd6ef971d4bdc611807d6bd3691223d9c01a18cec3677939e"},
+    {file = "yarl-1.8.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:93202666046d9edadfe9f2e7bf5e0782ea0d497b6d63da322e541665d65a044e"},
+    {file = "yarl-1.8.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fc77086ce244453e074e445104f0ecb27530d6fd3a46698e33f6c38951d5a0f1"},
+    {file = "yarl-1.8.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:64dd68a92cab699a233641f5929a40f02a4ede8c009068ca8aa1fe87b8c20ae3"},
+    {file = "yarl-1.8.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1b372aad2b5f81db66ee7ec085cbad72c4da660d994e8e590c997e9b01e44901"},
+    {file = "yarl-1.8.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:e6f3515aafe0209dd17fb9bdd3b4e892963370b3de781f53e1746a521fb39fc0"},
+    {file = "yarl-1.8.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:dfef7350ee369197106805e193d420b75467b6cceac646ea5ed3049fcc950a05"},
+    {file = "yarl-1.8.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:728be34f70a190566d20aa13dc1f01dc44b6aa74580e10a3fb159691bc76909d"},
+    {file = "yarl-1.8.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:ff205b58dc2929191f68162633d5e10e8044398d7a45265f90a0f1d51f85f72c"},
+    {file = "yarl-1.8.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:baf211dcad448a87a0d9047dc8282d7de59473ade7d7fdf22150b1d23859f946"},
+    {file = "yarl-1.8.2-cp38-cp38-win32.whl", hash = "sha256:272b4f1599f1b621bf2aabe4e5b54f39a933971f4e7c9aa311d6d7dc06965165"},
+    {file = "yarl-1.8.2-cp38-cp38-win_amd64.whl", hash = "sha256:326dd1d3caf910cd26a26ccbfb84c03b608ba32499b5d6eeb09252c920bcbe4f"},
+    {file = "yarl-1.8.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:f8ca8ad414c85bbc50f49c0a106f951613dfa5f948ab69c10ce9b128d368baf8"},
+    {file = "yarl-1.8.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:418857f837347e8aaef682679f41e36c24250097f9e2f315d39bae3a99a34cbf"},
+    {file = "yarl-1.8.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ae0eec05ab49e91a78700761777f284c2df119376e391db42c38ab46fd662b77"},
+    {file = "yarl-1.8.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:009a028127e0a1755c38b03244c0bea9d5565630db9c4cf9572496e947137a87"},
+    {file = "yarl-1.8.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3edac5d74bb3209c418805bda77f973117836e1de7c000e9755e572c1f7850d0"},
+    {file = "yarl-1.8.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:da65c3f263729e47351261351b8679c6429151ef9649bba08ef2528ff2c423b2"},
+    {file = "yarl-1.8.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0ef8fb25e52663a1c85d608f6dd72e19bd390e2ecaf29c17fb08f730226e3a08"},
+    {file = "yarl-1.8.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bcd7bb1e5c45274af9a1dd7494d3c52b2be5e6bd8d7e49c612705fd45420b12d"},
+    {file = "yarl-1.8.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:44ceac0450e648de86da8e42674f9b7077d763ea80c8ceb9d1c3e41f0f0a9951"},
+    {file = "yarl-1.8.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:97209cc91189b48e7cfe777237c04af8e7cc51eb369004e061809bcdf4e55220"},
+    {file = "yarl-1.8.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:48dd18adcf98ea9cd721a25313aef49d70d413a999d7d89df44f469edfb38a06"},
+    {file = "yarl-1.8.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:e59399dda559688461762800d7fb34d9e8a6a7444fd76ec33220a926c8be1516"},
+    {file = "yarl-1.8.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:d617c241c8c3ad5c4e78a08429fa49e4b04bedfc507b34b4d8dceb83b4af3588"},
+    {file = "yarl-1.8.2-cp39-cp39-win32.whl", hash = "sha256:cb6d48d80a41f68de41212f3dfd1a9d9898d7841c8f7ce6696cf2fd9cb57ef83"},
+    {file = "yarl-1.8.2-cp39-cp39-win_amd64.whl", hash = "sha256:6604711362f2dbf7160df21c416f81fac0de6dbcf0b5445a2ef25478ecc4c778"},
+    {file = "yarl-1.8.2.tar.gz", hash = "sha256:49d43402c6e3013ad0978602bf6bf5328535c48d192304b91b97a3c6790b1562"},
+]
+
+[package.dependencies]
+idna = ">=2.0"
+multidict = ">=4.0"
+typing-extensions = {version = ">=3.7.4", markers = "python_version < \"3.8\""}
+
+[[package]]
+name = "zipp"
+version = "3.11.0"
+description = "Backport of pathlib-compatible object wrapper for zip files"
+category = "main"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "zipp-3.11.0-py3-none-any.whl", hash = "sha256:83a28fcb75844b5c0cdaf5aa4003c2d728c77e05f5aeabe8e95e56727005fbaa"},
+    {file = "zipp-3.11.0.tar.gz", hash = "sha256:a7a22e05929290a67401440b39690ae6563279bced5f314609d9d03798f56766"},
+]
+
+[package.extras]
+docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)"]
+testing = ["flake8 (<5)", "func-timeout", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"]
+
+[metadata]
+lock-version = "2.0"
+python-versions = ">=3.7,<3.11"
+content-hash = "7b2376e5fb3097faae5241e72a82dbc1c68b6bf2b63da6698b2fe0eb1dde4dc1"
diff --git a/pyproject.toml b/pyproject.toml
index 1adb61ca..f2595b40 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -20,6 +20,7 @@ pyinstaller = "^5.4.1"
 httpx = "^0.23.0"
 pilot-platform-common = "^0.1.3"
 qrcode = "^7.4.2"
+pytest-click = "^1.1.0"
 
 [tool.poetry.dev-dependencies]
 pytest = "6.2.5"
diff --git a/tests/app/commands/test_file.py b/tests/app/commands/test_file.py
new file mode 100644
index 00000000..55aa78f1
--- /dev/null
+++ b/tests/app/commands/test_file.py
@@ -0,0 +1,27 @@
+# Copyright (C) 2022-2023 Indoc Research
+#
+# Contact Indoc Research for any questions regarding the use of this source code.
+
+from app.commands.file import file_resume
+from app.services.output_manager.error_handler import ECustomizedError
+from app.services.output_manager.error_handler import customized_error_msg
+
+
+def test_resumable_upload_command_success(mocker, cli_runner):
+    mocker.patch('os.path.exists', return_value=True)
+    # mock the open function
+    mocked_open_data = mocker.mock_open(read_data='test')
+    mocker.patch('builtins.open', mocked_open_data)
+    mocker.patch('json.load', return_value={'resumable_file': 'test.json', 'thread': 1})
+    mocker.patch('app.commands.file.resume_upload', return_value=None)
+
+    result = cli_runner.invoke(file_resume, ['--resumable-file', 'test.json', '--thread', 1])
+    assert result.exit_code == 0
+
+
+def test_resumable_upload_command_failed_with_file_not_exists(mocker, cli_runner):
+    mocker.patch('os.path.exists', return_value=False)
+
+    result = cli_runner.invoke(file_resume, ['--resumable-file', 'test.json', '--thread', 1])
+    assert result.exit_code == 0
+    assert result.output == customized_error_msg(ECustomizedError.INVALID_RESUMABLE) + '\n'

From 319673220854517246b80db16bb409584ddaed59 Mon Sep 17 00:00:00 2001
From: zhiren <zzhan@indocresearch.org>
Date: Tue, 4 Apr 2023 16:10:25 -0400
Subject: [PATCH 04/62] add the test case for resume upload command

---
 app/services/file_manager/file_upload/file_upload.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/services/file_manager/file_upload/file_upload.py b/app/services/file_manager/file_upload/file_upload.py
index 2bb35ead..2e43de7a 100644
--- a/app/services/file_manager/file_upload/file_upload.py
+++ b/app/services/file_manager/file_upload/file_upload.py
@@ -247,7 +247,7 @@ def resume_upload(
     # make them as FileObject
     unfinished_items = [
         FileObject(
-            x.get('resumable_id'), x.get('item_id'), x.get('job_id'), x.get('object_path'), x.get('local_paht'), []
+            x.get('resumable_id'), x.get('item_id'), x.get('job_id'), x.get('object_path'), x.get('local_path'), []
         )
         for x in unfinished_items
     ]

From 69f6b6710e88437328b5e13e0a89502ede31eb27 Mon Sep 17 00:00:00 2001
From: zhiren <zzhan@indocresearch.org>
Date: Tue, 4 Apr 2023 16:32:54 -0400
Subject: [PATCH 05/62] update the test case for resume uploading

---
 .../file_manager/file_upload/file_upload.py   |  7 ++--
 .../file_manager/file_upload/models.py        | 11 +++++++
 .../file_upload/test_file_upload.py           | 33 +++++++++++++++++++
 3 files changed, 49 insertions(+), 2 deletions(-)

diff --git a/app/services/file_manager/file_upload/file_upload.py b/app/services/file_manager/file_upload/file_upload.py
index 2e43de7a..ec33a4b5 100644
--- a/app/services/file_manager/file_upload/file_upload.py
+++ b/app/services/file_manager/file_upload/file_upload.py
@@ -17,6 +17,7 @@
 import app.services.output_manager.message_handler as mhandler
 from app.configs.app_config import AppConfig
 from app.services.file_manager.file_upload.models import FileObject
+from app.services.file_manager.file_upload.models import ItemStatus
 from app.services.file_manager.file_upload.models import UploadType
 from app.services.file_manager.file_upload.upload_client import UploadClient
 from app.services.output_manager.error_handler import ECustomizedError
@@ -241,9 +242,11 @@ def resume_upload(
     )
 
     # check files in manifest if some of them are already uploaded
-    item_ids = [x.get('item_id') for x in manifest_json.get('file_objects')]
+    item_ids = []
+    for item_id in manifest_json.get('file_objects'):
+        item_ids.append(item_id)
     items = get_file_info_by_geid(item_ids)
-    unfinished_items = [x for x in items if x.get('status') == 'REGISTERED']  # update to enum later
+    unfinished_items = [x for x in items if x.get('status') == ItemStatus.REGISTERED]  # update to enum later
     # make them as FileObject
     unfinished_items = [
         FileObject(
diff --git a/app/services/file_manager/file_upload/models.py b/app/services/file_manager/file_upload/models.py
index 918f04c7..e9ddf1be 100644
--- a/app/services/file_manager/file_upload/models.py
+++ b/app/services/file_manager/file_upload/models.py
@@ -21,6 +21,17 @@ def __str__(self):
         return '%s' % self.name
 
 
+class ItemStatus(str, Enum):
+    # the new enum type for file status
+
+    REGISTERED = 'REGISTERED'  # file is created by upload service but not complete yet. either in progress or fail.
+    ACTIVE = 'ACTIVE'  # file uploading is complete.
+    ARCHIVED = 'ARCHIVED'  # the file has been deleted
+
+    def __str__(self):
+        return '%s' % self.name
+
+
 class FileObject:
     """
     Summary:
diff --git a/tests/app/services/file_manager/file_upload/test_file_upload.py b/tests/app/services/file_manager/file_upload/test_file_upload.py
index 46569a61..1c4a121f 100644
--- a/tests/app/services/file_manager/file_upload/test_file_upload.py
+++ b/tests/app/services/file_manager/file_upload/test_file_upload.py
@@ -2,7 +2,11 @@
 #
 # Contact Indoc Research for any questions regarding the use of this source code.
 
+from app.configs.app_config import AppConfig
 from app.services.file_manager.file_upload.file_upload import assemble_path
+from app.services.file_manager.file_upload.file_upload import resume_upload
+from app.services.file_manager.file_upload.models import FileObject
+from app.services.file_manager.file_upload.models import ItemStatus
 
 
 def test_assemble_path_at_name_folder(mocker):
@@ -100,3 +104,32 @@ def test_assemble_path_at_non_existing_folder(mocker):
     assert current_file_path == 'admin/test_folder_not_exist/file.txt'
     assert parent_folder.get('name') == 'admin'
     assert create_folder_flag is True
+
+
+def test_resume_upload(mocker):
+    mocker.patch('app.services.file_manager.file_upload.models.FileObject.generate_meta', return_value=(1, 1))
+    test_obj = FileObject('resumable_id', 'job_id', 'item_id', 'object/path', 'local_path', [])
+
+    manifest_json = {
+        'project_code': 'project_code',
+        'operator': 'operator',
+        'zone': AppConfig.Env.green_zone,
+        'parent_folder_id': 'parent_folder_id',
+        'current_folder_node': 'current_folder_node',
+        'tags': 'tags',
+        'file_objects': {test_obj.item_id: test_obj.to_dict()},
+    }
+
+    get_return = test_obj.to_dict()
+    get_return.update({'status': ItemStatus.REGISTERED})
+    get_mock = mocker.patch(
+        'app.services.file_manager.file_upload.file_upload.get_file_info_by_geid', return_value=[get_return]
+    )
+    resume_upload_mock = mocker.patch(
+        'app.services.file_manager.file_upload.file_upload.UploadClient.resume_upload', return_value=[]
+    )
+
+    resume_upload(manifest_json, 1)
+
+    get_mock.assert_called_once()
+    resume_upload_mock.assert_called_once()

From 440edd1a6772d7aad76fdcce2c7c6bf79d79ccca Mon Sep 17 00:00:00 2001
From: zhiren <zzhan@indocresearch.org>
Date: Wed, 5 Apr 2023 10:12:44 -0400
Subject: [PATCH 06/62] remove the logic of resumable file upload from normal
 upload

---
 app/commands/file.py                          |  7 +----
 .../file_manager/file_upload/file_upload.py   | 26 +++++++------------
 2 files changed, 10 insertions(+), 23 deletions(-)

diff --git a/app/commands/file.py b/app/commands/file.py
index 5d3ae93b..eb751d52 100644
--- a/app/commands/file.py
+++ b/app/commands/file.py
@@ -140,9 +140,6 @@ def file_put(**kwargs):  # noqa: C901
     # check if user input at least one file/folder
     if len(paths) == 0:
         SrvErrorHandler.customized_handle(ECustomizedError.INVALID_PATHS, True)
-    # # check if resumable_id exist then job_id should also be inputed
-    # if (resumable_id is None) != (job_id is None):
-    #     SrvErrorHandler.customized_handle(ECustomizedError.INVALID_RESUMABLE, True)
 
     project_path = click.prompt('ProjectCode') if not project_path else project_path
     project_code, target_folder = identify_target_folder(project_path)
@@ -199,9 +196,7 @@ def file_put(**kwargs):  # noqa: C901
         if source_file:
             upload_event['valid_source'] = src_file_info
 
-        simple_upload(
-            upload_event, num_of_thread=thread, resumable_id=None, job_id=None, item_id=None, output_path=output_path
-        )
+        simple_upload(upload_event, num_of_thread=thread, output_path=output_path)
 
         srv_manifest.attach_manifest(attribute, result_file, zone) if attribute else None
         message_handler.SrvOutPutHandler.all_file_uploaded()
diff --git a/app/services/file_manager/file_upload/file_upload.py b/app/services/file_manager/file_upload/file_upload.py
index ec33a4b5..e65f5c03 100644
--- a/app/services/file_manager/file_upload/file_upload.py
+++ b/app/services/file_manager/file_upload/file_upload.py
@@ -104,9 +104,6 @@ def assemble_path(
 def simple_upload(  # noqa: C901
     upload_event,
     num_of_thread: int = 1,
-    resumable_id: str = None,
-    job_id: str = None,
-    item_id: str = None,
     output_path: str = None,
 ):
     upload_start_time = time.time()
@@ -134,7 +131,7 @@ def simple_upload(  # noqa: C901
             upload_file_path = [my_file.rstrip('/').lstrip() + '.zip']
             target_folder = '/'.join(target_folder.split('/')[:-1]).rstrip('/')
             compress_folder_to_zip(my_file)
-        elif job_type == UploadType.AS_FOLDER and resumable_id:
+        elif job_type == UploadType.AS_FOLDER:
             SrvErrorHandler.customized_handle(ECustomizedError.UNSUPPORTED_PROJECT, True, project_code)
         else:
             logger.warning('Current version does not support folder tagging, ' 'any selected tags will be ignored')
@@ -167,19 +164,14 @@ def simple_upload(  # noqa: C901
     # the result will store as (UploaderObject, preupload_id_mapping)
     pre_upload_infos = []
 
-    # TODO later will adapt the folder resumable upload
-    # for now it is only for file resumable
-    if resumable_id and job_id:
-        pre_upload_infos.extend(upload_client.resume_upload(resumable_id, job_id, item_id, upload_file_path[0]))
-    else:
-        for batch in range(0, num_of_batchs):
-            start_index = batch * AppConfig.Env.upload_batch_size
-            end_index = (batch + 1) * AppConfig.Env.upload_batch_size
-            file_batchs = upload_file_path[start_index:end_index]
-
-            # sending the pre upload request to generate
-            # the placeholder in object storage
-            pre_upload_infos.extend(upload_client.pre_upload(file_batchs, output_path))
+    for batch in range(0, num_of_batchs):
+        start_index = batch * AppConfig.Env.upload_batch_size
+        end_index = (batch + 1) * AppConfig.Env.upload_batch_size
+        file_batchs = upload_file_path[start_index:end_index]
+
+        # sending the pre upload request to generate
+        # the placeholder in object storage
+        pre_upload_infos.extend(upload_client.pre_upload(file_batchs, output_path))
 
     # now loop over each file under the folder and start
     # the chunk upload

From 5fd12f7e33698deaa6b8c20e5a9643b140e793e5 Mon Sep 17 00:00:00 2001
From: zhiren <zzhan@indocresearch.org>
Date: Wed, 5 Apr 2023 10:20:46 -0400
Subject: [PATCH 07/62] backup

---
 app/commands/file.py                                   | 2 ++
 app/services/file_manager/file_upload/upload_client.py | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/app/commands/file.py b/app/commands/file.py
index eb751d52..1e938f12 100644
--- a/app/commands/file.py
+++ b/app/commands/file.py
@@ -167,6 +167,8 @@ def file_put(**kwargs):  # noqa: C901
         if not upload_message:
             upload_message = AppConfig.Env.default_upload_message
 
+    raise Exception('test')
+
     # Unique Paths
     paths = set(paths)
     # the loop will read all input path(folder or files)
diff --git a/app/services/file_manager/file_upload/upload_client.py b/app/services/file_manager/file_upload/upload_client.py
index 41dcc768..d6657056 100644
--- a/app/services/file_manager/file_upload/upload_client.py
+++ b/app/services/file_manager/file_upload/upload_client.py
@@ -151,7 +151,7 @@ def resume_upload(self, unfinished_file_objects: List[FileObject]) -> List[FileO
 
         return unfinished_file_objects
 
-    # @require_valid_token()
+    @require_valid_token()
     def pre_upload(self, local_file_paths: List[str], output_path: str) -> List[FileObject]:
         """
         Summary:

From 13479a4159e1506ac4857e1c32d10e32849532db Mon Sep 17 00:00:00 2001
From: zhiren <zzhan@indocresearch.org>
Date: Wed, 5 Apr 2023 10:21:26 -0400
Subject: [PATCH 08/62] add back the token validation for pre upload

---
 app/services/file_manager/file_upload/upload_client.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/services/file_manager/file_upload/upload_client.py b/app/services/file_manager/file_upload/upload_client.py
index 41dcc768..d6657056 100644
--- a/app/services/file_manager/file_upload/upload_client.py
+++ b/app/services/file_manager/file_upload/upload_client.py
@@ -151,7 +151,7 @@ def resume_upload(self, unfinished_file_objects: List[FileObject]) -> List[FileO
 
         return unfinished_file_objects
 
-    # @require_valid_token()
+    @require_valid_token()
     def pre_upload(self, local_file_paths: List[str], output_path: str) -> List[FileObject]:
         """
         Summary:

From 6f4227e121d3167b39675aa4a74539aaaaa386ec Mon Sep 17 00:00:00 2001
From: zhiren <zzhan@indocresearch.org>
Date: Wed, 5 Apr 2023 11:06:08 -0400
Subject: [PATCH 09/62] testing on the logic

---
 app/commands/file.py                          | 18 ++++++++++++++-
 app/models/upload_form.py                     |  2 ++
 app/services/crypto/crypto.py                 |  3 ++-
 .../file_manager/file_upload/file_upload.py   |  8 +++++++
 .../file_manager/file_upload/upload_client.py | 23 ++++---------------
 5 files changed, 33 insertions(+), 21 deletions(-)

diff --git a/app/commands/file.py b/app/commands/file.py
index 1e938f12..bc8928e0 100644
--- a/app/commands/file.py
+++ b/app/commands/file.py
@@ -167,7 +167,16 @@ def file_put(**kwargs):  # noqa: C901
         if not upload_message:
             upload_message = AppConfig.Env.default_upload_message
 
-    raise Exception('test')
+    # for the path formating there will be following cases:
+    # - file:
+    #   1. the project path exist, then will be AS_FILE. nothing will be changed
+    #   2. the project path not exist, then will be AS_FOLDER. the current folder node will
+    #      be the parent folder node + parent folder id. (like one level up).
+    # - folder:
+    #   1. the project path exist, then will be AS_FOLDER. the current folder node will be
+    #      the one that user input.
+    #   2. the project path not exist, then will be AS_FOLDER. the current folder node will
+    #      be the parent folder node + parent folder id. (like one level up).
 
     # Unique Paths
     paths = set(paths)
@@ -181,6 +190,12 @@ def file_put(**kwargs):  # noqa: C901
             zone,
             zipping,
         )
+
+        current_folder_node = 'testproject/admin'
+        parent_folder = {'id': 'testproject'}
+        create_folder_flag = False
+        result_file = None
+
         upload_event = {
             'project_code': project_code,
             'file': f,
@@ -193,6 +208,7 @@ def file_put(**kwargs):  # noqa: C901
             'compress_zip': zipping,
             'attribute': attribute,
         }
+        # print(upload_event)
         if pipeline:
             upload_event['process_pipeline'] = pipeline
         if source_file:
diff --git a/app/models/upload_form.py b/app/models/upload_form.py
index fcbd22c5..358b4b19 100644
--- a/app/models/upload_form.py
+++ b/app/models/upload_form.py
@@ -129,6 +129,8 @@ def generate_pre_upload_form(
         - request_payload(dict): the payload for preupload api.
         - local_file_mapping(dict): the mapping from object path into local path.
     """
+    # print(project_code, operator, local_file_paths, input_path, zone, job_type, current_folder)
+
     data, local_file_mapping = [], {}
     for file_local_path in local_file_paths:
         # the rule here is:
diff --git a/app/services/crypto/crypto.py b/app/services/crypto/crypto.py
index 026d2d87..81fb6792 100644
--- a/app/services/crypto/crypto.py
+++ b/app/services/crypto/crypto.py
@@ -76,4 +76,5 @@ def decryption(encrypted_message, secret, interactive=True):
             else:
                 raise ex
     else:
-        ehandler.SrvErrorHandler.customized_handle(ehandler.ECustomizedError.LOGIN_SESSION_INVALID, True)
+        pass
+        # ehandler.SrvErrorHandler.customized_handle(ehandler.ECustomizedError.LOGIN_SESSION_INVALID, True)
diff --git a/app/services/file_manager/file_upload/file_upload.py b/app/services/file_manager/file_upload/file_upload.py
index e65f5c03..585996c5 100644
--- a/app/services/file_manager/file_upload/file_upload.py
+++ b/app/services/file_manager/file_upload/file_upload.py
@@ -146,6 +146,12 @@ def simple_upload(  # noqa: C901
         else:
             job_type = UploadType.AS_FILE
 
+    # print('upload_file_path:', upload_file_path)
+    # print('target_folder:', target_folder)
+    # print('my_file:', my_file)
+    # print('job_type:', job_type)
+    # print('zone:', zone)
+
     upload_client = UploadClient(
         input_path=my_file,
         project_code=project_code,
@@ -157,6 +163,8 @@ def simple_upload(  # noqa: C901
         tags=tags,
     )
 
+    # print()
+
     # here add the batch of 500 per loop, the pre upload api cannot
     # process very large amount of file at same time. otherwise it will timeout
     num_of_batchs = math.ceil(len(upload_file_path) / AppConfig.Env.upload_batch_size)
diff --git a/app/services/file_manager/file_upload/upload_client.py b/app/services/file_manager/file_upload/upload_client.py
index d6657056..df990d73 100644
--- a/app/services/file_manager/file_upload/upload_client.py
+++ b/app/services/file_manager/file_upload/upload_client.py
@@ -4,14 +4,12 @@
 
 import hashlib
 import json
-import math
 import os
 import time
 from multiprocessing.pool import ThreadPool
 from typing import Any
 from typing import Dict
 from typing import List
-from typing import Tuple
 
 import httpx
 
@@ -90,21 +88,6 @@ def __init__(
         # then the token refresh loop will end
         self.finish_upload = False
 
-    def generate_meta(self, local_path: str) -> Tuple[int, int]:
-        """
-        Summary:
-            The function is to generate chunk upload meatedata for a file.
-        Parameter:
-            - input_path: The path of the local file eg. a/b/c.txt.
-        return:
-            - total_size: the size of file.
-            - total_chunks: the number of chunks will be uploaded.
-        """
-        file_length_in_bytes = os.path.getsize(local_path)
-        total_size = file_length_in_bytes
-        total_chunks = math.ceil(total_size / self.chunk_size)
-        return total_size, total_chunks
-
     # @require_valid_token()
     def resume_upload(self, unfinished_file_objects: List[FileObject]) -> List[FileObject]:
         """
@@ -151,7 +134,7 @@ def resume_upload(self, unfinished_file_objects: List[FileObject]) -> List[FileO
 
         return unfinished_file_objects
 
-    @require_valid_token()
+    # @require_valid_token()
     def pre_upload(self, local_file_paths: List[str], output_path: str) -> List[FileObject]:
         """
         Summary:
@@ -166,7 +149,7 @@ def pre_upload(self, local_file_paths: List[str], output_path: str) -> List[File
                 - local_path(str): the local path of file.
                 - chunk_info(dict): the mapping for chunks that already been uploaded.
         """
-
+        # print('pre upload')
         headers = {'Authorization': 'Bearer ' + self.user.access_token, 'Session-ID': self.user.session_id}
         url = AppConfig.Connections.url_bff + '/v1/project/{}/files'.format(self.project_code)
         # the file mapping is a dictionary that present the map from object storage path
@@ -183,6 +166,8 @@ def pre_upload(self, local_file_paths: List[str], output_path: str) -> List[File
 
         payload.update({'parent_folder_id': self.parent_folder_id})
         payload.update({'folder_tags': self.tags})
+        # print('pre upload payload: ', payload)
+        # raise Exception('pre upload')
         response = resilient_session().post(url, json=payload, headers=headers, timeout=None)
         if response.status_code == 200:
             result = response.json().get('result')

From 15aeaea8693db7b1db93cda43ce246227c28f968 Mon Sep 17 00:00:00 2001
From: zhiren <zzhan@indocresearch.org>
Date: Wed, 5 Apr 2023 16:22:31 -0400
Subject: [PATCH 10/62] refactor with upload logic

---
 app/commands/file.py                          |   7 +-
 app/models/upload_form.py                     | 133 +++++++++---------
 .../file_manager/file_upload/file_upload.py   |  40 +++---
 .../file_manager/file_upload/models.py        |   8 +-
 .../file_manager/file_upload/upload_client.py |  25 ++--
 5 files changed, 112 insertions(+), 101 deletions(-)

diff --git a/app/commands/file.py b/app/commands/file.py
index bc8928e0..85dc0094 100644
--- a/app/commands/file.py
+++ b/app/commands/file.py
@@ -183,6 +183,7 @@ def file_put(**kwargs):  # noqa: C901
     # the loop will read all input path(folder or files)
     # and process them one by one
     for f in paths:
+        # so this function will always return the furthest folder node as current_folder_node+parent_folder_id
         current_folder_node, parent_folder, create_folder_flag, result_file = assemble_path(
             f,
             target_folder,
@@ -191,14 +192,14 @@ def file_put(**kwargs):  # noqa: C901
             zipping,
         )
 
-        current_folder_node = 'testproject/admin'
+        current_folder_node = 'testproject/admin/test11'
         parent_folder = {'id': 'testproject'}
-        create_folder_flag = False
+        create_folder_flag = True
         result_file = None
 
         upload_event = {
             'project_code': project_code,
-            'file': f,
+            'file': f.rstrip('/'),  # remove the ending slash
             'tags': tag if tag else [],
             'zone': zone,
             'upload_message': upload_message,
diff --git a/app/models/upload_form.py b/app/models/upload_form.py
index 358b4b19..85ecadde 100644
--- a/app/models/upload_form.py
+++ b/app/models/upload_form.py
@@ -2,13 +2,14 @@
 #
 # Contact Indoc Research for any questions regarding the use of this source code.
 
-from os.path import basename
-from os.path import dirname
-from os.path import join
+# from os.path import basename
+# from os.path import dirname
+# from os.path import join
 from typing import List
 
 from app.services.file_manager.file_upload.models import FileObject
-from app.services.file_manager.file_upload.models import UploadType
+
+# from app.services.file_manager.file_upload.models import UploadType
 
 
 class FileUploadForm:
@@ -103,68 +104,68 @@ def metadatas(self, metadatas):
         self._attribute_map['metadatas'] = metadatas
 
 
-def generate_pre_upload_form(
-    project_code: str,
-    operator: str,
-    local_file_paths: List[str],
-    input_path: str,
-    zone: str,
-    job_type: UploadType,
-    current_folder: str = '',
-) -> tuple[dict, dict]:
-    """
-    Summary:
-        The function is to generate the preupload payload for api. The operation
-        is per batch that it will try to generate one payload for all files.
-    Parameter:
-        - project_code(str): The unique identifier for project.
-        - operator(str): The name of operator.
-        - local_file_paths(list[str]): The list of name for input files.
-        - input_path: The path specified by user, if it is folder, it will be like
-            a/b . If it is a file it will be same as local_file_paths eg. a/b/c.txt.
-        - zone(str): The zone of user try to upload to.
-        - job_type(UploadType): the upload type, AS_FOLDER or AS_FILE.
-        - current_folder(str): the folder path on object storage that user specified.
-    return:
-        - request_payload(dict): the payload for preupload api.
-        - local_file_mapping(dict): the mapping from object path into local path.
-    """
-    # print(project_code, operator, local_file_paths, input_path, zone, job_type, current_folder)
-
-    data, local_file_mapping = [], {}
-    for file_local_path in local_file_paths:
-        # the rule here is:
-        # - if use input as a folder then <input_path> is the folder user key in
-        #   eg. a/b/ . the <local_file_paths> is files under eg a/b/c/d.txt. The
-        #   path in object storage will be <current_folder>/c/d.txt
-        # - if use input as a file then <input_path> is the file user key in eg.
-        #   a/b/c/d.txt. the <local_file_paths> will be same as it. The path in
-        #   object storage will be <current_folder>/d.txt
-        if job_type == UploadType.AS_FOLDER:
-            file_relative_path = file_local_path.replace(input_path + '/', '')
-            object_path = join(current_folder, file_relative_path)
-            parent_path, file_name = dirname(object_path), basename(object_path)
-
-        else:
-            file_name = basename(file_local_path)
-            parent_path = current_folder
-
-        data.append({'resumable_filename': file_name, 'resumable_relative_path': parent_path})
-        # make a mapping as <object_path>: <local_path>. This will be returned
-        # and used in chunk upload api.
-        object_path = join(parent_path, file_name)
-        local_file_mapping.update({object_path: file_local_path})
-
-    request_payload = {
-        'project_code': project_code,
-        'operator': operator,
-        'job_type': str(job_type),
-        'zone': zone,
-        'current_folder_node': current_folder,
-        'data': data,
-    }
-
-    return request_payload, local_file_mapping
+# def generate_pre_upload_form(
+#     project_code: str,
+#     operator: str,
+#     local_file_paths: List[str],
+#     input_path: str,
+#     zone: str,
+#     job_type: UploadType,
+#     current_folder: str = '',
+# ) -> tuple[dict, dict]:
+#     """
+#     Summary:
+#         The function is to generate the preupload payload for api. The operation
+#         is per batch that it will try to generate one payload for all files.
+#     Parameter:
+#         - project_code(str): The unique identifier for project.
+#         - operator(str): The name of operator.
+#         - local_file_paths(list[str]): The list of name for input files.
+#         - input_path: The path specified by user, if it is folder, it will be like
+#             a/b . If it is a file it will be same as local_file_paths eg. a/b/c.txt.
+#         - zone(str): The zone of user try to upload to.
+#         - job_type(UploadType): the upload type, AS_FOLDER or AS_FILE.
+#         - current_folder(str): the folder path on object storage that user specified.
+#     return:
+#         - request_payload(dict): the payload for preupload api.
+#         - local_file_mapping(dict): the mapping from object path into local path.
+#     """
+#     print(project_code, operator, local_file_paths, input_path, zone, job_type, current_folder)
+
+#     data, local_file_mapping = [], {}
+#     for file_local_path in local_file_paths:
+#         # the rule here is:
+#         # - if use input as a folder then <input_path> is the folder user key in
+#         #   eg. a/b/ . the <local_file_paths> is files under eg a/b/c/d.txt. The
+#         #   path in object storage will be <current_folder>/c/d.txt
+#         # - if use input as a file then <input_path> is the file user key in eg.
+#         #   a/b/c/d.txt. the <local_file_paths> will be same as it. The path in
+#         #   object storage will be <current_folder>/d.txt
+#         if job_type == UploadType.AS_FOLDER:
+#             file_relative_path = file_local_path.replace(input_path + '/', '')
+#             object_path = join(current_folder, file_relative_path)
+#             parent_path, file_name = dirname(object_path), basename(object_path)
+
+#         else:
+#             file_name = basename(file_local_path)
+#             parent_path = current_folder
+
+#         data.append({'resumable_filename': file_name, 'resumable_relative_path': parent_path})
+#         # make a mapping as <object_path>: <local_path>. This will be returned
+#         # and used in chunk upload api.
+#         object_path = join(parent_path, file_name)
+#         local_file_mapping.update({object_path: file_local_path})
+
+#     request_payload = {
+#         'project_code': project_code,
+#         'operator': operator,
+#         'job_type': str(job_type),
+#         'zone': zone,
+#         'current_folder_node': current_folder,
+#         'data': data,
+#     }
+
+#     return request_payload, local_file_mapping
 
 
 def generate_on_success_form(
diff --git a/app/services/file_manager/file_upload/file_upload.py b/app/services/file_manager/file_upload/file_upload.py
index 585996c5..de143610 100644
--- a/app/services/file_manager/file_upload/file_upload.py
+++ b/app/services/file_manager/file_upload/file_upload.py
@@ -107,7 +107,7 @@ def simple_upload(  # noqa: C901
     output_path: str = None,
 ):
     upload_start_time = time.time()
-    my_file = upload_event.get('file')
+    input_path = upload_event.get('file')
     project_code = upload_event.get('project_code')
     tags = upload_event.get('tags')
     zone = upload_event.get('zone')
@@ -121,39 +121,37 @@ def simple_upload(  # noqa: C901
     source_file = upload_event.get('valid_source')
     attribute = upload_event.get('attribute')
 
-    mhandler.SrvOutPutHandler.start_uploading(my_file)
+    mhandler.SrvOutPutHandler.start_uploading(input_path)
     # TODO: PILOT-2392 simplify the logic under
     # if the input request zip folder then process the path as single file
     # otherwise read throught the folder to get path underneath
-    if os.path.isdir(my_file):
+    if os.path.isdir(input_path):
         job_type = UploadType.AS_FILE if compress_zip else UploadType.AS_FOLDER
         if job_type == UploadType.AS_FILE:
-            upload_file_path = [my_file.rstrip('/').lstrip() + '.zip']
+            upload_file_path = [input_path.rstrip('/').lstrip() + '.zip']
             target_folder = '/'.join(target_folder.split('/')[:-1]).rstrip('/')
-            compress_folder_to_zip(my_file)
-        elif job_type == UploadType.AS_FOLDER:
-            SrvErrorHandler.customized_handle(ECustomizedError.UNSUPPORTED_PROJECT, True, project_code)
+            compress_folder_to_zip(input_path)
         else:
             logger.warning('Current version does not support folder tagging, ' 'any selected tags will be ignored')
-            upload_file_path = get_file_in_folder(my_file)
+            upload_file_path = get_file_in_folder(input_path)
     else:
-        upload_file_path = [my_file]
-        target_folder = '/'.join(target_folder.split('/')[:-1]).rstrip('/')
+        upload_file_path = [input_path]
 
         if create_folder_flag:
             job_type = UploadType.AS_FOLDER
-            my_file = os.path.dirname(my_file)  # update the path as folder
+            input_path = os.path.dirname(input_path)  # update the path as folder
+            target_folder = '/'.join(target_folder.split('/')[:-1]).rstrip('/')
         else:
             job_type = UploadType.AS_FILE
 
     # print('upload_file_path:', upload_file_path)
     # print('target_folder:', target_folder)
-    # print('my_file:', my_file)
+    # print('input_path:', input_path)
     # print('job_type:', job_type)
     # print('zone:', zone)
 
     upload_client = UploadClient(
-        input_path=my_file,
+        input_path=input_path,
         project_code=project_code,
         zone=zone,
         job_type=job_type,
@@ -163,19 +161,23 @@ def simple_upload(  # noqa: C901
         tags=tags,
     )
 
-    # print()
+    # format the local path into object storage path for preupload
+    file_objects = []
+    for file in upload_file_path:
+        # first remove the input path from the file path
+        file_path_sub = file.replace(input_path + '/', '')
+        object_path = os.path.join(target_folder, file_path_sub)
+        file_objects.append(FileObject(object_path, file, None))
 
     # here add the batch of 500 per loop, the pre upload api cannot
     # process very large amount of file at same time. otherwise it will timeout
-    num_of_batchs = math.ceil(len(upload_file_path) / AppConfig.Env.upload_batch_size)
+    num_of_batchs = math.ceil(len(file_objects) / AppConfig.Env.upload_batch_size)
     # here is list of pre upload result. We decided to call pre upload api by batch
-    # the result will store as (UploaderObject, preupload_id_mapping)
     pre_upload_infos = []
-
     for batch in range(0, num_of_batchs):
         start_index = batch * AppConfig.Env.upload_batch_size
         end_index = (batch + 1) * AppConfig.Env.upload_batch_size
-        file_batchs = upload_file_path[start_index:end_index]
+        file_batchs = file_objects[start_index:end_index]
 
         # sending the pre upload request to generate
         # the placeholder in object storage
@@ -212,7 +214,7 @@ def simple_upload(  # noqa: C901
             time.sleep(0.5)
         if source_file:
             upload_client.create_file_lineage(source_file)
-            os.remove(file_batchs[0]) if os.path.isdir(my_file) and job_type == UploadType.AS_FILE else None
+            os.remove(file_batchs[0]) if os.path.isdir(input_path) and job_type == UploadType.AS_FILE else None
 
     num_of_file = len(upload_file_path)
     logger.info(f'Upload Time: {time.time() - upload_start_time:.2f}s for {num_of_file:d} files')
diff --git a/app/services/file_manager/file_upload/models.py b/app/services/file_manager/file_upload/models.py
index e9ddf1be..b2e7e585 100644
--- a/app/services/file_manager/file_upload/models.py
+++ b/app/services/file_manager/file_upload/models.py
@@ -54,7 +54,13 @@ class FileObject:
     uploaded_chunks: List[dict]
 
     def __init__(
-        self, resumable_id: str, job_id: str, item_id: str, object_path: str, local_path: str, uploaded_chunks: List
+        self,
+        object_path: str,
+        local_path: str,
+        uploaded_chunks: List,
+        resumable_id: str = None,
+        job_id: str = None,
+        item_id: str = None,
     ) -> None:
         # object storage info
         self.resumable_id = resumable_id
diff --git a/app/services/file_manager/file_upload/upload_client.py b/app/services/file_manager/file_upload/upload_client.py
index df990d73..ace580f3 100644
--- a/app/services/file_manager/file_upload/upload_client.py
+++ b/app/services/file_manager/file_upload/upload_client.py
@@ -135,7 +135,7 @@ def resume_upload(self, unfinished_file_objects: List[FileObject]) -> List[FileO
         return unfinished_file_objects
 
     # @require_valid_token()
-    def pre_upload(self, local_file_paths: List[str], output_path: str) -> List[FileObject]:
+    def pre_upload(self, file_objects: List[FileObject], output_path: str) -> List[FileObject]:
         """
         Summary:
             The function is to initiate all the multipart upload.
@@ -152,17 +152,18 @@ def pre_upload(self, local_file_paths: List[str], output_path: str) -> List[File
         # print('pre upload')
         headers = {'Authorization': 'Bearer ' + self.user.access_token, 'Session-ID': self.user.session_id}
         url = AppConfig.Connections.url_bff + '/v1/project/{}/files'.format(self.project_code)
-        # the file mapping is a dictionary that present the map from object storage path
-        # with local file path. It will be used in chunk upload api.
-        payload, file_mapping = uf.generate_pre_upload_form(
-            self.project_code,
-            self.operator,
-            local_file_paths,
-            self.input_path,
-            zone=self.zone,
-            job_type=self.job_type,
-            current_folder=self.current_folder_node,
-        )
+
+        file_mapping = {x.object_path: x.local_path for x in file_objects}
+        payload = {
+            'project_code': self.project_code,
+            'operator': self.operator,
+            'job_type': str(self.job_type),
+            'zone': self.zone,
+            'current_folder_node': self.current_folder_node,
+            'data': [
+                {'resumable_filename': x.file_name, 'resumable_relative_path': x.parent_path} for x in file_objects
+            ],
+        }
 
         payload.update({'parent_folder_id': self.parent_folder_id})
         payload.update({'folder_tags': self.tags})

From 2ccdb2c213698b491ff06fc1a20e00118f2b2f3a Mon Sep 17 00:00:00 2001
From: zhiren <zzhan@indocresearch.org>
Date: Wed, 5 Apr 2023 16:44:46 -0400
Subject: [PATCH 11/62] fixup folder upload and ready to merge with fixing
 branch

---
 app/commands/file.py                                 | 2 +-
 app/configs/app_config.py                            | 2 +-
 app/services/file_manager/file_upload/file_upload.py | 2 --
 3 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/app/commands/file.py b/app/commands/file.py
index eb751d52..830463f7 100644
--- a/app/commands/file.py
+++ b/app/commands/file.py
@@ -108,7 +108,7 @@ def cli():
 @click.option(
     '--output-path',
     '-o',
-    default='./',
+    default='./manifest.json',
     required=False,
     help='The output path for the manifest file of resumable upload',
     show_default=True,
diff --git a/app/configs/app_config.py b/app/configs/app_config.py
index 6abd7013..3d2ba3e8 100644
--- a/app/configs/app_config.py
+++ b/app/configs/app_config.py
@@ -12,7 +12,7 @@ class Env(object):
         user_config_path = ConfigClass.config_path
         msg_path = ConfigClass.custom_path
         user_config_file = f'{user_config_path}/config.ini'
-        token_warn_need_refresh = 30  # refresh token if token is about to expire
+        token_warn_need_refresh = 250  # refresh token if token is about to expire
         token_refresh_interval = 120  # auto refresh token every 2 minutes
 
         # NOTE: there is a limitation on minio that
diff --git a/app/services/file_manager/file_upload/file_upload.py b/app/services/file_manager/file_upload/file_upload.py
index e65f5c03..93cb159c 100644
--- a/app/services/file_manager/file_upload/file_upload.py
+++ b/app/services/file_manager/file_upload/file_upload.py
@@ -131,8 +131,6 @@ def simple_upload(  # noqa: C901
             upload_file_path = [my_file.rstrip('/').lstrip() + '.zip']
             target_folder = '/'.join(target_folder.split('/')[:-1]).rstrip('/')
             compress_folder_to_zip(my_file)
-        elif job_type == UploadType.AS_FOLDER:
-            SrvErrorHandler.customized_handle(ECustomizedError.UNSUPPORTED_PROJECT, True, project_code)
         else:
             logger.warning('Current version does not support folder tagging, ' 'any selected tags will be ignored')
             upload_file_path = get_file_in_folder(my_file)

From 587e48fede88735fdae16a31da7091cfb95327bb Mon Sep 17 00:00:00 2001
From: zhiren <zzhan@indocresearch.org>
Date: Mon, 10 Apr 2023 16:18:36 -0400
Subject: [PATCH 12/62] fixup the resumable upload

---
 .gitignore                                    |  3 +++
 app/commands/entry_point.py                   |  2 ++
 app/commands/file.py                          |  7 ++++-
 app/resources/custom_error.py                 |  3 +++
 .../file_manager/file_upload/file_upload.py   | 26 ++++++++++++-------
 .../file_manager/file_upload/upload_client.py |  3 ++-
 app/services/output_manager/error_handler.py  |  1 +
 7 files changed, 34 insertions(+), 11 deletions(-)

diff --git a/.gitignore b/.gitignore
index 155f40bf..2dad06a2 100644
--- a/.gitignore
+++ b/.gitignore
@@ -151,3 +151,6 @@ integration_tests
 
 #Pycharm
 .idea
+
+# cli manifest data
+./manifest.json
diff --git a/app/commands/entry_point.py b/app/commands/entry_point.py
index f67b2a4b..0d65a6e4 100644
--- a/app/commands/entry_point.py
+++ b/app/commands/entry_point.py
@@ -22,6 +22,7 @@
 from .file import file_export_manifest
 from .file import file_list
 from .file import file_put
+from .file import file_resume
 
 # Import custom commands
 from .hpc import hpc_auth
@@ -98,6 +99,7 @@ def cr_group():
 file_group.add_command(file_export_manifest)
 file_group.add_command(file_list)
 file_group.add_command(file_download)
+file_group.add_command(file_resume)
 project_group.add_command(project_list_all)
 user_group.add_command(login)
 user_group.add_command(logout)
diff --git a/app/commands/file.py b/app/commands/file.py
index 830463f7..ad0686a6 100644
--- a/app/commands/file.py
+++ b/app/commands/file.py
@@ -141,6 +141,10 @@ def file_put(**kwargs):  # noqa: C901
     if len(paths) == 0:
         SrvErrorHandler.customized_handle(ECustomizedError.INVALID_PATHS, True)
 
+    # check if the manifest file exists
+    if os.path.exists(output_path):
+        click.confirm(customized_error_msg(ECustomizedError.MANIFEST_OF_FOLDER_FILE_EXIST) % (output_path), abort=True)
+
     project_path = click.prompt('ProjectCode') if not project_path else project_path
     project_code, target_folder = identify_target_folder(project_path)
     srv_manifest = SrvFileManifests()
@@ -219,7 +223,7 @@ def file_put(**kwargs):  # noqa: C901
     help='The manifest file for resumable upload',
     show_default=True,
 )
-@doc(file_help.file_help_page(file_help.FileHELP.FILE_UPLOAD))
+@doc(file_help.file_help_page(file_help.FileHELP.FILE_RESUME))
 def file_resume(**kwargs):  # noqa: C901
     """
     Summary:
@@ -243,6 +247,7 @@ def file_resume(**kwargs):  # noqa: C901
         # are rather similar with the input
         validate_upload_event(resumable_manifest)
 
+    # print(resumable_manifest)
     resume_upload(resumable_manifest, thread)
 
 
diff --git a/app/resources/custom_error.py b/app/resources/custom_error.py
index c05291b0..9b999788 100644
--- a/app/resources/custom_error.py
+++ b/app/resources/custom_error.py
@@ -58,6 +58,9 @@ class Error:
             'The specified multipart upload does not exist. '
             'The upload ID may be invalid, or the upload may have been aborted or completed.'
         ),
+        'MANIFEST_OF_FOLDER_FILE_EXIST': (
+            'The manifest file of folder %s already exist. ' 'Do you want to overwrite the existing manifest file?'
+        ),
         'INVALID_CHUNK_UPLOAD': (
             '\nThe chunk number %d is not the same with previous etag.\n'
             'It means the resumable file is not the same with previous one.\n'
diff --git a/app/services/file_manager/file_upload/file_upload.py b/app/services/file_manager/file_upload/file_upload.py
index 961751ff..64b77029 100644
--- a/app/services/file_manager/file_upload/file_upload.py
+++ b/app/services/file_manager/file_upload/file_upload.py
@@ -232,17 +232,25 @@ def resume_upload(
 
     # check files in manifest if some of them are already uploaded
     item_ids = []
-    for item_id in manifest_json.get('file_objects'):
+    all_files = manifest_json.get('file_objects')
+    for item_id in all_files:
         item_ids.append(item_id)
     items = get_file_info_by_geid(item_ids)
-    unfinished_items = [x for x in items if x.get('status') == ItemStatus.REGISTERED]  # update to enum later
-    # make them as FileObject
-    unfinished_items = [
-        FileObject(
-            x.get('resumable_id'), x.get('item_id'), x.get('job_id'), x.get('object_path'), x.get('local_path'), []
-        )
-        for x in unfinished_items
-    ]
+
+    unfinished_items = []
+    for x in items:
+        if x.get('result').get('status') == ItemStatus.REGISTERED:
+            file_info = all_files.get(x.get('result').get('id'))
+            unfinished_items.append(
+                FileObject(
+                    file_info.get('resumable_id'),
+                    file_info.get('job_id'),
+                    file_info.get('item_id'),
+                    file_info.get('object_path'),
+                    file_info.get('local_path'),
+                    [],
+                )
+            )
 
     # then for the rest of the files, check if any chunks are already uploaded
     unfinished_items = upload_client.resume_upload(unfinished_items)
diff --git a/app/services/file_manager/file_upload/upload_client.py b/app/services/file_manager/file_upload/upload_client.py
index b4f34f0e..5601db2d 100644
--- a/app/services/file_manager/file_upload/upload_client.py
+++ b/app/services/file_manager/file_upload/upload_client.py
@@ -317,7 +317,8 @@ def upload_chunk(self, file_object: FileObject, chunk_number: int, chunk: str) -
             }
             headers = {'Authorization': 'Bearer ' + self.user.access_token, 'Session-ID': self.user.session_id}
             response = httpx.get(
-                self.base_url + '/v1/files/chunks/presigned',
+                # self.base_url + '/v1/files/chunks/presigned',
+                'http://localhost:5079' + '/v1/files/chunks/presigned',
                 params=params,
                 headers=headers,
                 timeout=None,
diff --git a/app/services/output_manager/error_handler.py b/app/services/output_manager/error_handler.py
index 1b839008..463b4f4c 100644
--- a/app/services/output_manager/error_handler.py
+++ b/app/services/output_manager/error_handler.py
@@ -40,6 +40,7 @@ class ECustomizedError(enum.Enum):
     UPLOAD_FAIL = 'UPLOAD_FAIL'
     # the error when multipart upload id is not exist
     UPLOAD_ID_NOT_EXIST = 'UPLOAD_ID_NOT_EXIST'
+    MANIFEST_OF_FOLDER_FILE_EXIST = 'MANIFEST_OF_FOLDER_FILE_EXIST'
     # the error when chunk md5 is not match
     INVALID_CHUNK_UPLOAD = 'INVALID_CHUNK_UPLOAD'
     MANIFEST_NOT_FOUND = 'MANIFEST_NOT_FOUND'

From efba1f33741987413fbca46c4a8525dca2300d76 Mon Sep 17 00:00:00 2001
From: zhiren <zzhan@indocresearch.org>
Date: Tue, 11 Apr 2023 09:10:19 -0400
Subject: [PATCH 13/62] fixup the folder resumable uploading

---
 .gitignore                                             | 1 +
 app/services/file_manager/file_upload/file_upload.py   | 3 ++-
 app/services/file_manager/file_upload/upload_client.py | 3 +--
 3 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/.gitignore b/.gitignore
index 2dad06a2..7bf999b0 100644
--- a/.gitignore
+++ b/.gitignore
@@ -154,3 +154,4 @@ integration_tests
 
 # cli manifest data
 ./manifest.json
+manifest.json
diff --git a/app/services/file_manager/file_upload/file_upload.py b/app/services/file_manager/file_upload/file_upload.py
index 64b77029..b6f3ebb4 100644
--- a/app/services/file_manager/file_upload/file_upload.py
+++ b/app/services/file_manager/file_upload/file_upload.py
@@ -85,6 +85,7 @@ def assemble_path(
             # find the longest existing folder as parent folder
             # if user input a path that need to create some folders
             if not res.get('result'):
+                current_file_path = folder_path
                 click.confirm(customized_error_msg(ECustomizedError.CREATE_FOLDER_IF_NOT_EXIST), abort=True)
                 create_folder_flag = True
                 break
@@ -136,12 +137,12 @@ def simple_upload(  # noqa: C901
             upload_file_path = get_file_in_folder(my_file)
     else:
         upload_file_path = [my_file]
-        target_folder = '/'.join(target_folder.split('/')[:-1]).rstrip('/')
 
         if create_folder_flag:
             job_type = UploadType.AS_FOLDER
             my_file = os.path.dirname(my_file)  # update the path as folder
         else:
+            target_folder = '/'.join(target_folder.split('/')[:-1]).rstrip('/')
             job_type = UploadType.AS_FILE
 
     upload_client = UploadClient(
diff --git a/app/services/file_manager/file_upload/upload_client.py b/app/services/file_manager/file_upload/upload_client.py
index 5601db2d..b4f34f0e 100644
--- a/app/services/file_manager/file_upload/upload_client.py
+++ b/app/services/file_manager/file_upload/upload_client.py
@@ -317,8 +317,7 @@ def upload_chunk(self, file_object: FileObject, chunk_number: int, chunk: str) -
             }
             headers = {'Authorization': 'Bearer ' + self.user.access_token, 'Session-ID': self.user.session_id}
             response = httpx.get(
-                # self.base_url + '/v1/files/chunks/presigned',
-                'http://localhost:5079' + '/v1/files/chunks/presigned',
+                self.base_url + '/v1/files/chunks/presigned',
                 params=params,
                 headers=headers,
                 timeout=None,

From bd1e199390d25322bf4e342932abdbbd6944e717 Mon Sep 17 00:00:00 2001
From: zhiren <zzhan@indocresearch.org>
Date: Tue, 11 Apr 2023 09:15:57 -0400
Subject: [PATCH 14/62] fixup the test case

---
 tests/app/services/file_manager/file_upload/test_file_upload.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/app/services/file_manager/file_upload/test_file_upload.py b/tests/app/services/file_manager/file_upload/test_file_upload.py
index 1c4a121f..ea46fb03 100644
--- a/tests/app/services/file_manager/file_upload/test_file_upload.py
+++ b/tests/app/services/file_manager/file_upload/test_file_upload.py
@@ -101,7 +101,7 @@ def test_assemble_path_at_non_existing_folder(mocker):
     current_file_path, parent_folder, create_folder_flag, _ = assemble_path(
         local_file_path, target_folder, project_code, zone, resumable_id
     )
-    assert current_file_path == 'admin/test_folder_not_exist/file.txt'
+    assert current_file_path == 'admin/test_folder_not_exist'
     assert parent_folder.get('name') == 'admin'
     assert create_folder_flag is True
 

From 5fb271a61dff14fa8d0d949d44cf1fd467c9f19d Mon Sep 17 00:00:00 2001
From: zhiren <zzhan@indocresearch.org>
Date: Tue, 11 Apr 2023 09:41:44 -0400
Subject: [PATCH 15/62] fixup the test

---
 .../app/services/file_manager/file_upload/test_file_upload.py  | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/app/services/file_manager/file_upload/test_file_upload.py b/tests/app/services/file_manager/file_upload/test_file_upload.py
index ea46fb03..7e016b84 100644
--- a/tests/app/services/file_manager/file_upload/test_file_upload.py
+++ b/tests/app/services/file_manager/file_upload/test_file_upload.py
@@ -122,8 +122,9 @@ def test_resume_upload(mocker):
 
     get_return = test_obj.to_dict()
     get_return.update({'status': ItemStatus.REGISTERED})
+    get_return.update({'id': get_return.get('item_id')})
     get_mock = mocker.patch(
-        'app.services.file_manager.file_upload.file_upload.get_file_info_by_geid', return_value=[get_return]
+        'app.services.file_manager.file_upload.file_upload.get_file_info_by_geid', return_value=[{'result': get_return}]
     )
     resume_upload_mock = mocker.patch(
         'app.services.file_manager.file_upload.file_upload.UploadClient.resume_upload', return_value=[]

From 72d6f81a8e93b9d153286b1d9c5f7a3eea205108 Mon Sep 17 00:00:00 2001
From: zhiren <zzhan@indocresearch.org>
Date: Tue, 11 Apr 2023 09:52:47 -0400
Subject: [PATCH 16/62] fixup the token check test

---
 .../file_manager/file_upload/upload_client.py         |  2 +-
 .../file_manager/file_upload/test_upload_client.py    | 11 +++++++++++
 tests/conftest.py                                     |  6 +++---
 3 files changed, 15 insertions(+), 4 deletions(-)

diff --git a/app/services/file_manager/file_upload/upload_client.py b/app/services/file_manager/file_upload/upload_client.py
index b4f34f0e..66e887f4 100644
--- a/app/services/file_manager/file_upload/upload_client.py
+++ b/app/services/file_manager/file_upload/upload_client.py
@@ -103,7 +103,7 @@ def generate_meta(self, local_path: str) -> Tuple[int, int]:
         total_chunks = math.ceil(total_size / self.chunk_size)
         return total_size, total_chunks
 
-    # @require_valid_token()
+    @require_valid_token()
     def resume_upload(self, unfinished_file_objects: List[FileObject]) -> List[FileObject]:
         """
         Summary:
diff --git a/tests/app/services/file_manager/file_upload/test_upload_client.py b/tests/app/services/file_manager/file_upload/test_upload_client.py
index 50321551..e01ce291 100644
--- a/tests/app/services/file_manager/file_upload/test_upload_client.py
+++ b/tests/app/services/file_manager/file_upload/test_upload_client.py
@@ -10,6 +10,7 @@
 from app.configs.app_config import AppConfig
 from app.services.file_manager.file_upload.models import FileObject
 from app.services.file_manager.file_upload.upload_client import UploadClient
+from tests.conftest import decoded_token
 
 
 def test_chunk_upload(httpx_mock, mocker):
@@ -55,6 +56,11 @@ def test_token_refresh_auto(mocker):
 
 
 def test_resumable_pre_upload_success(httpx_mock, mocker):
+    mocker.patch(
+        'app.services.user_authentication.token_manager.SrvTokenManager.decode_access_token',
+        return_value=decoded_token(),
+    )
+
     upload_client = UploadClient('test', 'project_code', 'parent_folder_id')
     mocker.patch('app.services.file_manager.file_upload.models.FileObject.generate_meta', return_value=(1, 1))
     test_obj = FileObject('resumable_id', 'job_id', 'item_id', 'object/path', 'local_path', [])
@@ -72,6 +78,11 @@ def test_resumable_pre_upload_success(httpx_mock, mocker):
 
 
 def test_resumable_pre_upload_failed_with_404(httpx_mock, mocker):
+    mocker.patch(
+        'app.services.user_authentication.token_manager.SrvTokenManager.decode_access_token',
+        return_value=decoded_token(),
+    )
+
     upload_client = UploadClient('test', 'project_code', 'parent_folder_id')
     mocker.patch('app.services.file_manager.file_upload.models.FileObject.generate_meta', return_value=(1, 1))
     test_obj = FileObject('resumable_id', 'job_id', 'item_id', 'object/path', 'local_path', [])
diff --git a/tests/conftest.py b/tests/conftest.py
index 53cd9abc..beeb3785 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -32,15 +32,15 @@ def mock_settings(monkeypatch):
 def decoded_token():
     current_time = int(time.time()) + 1000
     return {
-        'exp': current_time,
+        'exp': current_time + 100,
         'iat': current_time,
-        'auth_time': current_time - 2,
+        'auth_time': current_time - 100,
         'jti': 'f0848a19-7ddb-4170-bca4-b2ee48512ac3',
         'iss': 'http://token-auth/issuer',
         'aud': 'account',
         'sub': 'a8b728f6-c95a-4999-b98e-0ccf7492a9b4',
         'typ': 'Bearer',
-        'azp': 'kong',
+        'azp': AppConfig.Env.keycloak_device_client_id,
         'nonce': 'a3cb03d0-b00a-480d-8fd2-e06f80898cf1',
         'session_state': 'b92a3847-a485-4060-91fd-83300b09acb6',
         'acr': '1',

From c9463184866ba969e1ca667b3309ecdf3c93c093 Mon Sep 17 00:00:00 2001
From: zhiren <zzhan@indocresearch.org>
Date: Tue, 11 Apr 2023 10:31:45 -0400
Subject: [PATCH 17/62] fixup with test

---
 app/commands/file.py                          |  5 ----
 .../file_manager/file_upload/file_upload.py   | 14 +++--------
 .../file_manager/file_upload/models.py        |  3 +--
 .../file_manager/file_upload/upload_client.py | 25 ++++++++-----------
 .../file_upload/test_file_upload.py           |  2 +-
 .../file_manager/file_upload/test_model.py    |  6 ++---
 .../file_upload/test_upload_client.py         |  8 +++---
 7 files changed, 23 insertions(+), 40 deletions(-)

diff --git a/app/commands/file.py b/app/commands/file.py
index 61c92527..8e5dea35 100644
--- a/app/commands/file.py
+++ b/app/commands/file.py
@@ -196,11 +196,6 @@ def file_put(**kwargs):  # noqa: C901
             zipping,
         )
 
-        current_folder_node = 'testproject/admin/test11'
-        parent_folder = {'id': 'testproject'}
-        create_folder_flag = True
-        result_file = None
-
         upload_event = {
             'project_code': project_code,
             'file': f.rstrip('/'),  # remove the ending slash
diff --git a/app/services/file_manager/file_upload/file_upload.py b/app/services/file_manager/file_upload/file_upload.py
index 78eb7f78..b5379a0c 100644
--- a/app/services/file_manager/file_upload/file_upload.py
+++ b/app/services/file_manager/file_upload/file_upload.py
@@ -141,17 +141,10 @@ def simple_upload(  # noqa: C901
         if create_folder_flag:
             job_type = UploadType.AS_FOLDER
             input_path = os.path.dirname(input_path)  # update the path as folder
-            target_folder = '/'.join(target_folder.split('/')[:-1]).rstrip('/')
         else:
             target_folder = '/'.join(target_folder.split('/')[:-1]).rstrip('/')
             job_type = UploadType.AS_FILE
 
-    # print('upload_file_path:', upload_file_path)
-    # print('target_folder:', target_folder)
-    # print('input_path:', input_path)
-    # print('job_type:', job_type)
-    # print('zone:', zone)
-
     upload_client = UploadClient(
         input_path=input_path,
         project_code=project_code,
@@ -169,7 +162,7 @@ def simple_upload(  # noqa: C901
         # first remove the input path from the file path
         file_path_sub = file.replace(input_path + '/', '')
         object_path = os.path.join(target_folder, file_path_sub)
-        file_objects.append(FileObject(object_path, file, None))
+        file_objects.append(FileObject(object_path, file))
 
     # here add the batch of 500 per loop, the pre upload api cannot
     # process very large amount of file at same time. otherwise it will timeout
@@ -257,12 +250,11 @@ def resume_upload(
             file_info = all_files.get(x.get('result').get('id'))
             unfinished_items.append(
                 FileObject(
+                    file_info.get('object_path'),
+                    file_info.get('local_path'),
                     file_info.get('resumable_id'),
                     file_info.get('job_id'),
                     file_info.get('item_id'),
-                    file_info.get('object_path'),
-                    file_info.get('local_path'),
-                    [],
                 )
             )
 
diff --git a/app/services/file_manager/file_upload/models.py b/app/services/file_manager/file_upload/models.py
index f96e6ba6..334e0d7a 100644
--- a/app/services/file_manager/file_upload/models.py
+++ b/app/services/file_manager/file_upload/models.py
@@ -62,7 +62,6 @@ def __init__(
         self,
         object_path: str,
         local_path: str,
-        uploaded_chunks: List,
         resumable_id: str = None,
         job_id: str = None,
         item_id: str = None,
@@ -79,7 +78,7 @@ def __init__(
         self.total_size, self.total_chunks = self.generate_meta(local_path)
 
         # resumable info
-        self.uploaded_chunks = uploaded_chunks
+        self.uploaded_chunks = {}
 
     def generate_meta(self, local_path: str) -> Tuple[int, int]:
         """
diff --git a/app/services/file_manager/file_upload/upload_client.py b/app/services/file_manager/file_upload/upload_client.py
index 09238b7c..bc858115 100644
--- a/app/services/file_manager/file_upload/upload_client.py
+++ b/app/services/file_manager/file_upload/upload_client.py
@@ -164,38 +164,35 @@ def pre_upload(self, file_objects: List[FileObject], output_path: str) -> List[F
                 - local_path(str): the local path of file.
                 - chunk_info(dict): the mapping for chunks that already been uploaded.
         """
-        # print('pre upload')
+
         headers = {'Authorization': 'Bearer ' + self.user.access_token, 'Session-ID': self.user.session_id}
         url = AppConfig.Connections.url_bff + '/v1/project/{}/files'.format(self.project_code)
-
-        file_mapping = {x.object_path: x.local_path for x in file_objects}
         payload = {
             'project_code': self.project_code,
             'operator': self.operator,
             'job_type': str(self.job_type),
             'zone': self.zone,
             'current_folder_node': self.current_folder_node,
+            'parent_folder_id': self.parent_folder_id,
+            'folder_tags': self.tags,
             'data': [
                 {'resumable_filename': x.file_name, 'resumable_relative_path': x.parent_path} for x in file_objects
             ],
         }
-
-        payload.update({'parent_folder_id': self.parent_folder_id})
-        payload.update({'folder_tags': self.tags})
-        # print('pre upload payload: ', payload)
-        # raise Exception('pre upload')
         response = resilient_session().post(url, json=payload, headers=headers, timeout=None)
+
         if response.status_code == 200:
             result = response.json().get('result')
+            file_mapping = {x.object_path: x for x in file_objects}
             file_objets = []
             for job in result:
                 object_path = job.get('target_names')[0]
-                resumable_id = job.get('payload').get('resumable_identifier')
-                item_id = job.get('payload').get('item_id')
-                job_id = job.get('job_id')
-                file_objets.append(
-                    FileObject(resumable_id, job_id, item_id, object_path, file_mapping.get(object_path), {})
-                )
+                # get the file object from mapping and update the attribute
+                file_object = file_mapping.get(object_path)
+                file_object.resumable_id = job.get('payload').get('resumable_identifier')
+                file_object.item_id = job.get('payload').get('item_id')
+                file_object.job_id = job.get('job_id')
+                file_objets.append(file_object)
 
             # then output manifest file to the output path
             self.output_manifest(file_objets, output_path)
diff --git a/tests/app/services/file_manager/file_upload/test_file_upload.py b/tests/app/services/file_manager/file_upload/test_file_upload.py
index 7e016b84..3f33298a 100644
--- a/tests/app/services/file_manager/file_upload/test_file_upload.py
+++ b/tests/app/services/file_manager/file_upload/test_file_upload.py
@@ -108,7 +108,7 @@ def test_assemble_path_at_non_existing_folder(mocker):
 
 def test_resume_upload(mocker):
     mocker.patch('app.services.file_manager.file_upload.models.FileObject.generate_meta', return_value=(1, 1))
-    test_obj = FileObject('resumable_id', 'job_id', 'item_id', 'object/path', 'local_path', [])
+    test_obj = FileObject('object/path', 'local_path', 'resumable_id', 'job_id', 'item_id')
 
     manifest_json = {
         'project_code': 'project_code',
diff --git a/tests/app/services/file_manager/file_upload/test_model.py b/tests/app/services/file_manager/file_upload/test_model.py
index da612bf7..7b9924ab 100644
--- a/tests/app/services/file_manager/file_upload/test_model.py
+++ b/tests/app/services/file_manager/file_upload/test_model.py
@@ -9,7 +9,7 @@
 def test_file_upload_model_update_progress_bar(mocker):
     mocker.patch('app.services.file_manager.file_upload.models.getsize', return_value=100)
 
-    file_obj = FileObject('test', 'test', 'test', 'test', 'test', [])
+    file_obj = FileObject('test', 'test', 'test', 'test', 'test')
     file_obj.update_progress(1)
 
     assert file_obj.progress_bar is not None
@@ -19,7 +19,7 @@ def test_file_upload_model_update_progress_bar(mocker):
 def test_file_upload_model_close_progress_bar(mocker):
     mocker.patch('app.services.file_manager.file_upload.models.getsize', return_value=100)
 
-    file_obj = FileObject('test', 'test', 'test', 'test', 'test', [])
+    file_obj = FileObject('test', 'test', 'test', 'test', 'test')
     file_obj.close_progress()
 
     assert file_obj.progress_bar is None
@@ -29,7 +29,7 @@ def test_file_upload_model_generate_meta(mocker):
     AppConfig.Env.chunk_size = 10
     mocker.patch('app.services.file_manager.file_upload.models.getsize', return_value=100)
 
-    file_obj = FileObject('test', 'test', 'test', 'test', 'test', [])
+    file_obj = FileObject('test', 'test', 'test', 'test', 'test')
     total_size, total_chunks = file_obj.generate_meta('test')
 
     assert total_size == 100
diff --git a/tests/app/services/file_manager/file_upload/test_upload_client.py b/tests/app/services/file_manager/file_upload/test_upload_client.py
index e01ce291..cec7523f 100644
--- a/tests/app/services/file_manager/file_upload/test_upload_client.py
+++ b/tests/app/services/file_manager/file_upload/test_upload_client.py
@@ -22,7 +22,7 @@ def test_chunk_upload(httpx_mock, mocker):
     httpx_mock.add_response(method='PUT', url=test_presigned_url, json={'result': ''})
     mocker.patch('app.services.file_manager.file_upload.models.FileObject.generate_meta', return_value=(1, 1))
 
-    test_obj = FileObject('test', 'test', 'test', 'test', 'test', [])
+    test_obj = FileObject('test', 'test', 'test', 'test', 'test')
     res = upload_client.upload_chunk(test_obj, 0, b'1')
 
     assert test_obj.progress_bar.n == 1
@@ -63,7 +63,7 @@ def test_resumable_pre_upload_success(httpx_mock, mocker):
 
     upload_client = UploadClient('test', 'project_code', 'parent_folder_id')
     mocker.patch('app.services.file_manager.file_upload.models.FileObject.generate_meta', return_value=(1, 1))
-    test_obj = FileObject('resumable_id', 'job_id', 'item_id', 'object/path', 'local_path', [])
+    test_obj = FileObject('object/path', 'local_path', 'resumable_id', 'job_id', 'item_id')
 
     url = AppConfig.Connections.url_bff + f'/v1/project/{upload_client.project_code}/files/resumable'
     httpx_mock.add_response(
@@ -85,7 +85,7 @@ def test_resumable_pre_upload_failed_with_404(httpx_mock, mocker):
 
     upload_client = UploadClient('test', 'project_code', 'parent_folder_id')
     mocker.patch('app.services.file_manager.file_upload.models.FileObject.generate_meta', return_value=(1, 1))
-    test_obj = FileObject('resumable_id', 'job_id', 'item_id', 'object/path', 'local_path', [])
+    test_obj = FileObject('object/path', 'local_path', 'resumable_id', 'job_id', 'item_id')
 
     url = AppConfig.Connections.url_bff + f'/v1/project/{upload_client.project_code}/files/resumable'
     httpx_mock.add_response(
@@ -107,7 +107,7 @@ def test_output_manifest_success(mocker):
     upload_client = UploadClient('test', 'project_code', 'parent_folder_id')
     json_dump_mocker = mocker.patch('json.dump', return_value=None)
     mocker.patch('app.services.file_manager.file_upload.models.FileObject.generate_meta', return_value=(1, 1))
-    test_obj = FileObject('resumable_id', 'job_id', 'item_id', 'object/path', 'local_path', [])
+    test_obj = FileObject('object/path', 'local_path', 'resumable_id', 'job_id', 'item_id')
 
     res = upload_client.output_manifest([test_obj], 'test')
 

From 8f918d44149f349ca1942f18aadcd0c3ffedcd72 Mon Sep 17 00:00:00 2001
From: zhiren <zzhan@indocresearch.org>
Date: Tue, 11 Apr 2023 11:01:58 -0400
Subject: [PATCH 18/62] test

---
 app/services/file_manager/file_upload/upload_client.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/app/services/file_manager/file_upload/upload_client.py b/app/services/file_manager/file_upload/upload_client.py
index bc858115..82a3ee5c 100644
--- a/app/services/file_manager/file_upload/upload_client.py
+++ b/app/services/file_manager/file_upload/upload_client.py
@@ -179,6 +179,9 @@ def pre_upload(self, file_objects: List[FileObject], output_path: str) -> List[F
                 {'resumable_filename': x.file_name, 'resumable_relative_path': x.parent_path} for x in file_objects
             ],
         }
+
+        # print('pre upload payload', payload)
+        raise Exception('stop here')
         response = resilient_session().post(url, json=payload, headers=headers, timeout=None)
 
         if response.status_code == 200:

From e849cff8cbdc08078160d2b27a760545da9cfec4 Mon Sep 17 00:00:00 2001
From: zhiren <zzhan@indocresearch.org>
Date: Tue, 11 Apr 2023 11:48:21 -0400
Subject: [PATCH 19/62] fixup the path formatting issue

---
 app/commands/file.py                          | 11 +++++----
 .../file_manager/file_upload/file_upload.py   | 24 +++++++++++--------
 .../file_manager/file_upload/upload_client.py |  2 --
 3 files changed, 20 insertions(+), 17 deletions(-)

diff --git a/app/commands/file.py b/app/commands/file.py
index 8e5dea35..8dd95926 100644
--- a/app/commands/file.py
+++ b/app/commands/file.py
@@ -173,14 +173,15 @@ def file_put(**kwargs):  # noqa: C901
 
     # for the path formating there will be following cases:
     # - file:
-    #   1. the project path exist, then will be AS_FILE. nothing will be changed
-    #   2. the project path not exist, then will be AS_FOLDER. the current folder node will
-    #      be the parent folder node + parent folder id. (like one level up).
+    #   1. the project path exist, then will be AS_FILE. nothing will be changed.
+    #      current_folder_node will be empty string.
+    #   2. the project path not exist, then will be AS_FOLDER. the current_folder_node will
+    #      be the parent folder node + the shortest non-exist folder. (like one level down).
     # - folder:
     #   1. the project path exist, then will be AS_FOLDER. the current folder node will be
     #      the one that user input.
     #   2. the project path not exist, then will be AS_FOLDER. the current folder node will
-    #      be the parent folder node + parent folder id. (like one level up).
+    #      be the parent folder node + the shortest non-exist folder. (like one level down).
 
     # Unique Paths
     paths = set(paths)
@@ -198,6 +199,7 @@ def file_put(**kwargs):  # noqa: C901
 
         upload_event = {
             'project_code': project_code,
+            'target_folder': target_folder,
             'file': f.rstrip('/'),  # remove the ending slash
             'tags': tag if tag else [],
             'zone': zone,
@@ -208,7 +210,6 @@ def file_put(**kwargs):  # noqa: C901
             'compress_zip': zipping,
             'attribute': attribute,
         }
-        # print(upload_event)
         if pipeline:
             upload_event['process_pipeline'] = pipeline
         if source_file:
diff --git a/app/services/file_manager/file_upload/file_upload.py b/app/services/file_manager/file_upload/file_upload.py
index b5379a0c..f13e6d25 100644
--- a/app/services/file_manager/file_upload/file_upload.py
+++ b/app/services/file_manager/file_upload/file_upload.py
@@ -69,13 +69,18 @@ def assemble_path(
 
     current_file_path = target_folder + '/' + f.rstrip('/').split('/')[-1]
     result_file = current_file_path
+    if zipping:
+        result_file = result_file + '.zip'
 
     # set name folder as first parent folder
-    name_folder = current_file_path.split('/')[0]
+    name_folder = target_folder.split('/')[0]
     parent_folder = search_item(project_code, zone, name_folder, 'name_folder')
     parent_folder = parent_folder.get('result')
-    create_folder_flag = False
 
+    # if f input is a file then current_folder_node is target_folder
+    # otherwise it is target_folder + f input name
+    current_folder_node = target_folder if os.path.isfile(f) else current_file_path
+    create_folder_flag = False
     if len(current_file_path.split('/')) > 2:
         sub_path = target_folder.split('/')
         for index in range(len(sub_path) - 1):
@@ -85,7 +90,7 @@ def assemble_path(
             # find the longest existing folder as parent folder
             # if user input a path that need to create some folders
             if not res.get('result'):
-                current_file_path = folder_path
+                current_folder_node = folder_path
                 click.confirm(customized_error_msg(ECustomizedError.CREATE_FOLDER_IF_NOT_EXIST), abort=True)
                 create_folder_flag = True
                 break
@@ -97,9 +102,7 @@ def assemble_path(
     if not parent_folder:
         SrvErrorHandler.customized_handle(ECustomizedError.PERMISSION_DENIED, True)
 
-    if zipping:
-        result_file = result_file + '.zip'
-    return current_file_path, parent_folder, create_folder_flag, result_file
+    return current_folder_node, parent_folder, create_folder_flag, result_file
 
 
 def simple_upload(  # noqa: C901
@@ -114,7 +117,7 @@ def simple_upload(  # noqa: C901
     zone = upload_event.get('zone')
     # process_pipeline = upload_event.get('process_pipeline', None)
     # upload_message = upload_event.get('upload_message')
-    target_folder = upload_event.get('current_folder_node', '')
+    current_folder_node = upload_event.get('current_folder_node', '')
     parent_folder_id = upload_event.get('parent_folder_id', '')
     create_folder_flag = upload_event.get('create_folder_flag', False)
     compress_zip = upload_event.get('compress_zip', False)
@@ -130,7 +133,7 @@ def simple_upload(  # noqa: C901
         job_type = UploadType.AS_FILE if compress_zip else UploadType.AS_FOLDER
         if job_type == UploadType.AS_FILE:
             upload_file_path = [input_path.rstrip('/').lstrip() + '.zip']
-            target_folder = '/'.join(target_folder.split('/')[:-1]).rstrip('/')
+            # target_folder = '/'.join(target_folder.split('/')[:-1]).rstrip('/')
             compress_folder_to_zip(input_path)
         else:
             logger.warning('Current version does not support folder tagging, ' 'any selected tags will be ignored')
@@ -142,7 +145,7 @@ def simple_upload(  # noqa: C901
             job_type = UploadType.AS_FOLDER
             input_path = os.path.dirname(input_path)  # update the path as folder
         else:
-            target_folder = '/'.join(target_folder.split('/')[:-1]).rstrip('/')
+            # target_folder = '/'.join(target_folder.split('/')[:-1]).rstrip('/')
             job_type = UploadType.AS_FILE
 
     upload_client = UploadClient(
@@ -150,7 +153,7 @@ def simple_upload(  # noqa: C901
         project_code=project_code,
         zone=zone,
         job_type=job_type,
-        current_folder_node=target_folder,
+        current_folder_node=current_folder_node,
         parent_folder_id=parent_folder_id,
         regular_file=regular_file,
         tags=tags,
@@ -158,6 +161,7 @@ def simple_upload(  # noqa: C901
 
     # format the local path into object storage path for preupload
     file_objects = []
+    target_folder = upload_event.get('target_folder', '')
     for file in upload_file_path:
         # first remove the input path from the file path
         file_path_sub = file.replace(input_path + '/', '')
diff --git a/app/services/file_manager/file_upload/upload_client.py b/app/services/file_manager/file_upload/upload_client.py
index 82a3ee5c..d74b4948 100644
--- a/app/services/file_manager/file_upload/upload_client.py
+++ b/app/services/file_manager/file_upload/upload_client.py
@@ -180,8 +180,6 @@ def pre_upload(self, file_objects: List[FileObject], output_path: str) -> List[F
             ],
         }
 
-        # print('pre upload payload', payload)
-        raise Exception('stop here')
         response = resilient_session().post(url, json=payload, headers=headers, timeout=None)
 
         if response.status_code == 200:

From 44f7cce12d6fd436024d99355c7142f4fb3e162e Mon Sep 17 00:00:00 2001
From: zhiren <zzhan@indocresearch.org>
Date: Tue, 11 Apr 2023 11:52:30 -0400
Subject: [PATCH 20/62] add the combine check to make sure the thread stop
 after all finished

---
 app/services/file_manager/file_upload/file_upload.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/app/services/file_manager/file_upload/file_upload.py b/app/services/file_manager/file_upload/file_upload.py
index f13e6d25..347edef7 100644
--- a/app/services/file_manager/file_upload/file_upload.py
+++ b/app/services/file_manager/file_upload/file_upload.py
@@ -126,14 +126,12 @@ def simple_upload(  # noqa: C901
     attribute = upload_event.get('attribute')
 
     mhandler.SrvOutPutHandler.start_uploading(input_path)
-    # TODO: PILOT-2392 simplify the logic under
     # if the input request zip folder then process the path as single file
     # otherwise read throught the folder to get path underneath
     if os.path.isdir(input_path):
         job_type = UploadType.AS_FILE if compress_zip else UploadType.AS_FOLDER
         if job_type == UploadType.AS_FILE:
             upload_file_path = [input_path.rstrip('/').lstrip() + '.zip']
-            # target_folder = '/'.join(target_folder.split('/')[:-1]).rstrip('/')
             compress_folder_to_zip(input_path)
         else:
             logger.warning('Current version does not support folder tagging, ' 'any selected tags will be ignored')
@@ -145,7 +143,6 @@ def simple_upload(  # noqa: C901
             job_type = UploadType.AS_FOLDER
             input_path = os.path.dirname(input_path)  # update the path as folder
         else:
-            # target_folder = '/'.join(target_folder.split('/')[:-1]).rstrip('/')
             job_type = UploadType.AS_FILE
 
     upload_client = UploadClient(
@@ -190,14 +187,21 @@ def simple_upload(  # noqa: C901
 
     pool = ThreadPool(num_of_thread + 1)
     pool.apply_async(upload_client.upload_token_refresh)
+    on_succeed_res = []
     for file_object in pre_upload_infos:
         chunk_res = upload_client.stream_upload(file_object, pool)
         # NOTE: if there is some racing error make the combine chunks
         # out of thread pool.
-        pool.apply_async(
+        res = pool.apply_async(
             upload_client.on_succeed,
             args=(file_object, tags, chunk_res),
         )
+        on_succeed_res.append(res)
+
+    # wait for all the chunk combination to finish
+    for res in on_succeed_res:
+        while res.get() is None:
+            time.sleep(0.5)
     upload_client.set_finish_upload()
 
     pool.close()

From 474866dfb991c96fc169faa5f80551748ca30131 Mon Sep 17 00:00:00 2001
From: zhiren <zzhan@indocresearch.org>
Date: Tue, 11 Apr 2023 11:54:52 -0400
Subject: [PATCH 21/62] remove the chunk combine check

---
 app/services/file_manager/file_upload/file_upload.py | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/app/services/file_manager/file_upload/file_upload.py b/app/services/file_manager/file_upload/file_upload.py
index 347edef7..ca1bfeac 100644
--- a/app/services/file_manager/file_upload/file_upload.py
+++ b/app/services/file_manager/file_upload/file_upload.py
@@ -197,11 +197,6 @@ def simple_upload(  # noqa: C901
             args=(file_object, tags, chunk_res),
         )
         on_succeed_res.append(res)
-
-    # wait for all the chunk combination to finish
-    for res in on_succeed_res:
-        while res.get() is None:
-            time.sleep(0.5)
     upload_client.set_finish_upload()
 
     pool.close()

From 728edae2b67ceba97a08b8099b90c6645e54ba84 Mon Sep 17 00:00:00 2001
From: zhiren <zzhan@indocresearch.org>
Date: Tue, 11 Apr 2023 15:44:42 -0400
Subject: [PATCH 22/62] add the constraint to block tagging and manifest
 attaching when folder uploading

---
 app/resources/custom_error.py                 |  1 +
 .../file_manager/file_upload/file_upload.py   |  3 +-
 app/services/output_manager/error_handler.py  |  1 +
 .../file_upload/test_file_upload.py           | 29 +++++++++++++++++++
 4 files changed, 33 insertions(+), 1 deletion(-)

diff --git a/app/resources/custom_error.py b/app/resources/custom_error.py
index 9dcfabd7..f3af1c87 100644
--- a/app/resources/custom_error.py
+++ b/app/resources/custom_error.py
@@ -63,6 +63,7 @@ class Error:
             'It means the resumable file is not the same with previous one.\n'
             'Please to double check the file content.'
         ),
+        'UNSUPPORT_TAG_MANIFEST': 'Tagging and manifest attaching are not supported for folder type.',
         'INVALID_INPUT': 'Invalid input. Please try again.',
         'UNSUPPORTED_PROJECT': 'This function is not supported in the given Project %s',
         'CREATE_FOLDER_IF_NOT_EXIST': 'Target folder does not exist. Would you like to create a new folder?',
diff --git a/app/services/file_manager/file_upload/file_upload.py b/app/services/file_manager/file_upload/file_upload.py
index f7f88feb..b8f22f00 100644
--- a/app/services/file_manager/file_upload/file_upload.py
+++ b/app/services/file_manager/file_upload/file_upload.py
@@ -134,8 +134,9 @@ def simple_upload(  # noqa: C901
             compress_folder_to_zip(my_file)
         elif job_type == UploadType.AS_FOLDER and resumable_id:
             SrvErrorHandler.customized_handle(ECustomizedError.UNSUPPORTED_PROJECT, True, project_code)
+        elif tags or attribute:
+            SrvErrorHandler.customized_handle(ECustomizedError.UNSUPPORT_TAG_MANIFEST, True)
         else:
-            logger.warning('Current version does not support folder tagging, ' 'any selected tags will be ignored')
             upload_file_path = get_file_in_folder(my_file)
     else:
         upload_file_path = [my_file]
diff --git a/app/services/output_manager/error_handler.py b/app/services/output_manager/error_handler.py
index 1b839008..4d4bcc60 100644
--- a/app/services/output_manager/error_handler.py
+++ b/app/services/output_manager/error_handler.py
@@ -42,6 +42,7 @@ class ECustomizedError(enum.Enum):
     UPLOAD_ID_NOT_EXIST = 'UPLOAD_ID_NOT_EXIST'
     # the error when chunk md5 is not match
     INVALID_CHUNK_UPLOAD = 'INVALID_CHUNK_UPLOAD'
+    UNSUPPORT_TAG_MANIFEST = 'UNSUPPORT_TAG_MANIFEST'
     MANIFEST_NOT_FOUND = 'MANIFEST_NOT_FOUND'
     INVALID_INPUT = 'INVALID_INPUT'
     UNSUPPORTED_PROJECT = 'UNSUPPORTED_PROJECT'
diff --git a/tests/app/services/file_manager/file_upload/test_file_upload.py b/tests/app/services/file_manager/file_upload/test_file_upload.py
index 46569a61..15b1da0e 100644
--- a/tests/app/services/file_manager/file_upload/test_file_upload.py
+++ b/tests/app/services/file_manager/file_upload/test_file_upload.py
@@ -3,6 +3,9 @@
 # Contact Indoc Research for any questions regarding the use of this source code.
 
 from app.services.file_manager.file_upload.file_upload import assemble_path
+from app.services.file_manager.file_upload.file_upload import simple_upload
+from app.services.output_manager.error_handler import ECustomizedError
+from app.services.output_manager.error_handler import customized_error_msg
 
 
 def test_assemble_path_at_name_folder(mocker):
@@ -100,3 +103,29 @@ def test_assemble_path_at_non_existing_folder(mocker):
     assert current_file_path == 'admin/test_folder_not_exist/file.txt'
     assert parent_folder.get('name') == 'admin'
     assert create_folder_flag is True
+
+
+def test_folder_upload_tagging_should_block(mocker, capfd):
+    file_name = 'test'
+    upload_event = {
+        'file': file_name,
+        'project_code': 'test_project',
+        'tags': ['test_tag'],
+        'zone': 0,
+        'manifest': 'test_manifest',
+    }
+
+    mocker.patch('os.path.isdir', return_value=True)
+
+    try:
+        simple_upload(upload_event)
+    except SystemExit:
+        out, err = capfd.readouterr()
+
+        expect = (
+            f'Starting upload of: {file_name}\n' + customized_error_msg(ECustomizedError.UNSUPPORT_TAG_MANIFEST) + '\n'
+        )
+
+        assert out == expect
+    else:
+        AssertionError('SystemExit not raised')

From bc99568f1a6be917ab0bc7f64c642d80a4f74ed6 Mon Sep 17 00:00:00 2001
From: zhiren <zzhan@indocresearch.org>
Date: Tue, 11 Apr 2023 15:46:39 -0400
Subject: [PATCH 23/62] add the constraint to block tagging and manifest
 attaching when folder uploading

---
 tests/app/services/file_manager/file_upload/test_file_upload.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/app/services/file_manager/file_upload/test_file_upload.py b/tests/app/services/file_manager/file_upload/test_file_upload.py
index 15b1da0e..c171e887 100644
--- a/tests/app/services/file_manager/file_upload/test_file_upload.py
+++ b/tests/app/services/file_manager/file_upload/test_file_upload.py
@@ -120,7 +120,7 @@ def test_folder_upload_tagging_should_block(mocker, capfd):
     try:
         simple_upload(upload_event)
     except SystemExit:
-        out, err = capfd.readouterr()
+        out, _ = capfd.readouterr()
 
         expect = (
             f'Starting upload of: {file_name}\n' + customized_error_msg(ECustomizedError.UNSUPPORT_TAG_MANIFEST) + '\n'

From ec88c91744bd9502386b37c6cadb575cdde11dfa Mon Sep 17 00:00:00 2001
From: zhiren <zzhan@indocresearch.org>
Date: Tue, 11 Apr 2023 15:50:51 -0400
Subject: [PATCH 24/62] add new test case for manifest attaching

---
 .../file_upload/test_file_upload.py           | 28 +++++++++++++++++--
 1 file changed, 26 insertions(+), 2 deletions(-)

diff --git a/tests/app/services/file_manager/file_upload/test_file_upload.py b/tests/app/services/file_manager/file_upload/test_file_upload.py
index c171e887..ef774fed 100644
--- a/tests/app/services/file_manager/file_upload/test_file_upload.py
+++ b/tests/app/services/file_manager/file_upload/test_file_upload.py
@@ -111,8 +111,32 @@ def test_folder_upload_tagging_should_block(mocker, capfd):
         'file': file_name,
         'project_code': 'test_project',
         'tags': ['test_tag'],
-        'zone': 0,
-        'manifest': 'test_manifest',
+        'zone': 'greenroom',
+    }
+
+    mocker.patch('os.path.isdir', return_value=True)
+
+    try:
+        simple_upload(upload_event)
+    except SystemExit:
+        out, _ = capfd.readouterr()
+
+        expect = (
+            f'Starting upload of: {file_name}\n' + customized_error_msg(ECustomizedError.UNSUPPORT_TAG_MANIFEST) + '\n'
+        )
+
+        assert out == expect
+    else:
+        AssertionError('SystemExit not raised')
+
+
+def test_folder_upload_manifest_should_block(mocker, capfd):
+    file_name = 'test'
+    upload_event = {
+        'file': file_name,
+        'project_code': 'test_project',
+        'zone': 'greenroom',
+        'attribute': 'test_manifest',
     }
 
     mocker.patch('os.path.isdir', return_value=True)

From 2a6f66eeea1d1c7c28760e3b1319406c83c0cf6f Mon Sep 17 00:00:00 2001
From: zhiren <zzhan@indocresearch.org>
Date: Tue, 11 Apr 2023 16:25:37 -0400
Subject: [PATCH 25/62] need merge back

---
 app/commands/file.py                                 |  7 +++++--
 app/services/file_manager/file_manifests.py          |  8 ++++----
 app/services/file_manager/file_upload/file_upload.py | 10 +++++++---
 3 files changed, 16 insertions(+), 9 deletions(-)

diff --git a/app/commands/file.py b/app/commands/file.py
index ceb65054..0912c95e 100644
--- a/app/commands/file.py
+++ b/app/commands/file.py
@@ -215,9 +215,12 @@ def file_put(**kwargs):  # noqa: C901
         if source_file:
             upload_event['valid_source'] = src_file_info
 
-        simple_upload(upload_event, num_of_thread=thread, resumable_id=resumable_id, job_id=job_id, item_id=item_id)
+        file_objects = simple_upload(
+            upload_event, num_of_thread=thread, resumable_id=resumable_id, job_id=job_id, item_id=item_id
+        )
 
-        srv_manifest.attach_manifest(attribute, result_file, zone) if attribute else None
+        # since only file upload can attach manifest, take the first file object
+        srv_manifest.attach_manifest(attribute, file_objects[0].item_id, zone) if attribute else None
         message_handler.SrvOutPutHandler.all_file_uploaded()
 
 
diff --git a/app/services/file_manager/file_manifests.py b/app/services/file_manager/file_manifests.py
index 7756a48f..e6e98093 100644
--- a/app/services/file_manager/file_manifests.py
+++ b/app/services/file_manager/file_manifests.py
@@ -61,9 +61,9 @@ def validate_template(self, manifest_json):
             return False, res_json
 
     @require_valid_token()
-    def attach(self, manifest_json, file_name, zone):
+    def attach(self, manifest_json: dict, item_id: str, zone: str):
         url = self.app_config.Connections.url_bff + '/v1/manifest/attach'
-        manifest_json['file_name'] = file_name
+        manifest_json['item_id'] = item_id
         manifest_json['zone'] = zone
         headers = {
             'Authorization': 'Bearer ' + self.user.access_token,
@@ -151,8 +151,8 @@ def validate_manifest(self, manifest, raise_error=True):
             validation_error = ''
         return validation, validation_error
 
-    def attach_manifest(self, manifest, file_name, zone):
-        res = self.attach(manifest, file_name, zone)
+    def attach_manifest(self, manifest: dict, item_id: str, zone: str):
+        res = self.attach(manifest, item_id, zone)
         if res.get('code') != 200:
             error = res.get('error_msg')
             if self.interactive:
diff --git a/app/services/file_manager/file_upload/file_upload.py b/app/services/file_manager/file_upload/file_upload.py
index f7f88feb..1da186aa 100644
--- a/app/services/file_manager/file_upload/file_upload.py
+++ b/app/services/file_manager/file_upload/file_upload.py
@@ -8,6 +8,7 @@
 import zipfile
 from multiprocessing.pool import ThreadPool
 from typing import Dict
+from typing import List
 from typing import Tuple
 
 import click
@@ -15,6 +16,7 @@
 import app.services.logger_services.log_functions as logger
 import app.services.output_manager.message_handler as mhandler
 from app.configs.app_config import AppConfig
+from app.services.file_manager.file_upload.models import FileObject
 from app.services.file_manager.file_upload.models import UploadType
 from app.services.file_manager.file_upload.upload_client import UploadClient
 from app.services.output_manager.error_handler import ECustomizedError
@@ -106,7 +108,7 @@ def simple_upload(  # noqa: C901
     resumable_id: str = None,
     job_id: str = None,
     item_id: str = None,
-):
+) -> List[FileObject]:
     upload_start_time = time.time()
     my_file = upload_event.get('file')
     project_code = upload_event.get('project_code')
@@ -120,7 +122,7 @@ def simple_upload(  # noqa: C901
     compress_zip = upload_event.get('compress_zip', False)
     regular_file = upload_event.get('regular_file', True)
     source_file = upload_event.get('valid_source')
-    attribute = upload_event.get('attribute')
+    # attribute = upload_event.get('attribute')
 
     mhandler.SrvOutPutHandler.start_uploading(my_file)
     # TODO: PILOT-2392 simplify the logic under
@@ -200,7 +202,7 @@ def simple_upload(  # noqa: C901
     pool.close()
     pool.join()
 
-    if source_file or attribute:
+    if source_file:
         continue_loop = True
         while continue_loop:
             # the last uploaded file
@@ -213,3 +215,5 @@ def simple_upload(  # noqa: C901
 
     num_of_file = len(upload_file_path)
     logger.info(f'Upload Time: {time.time() - upload_start_time:.2f}s for {num_of_file:d} files')
+
+    return pre_upload_infos

From 9bb1728f4f58ee188a73e8fed6c254e4fa747fdc Mon Sep 17 00:00:00 2001
From: zhiren <zzhan@indocresearch.org>
Date: Tue, 11 Apr 2023 16:40:11 -0400
Subject: [PATCH 26/62] change the option --resumable-file to
 --resumable-manifest

---
 app/commands/file.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/commands/file.py b/app/commands/file.py
index ad0686a6..25296508 100644
--- a/app/commands/file.py
+++ b/app/commands/file.py
@@ -216,7 +216,7 @@ def file_put(**kwargs):  # noqa: C901
     show_default=True,
 )
 @click.option(
-    '--resumable-file',
+    '--resumable-manifest',
     '-r',
     default=None,
     required=True,

From 2dbc68b8e1794de0a8846c2530c7d23867394237 Mon Sep 17 00:00:00 2001
From: zhiren <zzhan@indocresearch.org>
Date: Tue, 11 Apr 2023 16:48:38 -0400
Subject: [PATCH 27/62] fixup naming

---
 app/commands/file.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/commands/file.py b/app/commands/file.py
index 25296508..c17ed119 100644
--- a/app/commands/file.py
+++ b/app/commands/file.py
@@ -235,7 +235,7 @@ def file_resume(**kwargs):  # noqa: C901
     """
 
     thread = kwargs.get('thread')
-    resumable_manifest_file = kwargs.get('resumable_file')
+    resumable_manifest_file = kwargs.get('resumable_manifest')
 
     # check if manifest file exist then read the manifest file as json
     if not os.path.exists(resumable_manifest_file):

From c0884d76de02a3a80cc55bec2dc8d37a8e2ccb08 Mon Sep 17 00:00:00 2001
From: zhiren <zzhan@indocresearch.org>
Date: Tue, 11 Apr 2023 16:49:43 -0400
Subject: [PATCH 28/62] Revert "need merge back"

This reverts commit 2a6f66eeea1d1c7c28760e3b1319406c83c0cf6f.
---
 app/commands/file.py                                 |  7 ++-----
 app/services/file_manager/file_manifests.py          |  8 ++++----
 app/services/file_manager/file_upload/file_upload.py | 10 +++-------
 3 files changed, 9 insertions(+), 16 deletions(-)

diff --git a/app/commands/file.py b/app/commands/file.py
index 0912c95e..ceb65054 100644
--- a/app/commands/file.py
+++ b/app/commands/file.py
@@ -215,12 +215,9 @@ def file_put(**kwargs):  # noqa: C901
         if source_file:
             upload_event['valid_source'] = src_file_info
 
-        file_objects = simple_upload(
-            upload_event, num_of_thread=thread, resumable_id=resumable_id, job_id=job_id, item_id=item_id
-        )
+        simple_upload(upload_event, num_of_thread=thread, resumable_id=resumable_id, job_id=job_id, item_id=item_id)
 
-        # since only file upload can attach manifest, take the first file object
-        srv_manifest.attach_manifest(attribute, file_objects[0].item_id, zone) if attribute else None
+        srv_manifest.attach_manifest(attribute, result_file, zone) if attribute else None
         message_handler.SrvOutPutHandler.all_file_uploaded()
 
 
diff --git a/app/services/file_manager/file_manifests.py b/app/services/file_manager/file_manifests.py
index e6e98093..7756a48f 100644
--- a/app/services/file_manager/file_manifests.py
+++ b/app/services/file_manager/file_manifests.py
@@ -61,9 +61,9 @@ def validate_template(self, manifest_json):
             return False, res_json
 
     @require_valid_token()
-    def attach(self, manifest_json: dict, item_id: str, zone: str):
+    def attach(self, manifest_json, file_name, zone):
         url = self.app_config.Connections.url_bff + '/v1/manifest/attach'
-        manifest_json['item_id'] = item_id
+        manifest_json['file_name'] = file_name
         manifest_json['zone'] = zone
         headers = {
             'Authorization': 'Bearer ' + self.user.access_token,
@@ -151,8 +151,8 @@ def validate_manifest(self, manifest, raise_error=True):
             validation_error = ''
         return validation, validation_error
 
-    def attach_manifest(self, manifest: dict, item_id: str, zone: str):
-        res = self.attach(manifest, item_id, zone)
+    def attach_manifest(self, manifest, file_name, zone):
+        res = self.attach(manifest, file_name, zone)
         if res.get('code') != 200:
             error = res.get('error_msg')
             if self.interactive:
diff --git a/app/services/file_manager/file_upload/file_upload.py b/app/services/file_manager/file_upload/file_upload.py
index 1da186aa..f7f88feb 100644
--- a/app/services/file_manager/file_upload/file_upload.py
+++ b/app/services/file_manager/file_upload/file_upload.py
@@ -8,7 +8,6 @@
 import zipfile
 from multiprocessing.pool import ThreadPool
 from typing import Dict
-from typing import List
 from typing import Tuple
 
 import click
@@ -16,7 +15,6 @@
 import app.services.logger_services.log_functions as logger
 import app.services.output_manager.message_handler as mhandler
 from app.configs.app_config import AppConfig
-from app.services.file_manager.file_upload.models import FileObject
 from app.services.file_manager.file_upload.models import UploadType
 from app.services.file_manager.file_upload.upload_client import UploadClient
 from app.services.output_manager.error_handler import ECustomizedError
@@ -108,7 +106,7 @@ def simple_upload(  # noqa: C901
     resumable_id: str = None,
     job_id: str = None,
     item_id: str = None,
-) -> List[FileObject]:
+):
     upload_start_time = time.time()
     my_file = upload_event.get('file')
     project_code = upload_event.get('project_code')
@@ -122,7 +120,7 @@ def simple_upload(  # noqa: C901
     compress_zip = upload_event.get('compress_zip', False)
     regular_file = upload_event.get('regular_file', True)
     source_file = upload_event.get('valid_source')
-    # attribute = upload_event.get('attribute')
+    attribute = upload_event.get('attribute')
 
     mhandler.SrvOutPutHandler.start_uploading(my_file)
     # TODO: PILOT-2392 simplify the logic under
@@ -202,7 +200,7 @@ def simple_upload(  # noqa: C901
     pool.close()
     pool.join()
 
-    if source_file:
+    if source_file or attribute:
         continue_loop = True
         while continue_loop:
             # the last uploaded file
@@ -215,5 +213,3 @@ def simple_upload(  # noqa: C901
 
     num_of_file = len(upload_file_path)
     logger.info(f'Upload Time: {time.time() - upload_start_time:.2f}s for {num_of_file:d} files')
-
-    return pre_upload_infos

From cee2da5ecc73b53c285eb04718969471c2349665 Mon Sep 17 00:00:00 2001
From: zhiren <zzhan@indocresearch.org>
Date: Tue, 11 Apr 2023 17:07:06 -0400
Subject: [PATCH 29/62] fix up test case

---
 tests/app/commands/test_file.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/app/commands/test_file.py b/tests/app/commands/test_file.py
index 55aa78f1..d560be53 100644
--- a/tests/app/commands/test_file.py
+++ b/tests/app/commands/test_file.py
@@ -12,16 +12,16 @@ def test_resumable_upload_command_success(mocker, cli_runner):
     # mock the open function
     mocked_open_data = mocker.mock_open(read_data='test')
     mocker.patch('builtins.open', mocked_open_data)
-    mocker.patch('json.load', return_value={'resumable_file': 'test.json', 'thread': 1})
+    mocker.patch('json.load', return_value={'resumable_manifest': 'test.json', 'thread': 1})
     mocker.patch('app.commands.file.resume_upload', return_value=None)
 
-    result = cli_runner.invoke(file_resume, ['--resumable-file', 'test.json', '--thread', 1])
+    result = cli_runner.invoke(file_resume, ['--resumable-manifest', 'test.json', '--thread', 1])
     assert result.exit_code == 0
 
 
 def test_resumable_upload_command_failed_with_file_not_exists(mocker, cli_runner):
     mocker.patch('os.path.exists', return_value=False)
 
-    result = cli_runner.invoke(file_resume, ['--resumable-file', 'test.json', '--thread', 1])
+    result = cli_runner.invoke(file_resume, ['--resumable-manifest', 'test.json', '--thread', 1])
     assert result.exit_code == 0
     assert result.output == customized_error_msg(ECustomizedError.INVALID_RESUMABLE) + '\n'

From 06ea2697254b308d7641fc68b87841e8415b855f Mon Sep 17 00:00:00 2001
From: zhiren <zzhan@indocresearch.org>
Date: Tue, 11 Apr 2023 17:24:30 -0400
Subject: [PATCH 30/62] bumpup the version

---
 pyproject.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pyproject.toml b/pyproject.toml
index 40fd566e..1adb61ca 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "app"
-version = "2.2.0"
+version = "2.2.1"
 description = "This service is designed to support pilot platform"
 authors = ["Indoc Research"]
 

From 8e6aedfdf5e5f74b3bedc350a96177a589125d07 Mon Sep 17 00:00:00 2001
From: zhiren <zzhan@indocresearch.org>
Date: Tue, 11 Apr 2023 17:36:46 -0400
Subject: [PATCH 31/62] update the item_path in attribute attachment to item_id

---
 app/commands/file.py                                 | 7 +++++--
 app/services/file_manager/file_manifests.py          | 8 ++++----
 app/services/file_manager/file_upload/file_upload.py | 6 +++++-
 3 files changed, 14 insertions(+), 7 deletions(-)

diff --git a/app/commands/file.py b/app/commands/file.py
index f1defc0d..5f21a22d 100644
--- a/app/commands/file.py
+++ b/app/commands/file.py
@@ -215,9 +215,12 @@ def file_put(**kwargs):  # noqa: C901
         if source_file:
             upload_event['valid_source'] = src_file_info
 
-        simple_upload(upload_event, num_of_thread=thread, resumable_id=resumable_id, job_id=job_id, item_id=item_id)
+        file_objects = simple_upload(
+            upload_event, num_of_thread=thread, resumable_id=resumable_id, job_id=job_id, item_id=item_id
+        )
 
-        srv_manifest.attach_manifest(attribute, result_file, zone) if attribute else None
+        # since only file upload can attach manifest, take the first file object
+        srv_manifest.attach_manifest(attribute, file_objects[0], zone) if attribute else None
         message_handler.SrvOutPutHandler.all_file_uploaded()
 
 
diff --git a/app/services/file_manager/file_manifests.py b/app/services/file_manager/file_manifests.py
index 7756a48f..e6e98093 100644
--- a/app/services/file_manager/file_manifests.py
+++ b/app/services/file_manager/file_manifests.py
@@ -61,9 +61,9 @@ def validate_template(self, manifest_json):
             return False, res_json
 
     @require_valid_token()
-    def attach(self, manifest_json, file_name, zone):
+    def attach(self, manifest_json: dict, item_id: str, zone: str):
         url = self.app_config.Connections.url_bff + '/v1/manifest/attach'
-        manifest_json['file_name'] = file_name
+        manifest_json['item_id'] = item_id
         manifest_json['zone'] = zone
         headers = {
             'Authorization': 'Bearer ' + self.user.access_token,
@@ -151,8 +151,8 @@ def validate_manifest(self, manifest, raise_error=True):
             validation_error = ''
         return validation, validation_error
 
-    def attach_manifest(self, manifest, file_name, zone):
-        res = self.attach(manifest, file_name, zone)
+    def attach_manifest(self, manifest: dict, item_id: str, zone: str):
+        res = self.attach(manifest, item_id, zone)
         if res.get('code') != 200:
             error = res.get('error_msg')
             if self.interactive:
diff --git a/app/services/file_manager/file_upload/file_upload.py b/app/services/file_manager/file_upload/file_upload.py
index 93680427..5e5d251a 100644
--- a/app/services/file_manager/file_upload/file_upload.py
+++ b/app/services/file_manager/file_upload/file_upload.py
@@ -8,6 +8,7 @@
 import zipfile
 from multiprocessing.pool import ThreadPool
 from typing import Dict
+from typing import List
 from typing import Tuple
 
 import click
@@ -15,6 +16,7 @@
 import app.services.logger_services.log_functions as logger
 import app.services.output_manager.message_handler as mhandler
 from app.configs.app_config import AppConfig
+from app.services.file_manager.file_upload.models import FileObject
 from app.services.file_manager.file_upload.models import UploadType
 from app.services.file_manager.file_upload.upload_client import UploadClient
 from app.services.output_manager.error_handler import ECustomizedError
@@ -108,7 +110,7 @@ def simple_upload(  # noqa: C901
     resumable_id: str = None,
     job_id: str = None,
     item_id: str = None,
-):
+) -> List[FileObject]:
     upload_start_time = time.time()
     my_file = upload_event.get('file')
     project_code = upload_event.get('project_code')
@@ -215,3 +217,5 @@ def simple_upload(  # noqa: C901
 
     num_of_file = len(upload_file_path)
     logger.info(f'Upload Time: {time.time() - upload_start_time:.2f}s for {num_of_file:d} files')
+
+    return pre_upload_infos

From 0f010d719468f0b7a16288b09854e5eb5f0f4489 Mon Sep 17 00:00:00 2001
From: QXgu <amy.gubca@gmail.com>
Date: Tue, 11 Apr 2023 21:53:27 -0400
Subject: [PATCH 32/62] PILOT-2677: Fix the error message error when download
 empty folder

---
 .../file_manager/file_download/download_client.py        | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/app/services/file_manager/file_download/download_client.py b/app/services/file_manager/file_download/download_client.py
index 7ee59a08..2586e580 100644
--- a/app/services/file_manager/file_download/download_client.py
+++ b/app/services/file_manager/file_download/download_client.py
@@ -90,18 +90,19 @@ def prepare_download(self):
         }
         url = self.appconfig.Connections.url_v2_download_pre % (self.project_code)
         res = resilient_session().post(url, headers=headers, json=payload)
-        res_json = res.json().get('result')
+        res_json = res.json()
         self.check_point = True
 
         if res.status_code == 200:
             # fetch the info from hash token
-            self.hash_code = res_json.get('payload', {}).get('hash_code')
+            response = res.json().get('result')
+            self.hash_code = response.get('payload', {}).get('hash_code')
             download_info = jwt.decode(self.hash_code, options={'verify_signature': False})
             file_path = download_info.get('file_path')
-            pre_status = EFileStatus(res_json.get('status'))
+            pre_status = EFileStatus(response.get('status'))
         elif res.status_code == 403:
             SrvErrorHandler.customized_handle(ECustomizedError.NO_FILE_PERMMISION, self.interactive)
-        elif res.status_code == 400 and res_json.get('error_msg') == 'Folder is empty':
+        elif res.status_code == 400 and 'number of file must greater than 0' in res_json.get('error_msg'):
             SrvErrorHandler.customized_handle(ECustomizedError.FOLDER_EMPTY, self.interactive)
         else:
             SrvErrorHandler.customized_handle(ECustomizedError.DOWNLOAD_FAIL, self.interactive)

From 2067509d180cc9cd743b1231bccb781dc62440f7 Mon Sep 17 00:00:00 2001
From: zhiren <zzhan@indocresearch.org>
Date: Wed, 12 Apr 2023 09:03:25 -0400
Subject: [PATCH 33/62] update with request changes

---
 app/services/file_manager/file_upload/models.py        | 4 ++--
 app/services/file_manager/file_upload/upload_client.py | 8 ++++----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/app/services/file_manager/file_upload/models.py b/app/services/file_manager/file_upload/models.py
index c84ad4ef..ff4728ca 100644
--- a/app/services/file_manager/file_upload/models.py
+++ b/app/services/file_manager/file_upload/models.py
@@ -20,7 +20,7 @@ class UploadType(Enum):
     AS_FOLDER = 'AS_FOLDER'
 
     def __str__(self):
-        return '%s' % self.name
+        return self.name
 
 
 class ItemStatus(str, Enum):
@@ -31,7 +31,7 @@ class ItemStatus(str, Enum):
     ARCHIVED = 'ARCHIVED'  # the file has been deleted
 
     def __str__(self):
-        return '%s' % self.name
+        return self.name
 
 
 class FileObject:
diff --git a/app/services/file_manager/file_upload/upload_client.py b/app/services/file_manager/file_upload/upload_client.py
index 66e887f4..a1a95461 100644
--- a/app/services/file_manager/file_upload/upload_client.py
+++ b/app/services/file_manager/file_upload/upload_client.py
@@ -127,11 +127,11 @@ def resume_upload(self, unfinished_file_objects: List[FileObject]) -> List[FileO
             'zone': self.zone,
             'object_infos': [
                 {
-                    'object_path': x.object_path,
-                    'item_id': x.item_id,
-                    'resumable_id': x.resumable_id,
+                    'object_path': file_object.object_path,
+                    'item_id': file_object.item_id,
+                    'resumable_id': file_object.resumable_id,
                 }
-                for x in unfinished_file_objects
+                for file_object in unfinished_file_objects
             ],
         }
 

From faba92f752036b7e57c9dc253fa72df77d5a3a5a Mon Sep 17 00:00:00 2001
From: zhiren <zzhan@indocresearch.org>
Date: Wed, 12 Apr 2023 09:34:55 -0400
Subject: [PATCH 34/62] update docstring for ItemStatus

---
 app/services/file_manager/file_upload/models.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/services/file_manager/file_upload/models.py b/app/services/file_manager/file_upload/models.py
index ff4728ca..85993a80 100644
--- a/app/services/file_manager/file_upload/models.py
+++ b/app/services/file_manager/file_upload/models.py
@@ -24,7 +24,7 @@ def __str__(self):
 
 
 class ItemStatus(str, Enum):
-    # the new enum type for file status
+    """Enum type for item status where."""
 
     REGISTERED = 'REGISTERED'  # file is created by upload service but not complete yet. either in progress or fail.
     ACTIVE = 'ACTIVE'  # file uploading is complete.

From b31e8509deae44b5acdcefe5b39784dc927086cc Mon Sep 17 00:00:00 2001
From: zhiren <zzhan@indocresearch.org>
Date: Wed, 12 Apr 2023 09:36:46 -0400
Subject: [PATCH 35/62] update docstring for ItemStatus

---
 app/services/file_manager/file_upload/models.py | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/app/services/file_manager/file_upload/models.py b/app/services/file_manager/file_upload/models.py
index 85993a80..e8560cde 100644
--- a/app/services/file_manager/file_upload/models.py
+++ b/app/services/file_manager/file_upload/models.py
@@ -24,11 +24,17 @@ def __str__(self):
 
 
 class ItemStatus(str, Enum):
-    """Enum type for item status where."""
+    """Enum type for item status where:
 
-    REGISTERED = 'REGISTERED'  # file is created by upload service but not complete yet. either in progress or fail.
-    ACTIVE = 'ACTIVE'  # file uploading is complete.
-    ARCHIVED = 'ARCHIVED'  # the file has been deleted
+        - REGISTERED means file is created by upload service but not complete yet. either in progress or fail.
+        - ACTIVE means file uploading is complete.
+        - ARCHIVED means the file has been deleted
+    The status will be stored at metadata table.
+    """
+
+    REGISTERED = 'REGISTERED'
+    ACTIVE = 'ACTIVE'
+    ARCHIVED = 'ARCHIVED'
 
     def __str__(self):
         return self.name

From c427fddf52a802a063c6d8c2b4830e4f0e210edd Mon Sep 17 00:00:00 2001
From: zhiren <zzhan@indocresearch.org>
Date: Wed, 12 Apr 2023 09:36:59 -0400
Subject: [PATCH 36/62] update docstring for ItemStatus

---
 app/services/file_manager/file_upload/models.py | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/app/services/file_manager/file_upload/models.py b/app/services/file_manager/file_upload/models.py
index e8560cde..6f548652 100644
--- a/app/services/file_manager/file_upload/models.py
+++ b/app/services/file_manager/file_upload/models.py
@@ -24,12 +24,13 @@ def __str__(self):
 
 
 class ItemStatus(str, Enum):
-    """Enum type for item status where:
-
-        - REGISTERED means file is created by upload service but not complete yet. either in progress or fail.
-        - ACTIVE means file uploading is complete.
-        - ARCHIVED means the file has been deleted
-    The status will be stored at metadata table.
+    """
+    Summary:
+        Enum type for item status where:
+            - REGISTERED means file is created by upload service but not complete yet. either in progress or fail.
+            - ACTIVE means file uploading is complete.
+            - ARCHIVED means the file has been deleted
+        The status will be stored at metadata table.
     """
 
     REGISTERED = 'REGISTERED'

From 6cf2ce99508853893daa272963fa5c42303ee156 Mon Sep 17 00:00:00 2001
From: zhiren <zzhan@indocresearch.org>
Date: Wed, 12 Apr 2023 09:40:22 -0400
Subject: [PATCH 37/62] bumpup the version

---
 pyproject.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pyproject.toml b/pyproject.toml
index f2595b40..cadb1909 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "app"
-version = "2.2.1"
+version = "2.2.2"
 description = "This service is designed to support pilot platform"
 authors = ["Indoc Research"]
 

From 34d9219b5bea4aab70de0824c61da3bdc69657c6 Mon Sep 17 00:00:00 2001
From: zhiren <zzhan@indocresearch.org>
Date: Wed, 12 Apr 2023 10:04:01 -0400
Subject: [PATCH 38/62] update the poetry version in cicd

---
 .github/workflows/build-and-publish.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml
index 7427b3b3..7af02d06 100644
--- a/.github/workflows/build-and-publish.yml
+++ b/.github/workflows/build-and-publish.yml
@@ -39,7 +39,7 @@ jobs:
       - name: Install Poetry
         uses: snok/install-poetry@v1
         with:
-          version: 1.1.15
+          version: 1.3.2
           virtualenvs-create: true
           virtualenvs-in-project: true
           installer-parallel: true

From dd6b4798e09efe0f8a9ef6684d46ebde2b4c333f Mon Sep 17 00:00:00 2001
From: zhiren <zzhan@indocresearch.org>
Date: Wed, 12 Apr 2023 10:41:42 -0400
Subject: [PATCH 39/62] fixup the test case

---
 .../services/file_manager/file_upload/test_upload_client.py   | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/app/services/file_manager/file_upload/test_upload_client.py b/tests/app/services/file_manager/file_upload/test_upload_client.py
index 3f181e4b..03ce331a 100644
--- a/tests/app/services/file_manager/file_upload/test_upload_client.py
+++ b/tests/app/services/file_manager/file_upload/test_upload_client.py
@@ -38,7 +38,7 @@ def test_check_status_success(httpx_mock, mocker):
         status_code=200,
     )
 
-    test_obj = FileObject('test', 'test', 'test', 'test', 'test', [])
+    test_obj = FileObject('test', 'test', 'test', 'test', 'test')
     result = upload_client.check_status(test_obj)
 
     assert result is True
@@ -57,7 +57,7 @@ def test_check_status_fail(httpx_mock, mocker):
         status_code=200,
     )
 
-    test_obj = FileObject('test', 'test', 'test', 'test', 'test', [])
+    test_obj = FileObject('test', 'test', 'test', 'test', 'test')
     result = upload_client.check_status(test_obj)
 
     assert result is False

From 5f6ca44a6dad075addfc7e9847087d9e141061ee Mon Sep 17 00:00:00 2001
From: zhiren <zzhan@indocresearch.org>
Date: Wed, 12 Apr 2023 10:43:28 -0400
Subject: [PATCH 40/62] fixup the test case

---
 app/services/crypto/crypto.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/app/services/crypto/crypto.py b/app/services/crypto/crypto.py
index 81fb6792..026d2d87 100644
--- a/app/services/crypto/crypto.py
+++ b/app/services/crypto/crypto.py
@@ -76,5 +76,4 @@ def decryption(encrypted_message, secret, interactive=True):
             else:
                 raise ex
     else:
-        pass
-        # ehandler.SrvErrorHandler.customized_handle(ehandler.ECustomizedError.LOGIN_SESSION_INVALID, True)
+        ehandler.SrvErrorHandler.customized_handle(ehandler.ECustomizedError.LOGIN_SESSION_INVALID, True)

From 8e46af004f0b79c351131a394e69d3da37a119ae Mon Sep 17 00:00:00 2001
From: zhiren <zzhan@indocresearch.org>
Date: Wed, 12 Apr 2023 11:45:50 -0400
Subject: [PATCH 41/62] fixup the token fresh will immediately exit after the
 async function

---
 app/configs/app_config.py                     |  2 +-
 .../file_manager/file_upload/file_upload.py   | 23 +++++++++++++++----
 2 files changed, 20 insertions(+), 5 deletions(-)

diff --git a/app/configs/app_config.py b/app/configs/app_config.py
index 3d2ba3e8..12b2bf45 100644
--- a/app/configs/app_config.py
+++ b/app/configs/app_config.py
@@ -13,7 +13,7 @@ class Env(object):
         msg_path = ConfigClass.custom_path
         user_config_file = f'{user_config_path}/config.ini'
         token_warn_need_refresh = 250  # refresh token if token is about to expire
-        token_refresh_interval = 120  # auto refresh token every 2 minutes
+        token_refresh_interval = 10  # auto refresh token every 2 minutes
 
         # NOTE: there is a limitation on minio that
         # the multipart number is 10000. so we set
diff --git a/app/services/file_manager/file_upload/file_upload.py b/app/services/file_manager/file_upload/file_upload.py
index 8b1b7e2e..be4e747b 100644
--- a/app/services/file_manager/file_upload/file_upload.py
+++ b/app/services/file_manager/file_upload/file_upload.py
@@ -181,14 +181,22 @@ def simple_upload(  # noqa: C901
 
     pool = ThreadPool(num_of_thread + 1)
     pool.apply_async(upload_client.upload_token_refresh)
+    on_success_res = []
     for file_object in pre_upload_infos:
         chunk_res = upload_client.stream_upload(file_object, pool)
         # NOTE: if there is some racing error make the combine chunks
         # out of thread pool.
-        pool.apply_async(
+        res = pool.apply_async(
             upload_client.on_succeed,
             args=(file_object, tags, chunk_res),
         )
+        on_success_res.append(res)
+
+    # finish the upload once all on success api return
+    # otherwise wait for 1 second and check again
+    for res in on_success_res:
+        while res.get() is None:
+            time.sleep(1)
     upload_client.set_finish_upload()
 
     pool.close()
@@ -263,15 +271,22 @@ def resume_upload(
 
     pool = ThreadPool(num_of_thread + 1)
     pool.apply_async(upload_client.upload_token_refresh)
+    on_success_res = []
     for file_object in unfinished_items:
-        upload_client.stream_upload(file_object, pool)
+        chunk_res = upload_client.stream_upload(file_object, pool)
         # NOTE: if there is some racing error make the combine chunks
         # out of thread pool.
-        pool.apply_async(
+        res = pool.apply_async(
             upload_client.on_succeed,
-            args=(file_object, manifest_json.get('tags')),
+            args=(file_object, manifest_json.get('tags'), chunk_res),
         )
+        on_success_res.append(res)
 
+    # finish the upload once all on success api return
+    # otherwise wait for 1 second and check again
+    for res in on_success_res:
+        while res.get() is None:
+            time.sleep(1)
     upload_client.set_finish_upload()
 
     pool.close()

From 112c076ee1a4582b444869ca5e58fef328c36c34 Mon Sep 17 00:00:00 2001
From: zhiren <zzhan@indocresearch.org>
Date: Wed, 12 Apr 2023 11:50:29 -0400
Subject: [PATCH 42/62] update the refresh interval

---
 app/configs/app_config.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/configs/app_config.py b/app/configs/app_config.py
index 12b2bf45..6abd7013 100644
--- a/app/configs/app_config.py
+++ b/app/configs/app_config.py
@@ -12,8 +12,8 @@ class Env(object):
         user_config_path = ConfigClass.config_path
         msg_path = ConfigClass.custom_path
         user_config_file = f'{user_config_path}/config.ini'
-        token_warn_need_refresh = 250  # refresh token if token is about to expire
-        token_refresh_interval = 10  # auto refresh token every 2 minutes
+        token_warn_need_refresh = 30  # refresh token if token is about to expire
+        token_refresh_interval = 120  # auto refresh token every 2 minutes
 
         # NOTE: there is a limitation on minio that
         # the multipart number is 10000. so we set

From d56e3f161034e20281b992627e5264250b5b04d5 Mon Sep 17 00:00:00 2001
From: zhiren <zzhan@indocresearch.org>
Date: Wed, 12 Apr 2023 14:59:15 -0400
Subject: [PATCH 43/62] update the test case

---
 .../file_manager/file_upload/test_file_upload.py      | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/tests/app/services/file_manager/file_upload/test_file_upload.py b/tests/app/services/file_manager/file_upload/test_file_upload.py
index 0e1282a4..2faca549 100644
--- a/tests/app/services/file_manager/file_upload/test_file_upload.py
+++ b/tests/app/services/file_manager/file_upload/test_file_upload.py
@@ -4,12 +4,12 @@
 
 from app.configs.app_config import AppConfig
 from app.services.file_manager.file_upload.file_upload import assemble_path
-from app.services.file_manager.file_upload.file_upload import simple_upload
-from app.services.output_manager.error_handler import ECustomizedError
-from app.services.output_manager.error_handler import customized_error_msg
 from app.services.file_manager.file_upload.file_upload import resume_upload
+from app.services.file_manager.file_upload.file_upload import simple_upload
 from app.services.file_manager.file_upload.models import FileObject
 from app.services.file_manager.file_upload.models import ItemStatus
+from app.services.output_manager.error_handler import ECustomizedError
+from app.services.output_manager.error_handler import customized_error_msg
 
 
 def test_assemble_path_at_name_folder(mocker):
@@ -108,7 +108,8 @@ def test_assemble_path_at_non_existing_folder(mocker):
     assert parent_folder.get('name') == 'admin'
     assert create_folder_flag is True
 
-def test_folder_upload_tagging_should_block(mocker, capfd):
+
+def test_dont_allow_tagging_when_folder_upload(mocker, capfd):
     file_name = 'test'
     upload_event = {
         'file': file_name,
@@ -133,7 +134,7 @@ def test_folder_upload_tagging_should_block(mocker, capfd):
         AssertionError('SystemExit not raised')
 
 
-def test_folder_upload_manifest_should_block(mocker, capfd):
+def test_dont_allow_attribute_attaching_when_folder_upload(mocker, capfd):
     file_name = 'test'
     upload_event = {
         'file': file_name,

From 605d83a4eefdff88a4f7fb76ca0fe1b7f4b70d7f Mon Sep 17 00:00:00 2001
From: zhiren <zzhan@indocresearch.org>
Date: Wed, 12 Apr 2023 15:22:36 -0400
Subject: [PATCH 44/62] adding the item id as identifier for manifest attaching

---
 app/commands/file.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/commands/file.py b/app/commands/file.py
index 5f21a22d..a8fd5224 100644
--- a/app/commands/file.py
+++ b/app/commands/file.py
@@ -220,7 +220,7 @@ def file_put(**kwargs):  # noqa: C901
         )
 
         # since only file upload can attach manifest, take the first file object
-        srv_manifest.attach_manifest(attribute, file_objects[0], zone) if attribute else None
+        srv_manifest.attach_manifest(attribute, file_objects[0].item_id, zone) if attribute else None
         message_handler.SrvOutPutHandler.all_file_uploaded()
 
 

From e03209b745dc0c56f1a941fe82a8deef59666fb8 Mon Sep 17 00:00:00 2001
From: zhiren <zzhan@indocresearch.org>
Date: Wed, 12 Apr 2023 16:31:56 -0400
Subject: [PATCH 45/62] add the test case for normal upload w/o attribute

---
 tests/app/commands/test_file.py | 54 +++++++++++++++++++++++++++++++++
 1 file changed, 54 insertions(+)

diff --git a/tests/app/commands/test_file.py b/tests/app/commands/test_file.py
index d560be53..160299b1 100644
--- a/tests/app/commands/test_file.py
+++ b/tests/app/commands/test_file.py
@@ -2,11 +2,65 @@
 #
 # Contact Indoc Research for any questions regarding the use of this source code.
 
+import click
+
+from app.commands.file import file_put
 from app.commands.file import file_resume
+from app.services.file_manager.file_upload.models import FileObject
 from app.services.output_manager.error_handler import ECustomizedError
 from app.services.output_manager.error_handler import customized_error_msg
 
 
+def test_file_upload_command_success(mocker, cli_runner):
+    project_code = 'test_project'
+    target_folder = 'admin'
+
+    mocker.patch('app.commands.file.identify_target_folder', return_value=(project_code, target_folder))
+    mocker.patch('app.commands.file.validate_upload_event', return_value={'source_file': '', 'attribute': ''})
+    mocker.patch('app.commands.file.assemble_path', return_value=('test', {'id': 'id'}, True, 'test'))
+    simple_upload_mock = mocker.patch('app.commands.file.simple_upload', return_value=None)
+
+    # create a test file
+    runner = click.testing.CliRunner()
+    with runner.isolated_filesystem():
+        with open('test.txt', 'w') as f:
+            f.write('test.txt')
+
+        result = cli_runner.invoke(file_put, ['--project-path', 'test', '--thread', 1, 'test.txt'])
+    assert result.exit_code == 0
+    simple_upload_mock.assert_called_once()
+
+
+def test_file_upload_command_success_with_attribute(mocker, cli_runner):
+    project_code = 'test_project'
+    target_folder = 'admin'
+
+    mocker.patch('app.commands.file.identify_target_folder', return_value=(project_code, target_folder))
+    mocker.patch('app.commands.file.validate_upload_event', return_value={'source_file': '', 'attribute': 'test'})
+    mocker.patch('app.commands.file.assemble_path', return_value=('test', {'id': 'id'}, True, 'test'))
+
+    mocker.patch('app.services.file_manager.file_upload.models.FileObject.generate_meta', return_value=(1, 1))
+    test_obj = FileObject('resumable_id', 'job_id', 'item_id', 'object/path', 'local_path', [])
+
+    simple_upload_mock = mocker.patch('app.commands.file.simple_upload', return_value=[test_obj])
+    attribute_mock = mocker.patch(
+        'app.services.file_manager.file_manifests.SrvFileManifests.attach_manifest', return_value=None
+    )
+
+    # create a test file
+    runner = click.testing.CliRunner()
+    with runner.isolated_filesystem():
+        with open('test.txt', 'w') as f:
+            f.write('test.txt')
+
+        result = cli_runner.invoke(
+            file_put, ['--project-path', 'test', '--thread', 1, '--attribute', 'test.json', 'test.txt']
+        )
+    assert result.exit_code == 0
+    simple_upload_mock.assert_called_once()
+    attribute_mock.assert_called_once()
+
+
 def test_resumable_upload_command_success(mocker, cli_runner):
     mocker.patch('os.path.exists', return_value=True)
     # mock the open function

From d1184e88c92f26529a1e5bc1d5b6dd5344b01d0a Mon Sep 17 00:00:00 2001
From: zhiren <zzhan@indocresearch.org>
Date: Thu, 13 Apr 2023 10:20:36 -0400
Subject: [PATCH 46/62] add back the toekn refresh interval

---
 app/configs/app_config.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/configs/app_config.py b/app/configs/app_config.py
index 12b2bf45..3d2ba3e8 100644
--- a/app/configs/app_config.py
+++ b/app/configs/app_config.py
@@ -13,7 +13,7 @@ class Env(object):
         msg_path = ConfigClass.custom_path
         user_config_file = f'{user_config_path}/config.ini'
         token_warn_need_refresh = 250  # refresh token if token is about to expire
-        token_refresh_interval = 10  # auto refresh token every 2 minutes
+        token_refresh_interval = 120  # auto refresh token every 2 minutes
 
         # NOTE: there is a limitation on minio that
         # the multipart number is 10000. so we set

From 0d4cb0482aaac141928db44eb3302e68ea0765c5 Mon Sep 17 00:00:00 2001
From: zhiren <zzhan@indocresearch.org>
Date: Thu, 13 Apr 2023 10:37:00 -0400
Subject: [PATCH 47/62] add back the time interval

---
 app/configs/app_config.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/configs/app_config.py b/app/configs/app_config.py
index 12b2bf45..3d2ba3e8 100644
--- a/app/configs/app_config.py
+++ b/app/configs/app_config.py
@@ -13,7 +13,7 @@ class Env(object):
         msg_path = ConfigClass.custom_path
         user_config_file = f'{user_config_path}/config.ini'
         token_warn_need_refresh = 250  # refresh token if token is about to expire
-        token_refresh_interval = 10  # auto refresh token every 2 minutes
+        token_refresh_interval = 120  # auto refresh token every 2 minutes
 
         # NOTE: there is a limitation on minio that
         # the multipart number is 10000. so we set

From b65b64f48f3c6e17f07c238f65fea9a86bac1db9 Mon Sep 17 00:00:00 2001
From: zhiren <zzhan@indocresearch.org>
Date: Thu, 13 Apr 2023 10:37:44 -0400
Subject: [PATCH 48/62] add back the refresh interval

---
 app/configs/app_config.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/configs/app_config.py b/app/configs/app_config.py
index 6abd7013..3d2ba3e8 100644
--- a/app/configs/app_config.py
+++ b/app/configs/app_config.py
@@ -12,7 +12,7 @@ class Env(object):
         user_config_path = ConfigClass.config_path
         msg_path = ConfigClass.custom_path
         user_config_file = f'{user_config_path}/config.ini'
-        token_warn_need_refresh = 30  # refresh token if token is about to expire
+        token_warn_need_refresh = 250  # refresh token if token is about to expire
         token_refresh_interval = 120  # auto refresh token every 2 minutes
 
         # NOTE: there is a limitation on minio that

From 9d13ac1406ecff1d9f99b650768ba236f42e49a2 Mon Sep 17 00:00:00 2001
From: zhiren <zzhan@indocresearch.org>
Date: Thu, 13 Apr 2023 10:50:13 -0400
Subject: [PATCH 49/62] manually merged

---
 app/services/file_manager/file_upload/file_upload.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/app/services/file_manager/file_upload/file_upload.py b/app/services/file_manager/file_upload/file_upload.py
index fd640dce..aded3707 100644
--- a/app/services/file_manager/file_upload/file_upload.py
+++ b/app/services/file_manager/file_upload/file_upload.py
@@ -131,13 +131,12 @@ def simple_upload(  # noqa: C901
     if os.path.isdir(input_path):
         job_type = UploadType.AS_FILE if compress_zip else UploadType.AS_FOLDER
         if job_type == UploadType.AS_FILE:
-            upload_file_path = [my_file.rstrip('/').lstrip() + '.zip']
-            target_folder = '/'.join(target_folder.split('/')[:-1]).rstrip('/')
-            compress_folder_to_zip(my_file)
+            upload_file_path = [input_path.rstrip('/').lstrip() + '.zip']
+            compress_folder_to_zip(input_path)
         elif tags or attribute:
             SrvErrorHandler.customized_handle(ECustomizedError.UNSUPPORT_TAG_MANIFEST, True)
         else:
-            upload_file_path = get_file_in_folder(my_file)
+            upload_file_path = get_file_in_folder(input_path)
     else:
         upload_file_path = [input_path]
 
@@ -189,7 +188,7 @@ def simple_upload(  # noqa: C901
 
     pool = ThreadPool(num_of_thread + 1)
     pool.apply_async(upload_client.upload_token_refresh)
-    on_succeed_res = []
+    on_success_res = []
     for file_object in pre_upload_infos:
         chunk_res = upload_client.stream_upload(file_object, pool)
         # NOTE: if there is some racing error make the combine chunks

From 052e304ed5e8eeb424b2e3acace4f283b2ebe741 Mon Sep 17 00:00:00 2001
From: zhiren <zzhan@indocresearch.org>
Date: Thu, 13 Apr 2023 11:39:11 -0400
Subject: [PATCH 50/62] update the while loop into wait() function

---
 app/commands/file.py                                |  4 ++--
 .../file_manager/file_upload/file_upload.py         | 13 +++++--------
 2 files changed, 7 insertions(+), 10 deletions(-)

diff --git a/app/commands/file.py b/app/commands/file.py
index 516777df..39b287af 100644
--- a/app/commands/file.py
+++ b/app/commands/file.py
@@ -200,10 +200,10 @@ def file_put(**kwargs):  # noqa: C901
         if source_file:
             upload_event['valid_source'] = src_file_info
 
-        file_objects = simple_upload(upload_event, num_of_thread=thread, output_path=output_path)
+        item_ids = simple_upload(upload_event, num_of_thread=thread, output_path=output_path)
 
         # since only file upload can attach manifest, take the first file object
-        srv_manifest.attach_manifest(attribute, file_objects[0].item_id, zone) if attribute else None
+        srv_manifest.attach_manifest(attribute, item_ids[0].item_id, zone) if attribute else None
         message_handler.SrvOutPutHandler.all_file_uploaded()
 
 
diff --git a/app/services/file_manager/file_upload/file_upload.py b/app/services/file_manager/file_upload/file_upload.py
index 92207b1a..eea78d61 100644
--- a/app/services/file_manager/file_upload/file_upload.py
+++ b/app/services/file_manager/file_upload/file_upload.py
@@ -9,6 +9,7 @@
 from multiprocessing.pool import ThreadPool
 from typing import Any
 from typing import Dict
+from typing import List
 from typing import Tuple
 
 import click
@@ -107,7 +108,7 @@ def simple_upload(  # noqa: C901
     upload_event,
     num_of_thread: int = 1,
     output_path: str = None,
-):
+) -> List[str]:
     upload_start_time = time.time()
     my_file = upload_event.get('file')
     project_code = upload_event.get('project_code')
@@ -194,9 +195,7 @@ def simple_upload(  # noqa: C901
 
     # finish the upload once all on success api return
     # otherwise wait for 1 second and check again
-    for res in on_success_res:
-        while res.get() is None:
-            time.sleep(1)
+    [res.wait() for res in on_success_res]
     upload_client.set_finish_upload()
 
     pool.close()
@@ -216,7 +215,7 @@ def simple_upload(  # noqa: C901
     num_of_file = len(upload_file_path)
     logger.info(f'Upload Time: {time.time() - upload_start_time:.2f}s for {num_of_file:d} files')
 
-    return pre_upload_infos
+    return [file_object.item_id for file_object in pre_upload_infos]
 
 
 def resume_upload(
@@ -286,9 +285,7 @@ def resume_upload(
 
     # finish the upload once all on success api return
     # otherwise wait for 1 second and check again
-    for res in on_success_res:
-        while res.get() is None:
-            time.sleep(1)
+    [res.wait() for res in on_success_res]
     upload_client.set_finish_upload()
 
     pool.close()

From 3c6a0afae2b3ef14f0a4a472af19673b6fad8e9d Mon Sep 17 00:00:00 2001
From: zhiren <zzhan@indocresearch.org>
Date: Thu, 13 Apr 2023 11:39:48 -0400
Subject: [PATCH 51/62] remove the unnecessary test

---
 tests/app/commands/test_file.py | 20 --------------------
 1 file changed, 20 deletions(-)

diff --git a/tests/app/commands/test_file.py b/tests/app/commands/test_file.py
index 160299b1..d6f25f49 100644
--- a/tests/app/commands/test_file.py
+++ b/tests/app/commands/test_file.py
@@ -11,26 +11,6 @@
 from app.services.output_manager.error_handler import customized_error_msg
 
 
-def test_file_upload_command_success(mocker, cli_runner):
-    project_code = 'test_project'
-    target_folder = 'admin'
-
-    mocker.patch('app.commands.file.identify_target_folder', return_value=(project_code, target_folder))
-    mocker.patch('app.commands.file.validate_upload_event', return_value={'source_file': '', 'attribute': ''})
-    mocker.patch('app.commands.file.assemble_path', return_value=('test', {'id': 'id'}, True, 'test'))
-    simple_upload_mock = mocker.patch('app.commands.file.simple_upload', return_value=None)
-
-    # create a test file
-    runner = click.testing.CliRunner()
-    with runner.isolated_filesystem():
-        with open('test.txt', 'w') as f:
-            f.write('test.txt')
-
-        result = cli_runner.invoke(file_put, ['--project-path', 'test', '--thread', 1, 'test.txt'])
-    assert result.exit_code == 0
-    simple_upload_mock.assert_called_once()
-
-
 def test_file_upload_command_success_with_attribute(mocker, cli_runner):
     project_code = 'test_project'
     target_folder = 'admin'

From 1783d53a92caba2a65330654cb8a92ed8577b5d7 Mon Sep 17 00:00:00 2001
From: zhiren <zzhan@indocresearch.org>
Date: Thu, 13 Apr 2023 12:09:31 -0400
Subject: [PATCH 52/62] fixup the test case

---
 test                            | 0
 tests/app/commands/test_file.py | 2 +-
 2 files changed, 1 insertion(+), 1 deletion(-)
 create mode 100644 test

diff --git a/test b/test
new file mode 100644
index 00000000..e69de29b
diff --git a/tests/app/commands/test_file.py b/tests/app/commands/test_file.py
index d6f25f49..a82fd154 100644
--- a/tests/app/commands/test_file.py
+++ b/tests/app/commands/test_file.py
@@ -20,7 +20,7 @@ def test_file_upload_command_success_with_attribute(mocker, cli_runner):
     mocker.patch('app.commands.file.assemble_path', return_value=('test', {'id': 'id'}, True, 'test'))
 
     mocker.patch('app.services.file_manager.file_upload.models.FileObject.generate_meta', return_value=(1, 1))
-    test_obj = FileObject('resumable_id', 'job_id', 'item_id', 'object/path', 'local_path', [])
+    test_obj = FileObject('resumable_id', 'job_id', 'item_id', 'object/path', 'local_path')
 
     simple_upload_mock = mocker.patch('app.commands.file.simple_upload', return_value=[test_obj])
     attribute_mock = mocker.patch(

From 20cc11221a3d19d37ff6fc14d8cd47de7e3a099a Mon Sep 17 00:00:00 2001
From: zhiren <zzhan@indocresearch.org>
Date: Thu, 13 Apr 2023 15:22:04 -0400
Subject: [PATCH 53/62] fixup the item_id issue

---
 app/commands/file.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/commands/file.py b/app/commands/file.py
index 39b287af..4d4aacca 100644
--- a/app/commands/file.py
+++ b/app/commands/file.py
@@ -203,7 +203,7 @@ def file_put(**kwargs):  # noqa: C901
         item_ids = simple_upload(upload_event, num_of_thread=thread, output_path=output_path)
 
         # since only file upload can attach manifest, take the first file object
-        srv_manifest.attach_manifest(attribute, item_ids[0].item_id, zone) if attribute else None
+        srv_manifest.attach_manifest(attribute, item_ids[0], zone) if attribute else None
         message_handler.SrvOutPutHandler.all_file_uploaded()
 
 

From db3df9a741ab1bfda1a0d4eb4cc01a8d11f8ff17 Mon Sep 17 00:00:00 2001
From: zhiren <zzhan@indocresearch.org>
Date: Thu, 13 Apr 2023 15:41:38 -0400
Subject: [PATCH 54/62] add the constraint in file upload to block file size
 with 0

---
 .../file_manager/file_upload/file_upload.py      | 16 ++++++++++++----
 .../file_manager/file_upload/upload_client.py    |  4 +---
 test                                             |  0
 3 files changed, 13 insertions(+), 7 deletions(-)
 delete mode 100644 test

diff --git a/app/services/file_manager/file_upload/file_upload.py b/app/services/file_manager/file_upload/file_upload.py
index f0f8b26f..86815d06 100644
--- a/app/services/file_manager/file_upload/file_upload.py
+++ b/app/services/file_manager/file_upload/file_upload.py
@@ -165,7 +165,14 @@ def simple_upload(  # noqa: C901
         # first remove the input path from the file path
         file_path_sub = file.replace(input_path + '/', '')
         object_path = os.path.join(target_folder, file_path_sub)
-        file_objects.append(FileObject(object_path, file))
+
+        # generate a placeholder for each file
+        file_object = FileObject(object_path, file)
+        # skip the file with 0 size
+        if file_object.total_size == 0:
+            logger.warning(f'Skip the file with 0 size: {file_object.file_name}')
+        else:
+            file_objects.append(FileObject(object_path, file))
 
     # here add the batch of 500 per loop, the pre upload api cannot
     # process very large amount of file at same time. otherwise it will timeout
@@ -190,10 +197,11 @@ def simple_upload(  # noqa: C901
     pool = ThreadPool(num_of_thread + 1)
     pool.apply_async(upload_client.upload_token_refresh)
     on_success_res = []
+
+    file_object: FileObject
     for file_object in pre_upload_infos:
         chunk_res = upload_client.stream_upload(file_object, pool)
-        # NOTE: if there is some racing error make the combine chunks
-        # out of thread pool.
+        # the on_success api will be called after all chunk uploaded
         res = pool.apply_async(
             upload_client.on_succeed,
             args=(file_object, tags, chunk_res),
@@ -219,7 +227,7 @@ def simple_upload(  # noqa: C901
             upload_client.create_file_lineage(source_file)
             os.remove(file_batchs[0]) if os.path.isdir(input_path) and job_type == UploadType.AS_FILE else None
 
-    num_of_file = len(upload_file_path)
+    num_of_file = len(pre_upload_infos)
     logger.info(f'Upload Time: {time.time() - upload_start_time:.2f}s for {num_of_file:d} files')
 
     return [file_object.item_id for file_object in pre_upload_infos]
diff --git a/app/services/file_manager/file_upload/upload_client.py b/app/services/file_manager/file_upload/upload_client.py
index cbeaf7f1..8dac4b2f 100644
--- a/app/services/file_manager/file_upload/upload_client.py
+++ b/app/services/file_manager/file_upload/upload_client.py
@@ -364,9 +364,7 @@ def on_succeed(self, file_object: FileObject, tags: List[str], chunk_result: Lis
         """
 
         # check if all the chunks have been uploaded
-        for res in chunk_result:
-            while res.get() is None:
-                time.sleep(1)
+        [res.wait() for res in chunk_result]
 
         for i in range(AppConfig.Env.resilient_retry):
             url = self.base_url + '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/v1/files'
diff --git a/test b/test
deleted file mode 100644
index e69de29b..00000000

From b18a24986fc537139c3847b1df11638dffb5415d Mon Sep 17 00:00:00 2001
From: zhiren <zzhan@indocresearch.org>
Date: Thu, 13 Apr 2023 15:47:00 -0400
Subject: [PATCH 55/62] add the test to skip the empty file

---
 .gitignore                                        |  1 +
 .../file_manager/file_upload/test_file_upload.py  | 15 +++++++++++++++
 2 files changed, 16 insertions(+)

diff --git a/.gitignore b/.gitignore
index 7bf999b0..e33d6c8d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -155,3 +155,4 @@ integration_tests
 # cli manifest data
 ./manifest.json
 manifest.json
+test
diff --git a/tests/app/services/file_manager/file_upload/test_file_upload.py b/tests/app/services/file_manager/file_upload/test_file_upload.py
index 5044f9c4..efb2a3e0 100644
--- a/tests/app/services/file_manager/file_upload/test_file_upload.py
+++ b/tests/app/services/file_manager/file_upload/test_file_upload.py
@@ -109,6 +109,21 @@ def test_assemble_path_at_non_existing_folder(mocker):
     assert create_folder_flag is True
 
 
+def test_file_upload_skip_empty_file(mocker):
+    file_name = 'test'
+    upload_event = {
+        'file': file_name,
+        'project_code': 'test_project',
+        'zone': 'greenroom',
+    }
+
+    mocker.patch('os.path.isdir', return_value=False)
+    mocker.patch('app.services.file_manager.file_upload.models.FileObject.generate_meta', return_value=(0, 0))
+
+    item_ids = simple_upload(upload_event)
+    assert len(item_ids) == 0
+
+
 def test_dont_allow_tagging_when_folder_upload(mocker, capfd):
     file_name = 'test'
     upload_event = {

From 3f07c2846d3edeb4d7ffd96141ef2bd2b92b2a0c Mon Sep 17 00:00:00 2001
From: zhiren <zzhan@indocresearch.org>
Date: Mon, 17 Apr 2023 11:57:12 -0400
Subject: [PATCH 56/62] update cli with new lineage workflow which will create
 the lineage when pre-registration

---
 app/commands/file.py                          | 25 +--------
 app/configs/app_config.py                     |  2 +-
 app/models/upload_form.py                     |  8 ---
 app/resources/custom_error.py                 |  2 +-
 app/services/file_manager/file_lineage.py     | 32 ------------
 .../file_manager/file_upload/file_upload.py   | 13 ++---
 .../file_manager/file_upload/upload_client.py | 52 ++++---------------
 .../file_upload/upload_validator.py           | 16 ++----
 8 files changed, 23 insertions(+), 127 deletions(-)
 delete mode 100644 app/services/file_manager/file_lineage.py

diff --git a/app/commands/file.py b/app/commands/file.py
index f2970a0e..ee007762 100644
--- a/app/commands/file.py
+++ b/app/commands/file.py
@@ -4,7 +4,6 @@
 
 import json
 import os
-import re
 
 import click
 
@@ -82,13 +81,6 @@ def cli():
     help=file_help.file_help_page(file_help.FileHELP.FILE_UPLOAD_S),
     show_default=True,
 )
-@click.option(
-    '--pipeline',
-    default=None,
-    required=False,
-    help=file_help.file_help_page(file_help.FileHELP.FILE_UPLOAD_PIPELINE),
-    show_default=True,
-)
 @click.option(
     '--zip',
     default=None,
@@ -123,7 +115,6 @@ def file_put(**kwargs):  # noqa: C901
     zone = kwargs.get('zone')
     upload_message = kwargs.get('upload_message')
     source_file = kwargs.get('source_file')
-    pipeline = kwargs.get('pipeline')
     zipping = kwargs.get('zip')
     attribute = kwargs.get('attribute')
     thread = kwargs.get('thread')
@@ -152,7 +143,6 @@ def file_put(**kwargs):  # noqa: C901
         'zone': zone,
         'upload_message': upload_message,
         'source': source_file,
-        'process_pipeline': pipeline,
         'project_code': project_code,
         'token': user.access_token,
         'attribute': attribute,
@@ -162,12 +152,6 @@ def file_put(**kwargs):  # noqa: C901
     src_file_info = validated_fieds['source_file']
     attribute = validated_fieds['attribute']
     if zone == AppConfig.Env.core_zone.lower():
-        if not pipeline:
-            # after validation, if not pipeline, provide default value
-            pipeline = AppConfig.Env.pipeline_straight_upload
-        else:
-            if not bool(re.match(r'^[a-z0-9_-]{1,20}$', pipeline)):
-                SrvErrorHandler.customized_handle(ECustomizedError.INVALID_PIPELINENAME, True)
         if not upload_message:
             upload_message = AppConfig.Env.default_upload_message
 
@@ -210,10 +194,8 @@ def file_put(**kwargs):  # noqa: C901
             'compress_zip': zipping,
             'attribute': attribute,
         }
-        if pipeline:
-            upload_event['process_pipeline'] = pipeline
         if source_file:
-            upload_event['valid_source'] = src_file_info
+            upload_event['source_id'] = src_file_info.get('id')
 
         item_ids = simple_upload(upload_event, num_of_thread=thread, output_path=output_path)
 
@@ -272,14 +254,11 @@ def validate_upload_event(event):
     zone = event.get('zone')
     upload_message = event.get('upload_message')
     source = event.get('source')
-    process_pipeline = event.get('process_pipeline')
     project_code = event.get('project_code')
     token = event.get('token')
     attribute = event.get('attribute')
     tag = event.get('tag')
-    validator = UploadEventValidator(
-        project_code, zone, upload_message, source, process_pipeline, token, attribute, tag
-    )
+    validator = UploadEventValidator(project_code, zone, upload_message, source, token, attribute, tag)
     converted_content = validator.validate_upload_event()
     return converted_content
 
diff --git a/app/configs/app_config.py b/app/configs/app_config.py
index 3d2ba3e8..6abd7013 100644
--- a/app/configs/app_config.py
+++ b/app/configs/app_config.py
@@ -12,7 +12,7 @@ class Env(object):
         user_config_path = ConfigClass.config_path
         msg_path = ConfigClass.custom_path
         user_config_file = f'{user_config_path}/config.ini'
-        token_warn_need_refresh = 250  # refresh token if token is about to expire
+        token_warn_need_refresh = 30  # refresh token if token is about to expire
         token_refresh_interval = 120  # auto refresh token every 2 minutes
 
         # NOTE: there is a limitation on minio that
diff --git a/app/models/upload_form.py b/app/models/upload_form.py
index d51e7713..032d7692 100644
--- a/app/models/upload_form.py
+++ b/app/models/upload_form.py
@@ -2,8 +2,6 @@
 #
 # Contact Indoc Research for any questions regarding the use of this source code.
 
-from typing import List
-
 from app.services.file_manager.file_upload.models import FileObject
 
 
@@ -103,9 +101,7 @@ def generate_on_success_form(
     project_code: str,
     operator: str,
     file_object: FileObject,
-    tags: List[str],
     from_parents: str = None,
-    process_pipeline: str = None,
     upload_message: str = None,
 ):
     """
@@ -118,7 +114,6 @@ def generate_on_success_form(
         - file_object(FileObject): The object that contains the file information.
         - tags(list[str]): The tags that will be attached with file.
         - from_parents(str): indicate it is parent node.
-        - process_pipeline(str): the name of pipeline.
         - upload_message(str): the message for uploading.
     return:
         - request_payload(dict): the payload for preupload api.
@@ -135,12 +130,9 @@ def generate_on_success_form(
         'resumable_total_chunks': file_object.total_chunks,
         'resumable_total_size': file_object.total_size,
         'resumable_relative_path': file_object.parent_path,
-        'tags': tags,
     }
     if from_parents:
         request_payload['from_parents'] = from_parents
-    if process_pipeline:
-        request_payload['process_pipeline'] = process_pipeline
     if upload_message:
         request_payload['upload_message'] = upload_message
     return request_payload
diff --git a/app/resources/custom_error.py b/app/resources/custom_error.py
index c0264140..ed47b7be 100644
--- a/app/resources/custom_error.py
+++ b/app/resources/custom_error.py
@@ -66,7 +66,7 @@ class Error:
             'It means the resumable file is not the same with previous one.\n'
             'Please to double check the file content.'
         ),
-        'UNSUPPORT_TAG_MANIFEST': 'Tagging and manifest attaching are not supported for folder type.',
+        'UNSUPPORT_TAG_MANIFEST': 'Tagging, lineage and manifest attaching are not supported for folder type.',
         'INVALID_INPUT': 'Invalid input. Please try again.',
         'UNSUPPORTED_PROJECT': 'This function is not supported in the given Project %s',
         'CREATE_FOLDER_IF_NOT_EXIST': 'Target folder does not exist. Would you like to create a new folder?',
diff --git a/app/services/file_manager/file_lineage.py b/app/services/file_manager/file_lineage.py
deleted file mode 100644
index 439b2e04..00000000
--- a/app/services/file_manager/file_lineage.py
+++ /dev/null
@@ -1,32 +0,0 @@
-# Copyright (C) 2022-2023 Indoc Research
-#
-# Contact Indoc Research for any questions regarding the use of this source code.
-
-import requests
-
-from app.configs.app_config import AppConfig
-from app.services.output_manager.error_handler import ECustomizedError
-from app.services.output_manager.error_handler import SrvErrorHandler
-
-
-def create_lineage(lineage_event):
-    url = AppConfig.Connections.url_lineage
-    payload = {
-        'input_id': lineage_event['input_id'],
-        'output_id': lineage_event['output_id'],
-        'project_code': lineage_event['project_code'],
-        'action_type': lineage_event['action_type'],
-        'input_path': lineage_event['input_path'],
-        'output_path': lineage_event['output_path'],
-        'description': 'straight upload by ' + lineage_event['operator'],
-    }
-    headers = {
-        'Authorization': 'Bearer ' + lineage_event['token'],
-    }
-    __res = requests.post(url, json=payload, headers=headers)
-    if __res.status_code == 200:
-        return __res.json()['result']
-    else:
-        SrvErrorHandler.customized_handle(
-            ECustomizedError.INVALID_LINEAGE, True, value=str(__res.status_code) + str(__res.text)
-        )
diff --git a/app/services/file_manager/file_upload/file_upload.py b/app/services/file_manager/file_upload/file_upload.py
index f0f8b26f..c85e7c15 100644
--- a/app/services/file_manager/file_upload/file_upload.py
+++ b/app/services/file_manager/file_upload/file_upload.py
@@ -123,7 +123,7 @@ def simple_upload(  # noqa: C901
     create_folder_flag = upload_event.get('create_folder_flag', False)
     compress_zip = upload_event.get('compress_zip', False)
     regular_file = upload_event.get('regular_file', True)
-    source_file = upload_event.get('valid_source')
+    source_id = upload_event.get('source_id', None)
     attribute = upload_event.get('attribute')
 
     mhandler.SrvOutPutHandler.start_uploading(input_path)
@@ -134,7 +134,7 @@ def simple_upload(  # noqa: C901
         if job_type == UploadType.AS_FILE:
             upload_file_path = [input_path.rstrip('/').lstrip() + '.zip']
             compress_folder_to_zip(input_path)
-        elif tags or attribute:
+        elif tags or attribute or source_id:
             SrvErrorHandler.customized_handle(ECustomizedError.UNSUPPORT_TAG_MANIFEST, True)
         else:
             upload_file_path = get_file_in_folder(input_path)
@@ -143,12 +143,10 @@ def simple_upload(  # noqa: C901
 
         if create_folder_flag:
             job_type = UploadType.AS_FOLDER
-            input_path = os.path.dirname(input_path)  # update the path as folder
         else:
             job_type = UploadType.AS_FILE
 
     upload_client = UploadClient(
-        input_path=input_path,
         project_code=project_code,
         zone=zone,
         job_type=job_type,
@@ -156,11 +154,13 @@ def simple_upload(  # noqa: C901
         parent_folder_id=parent_folder_id,
         regular_file=regular_file,
         tags=tags,
+        source_id=source_id,
     )
 
     # format the local path into object storage path for preupload
     file_objects = []
     target_folder = upload_event.get('target_folder', '')
+    input_path = os.path.dirname(input_path)
     for file in upload_file_path:
         # first remove the input path from the file path
         file_path_sub = file.replace(input_path + '/', '')
@@ -208,16 +208,13 @@ def simple_upload(  # noqa: C901
     pool.close()
     pool.join()
 
-    if source_file or attribute:
+    if attribute:
         continue_loop = True
         while continue_loop:
             # the last uploaded file
             succeed = upload_client.check_status(file_object)
             continue_loop = not succeed
             time.sleep(0.5)
-        if source_file:
-            upload_client.create_file_lineage(source_file)
-            os.remove(file_batchs[0]) if os.path.isdir(input_path) and job_type == UploadType.AS_FILE else None
 
     num_of_file = len(upload_file_path)
     logger.info(f'Upload Time: {time.time() - upload_start_time:.2f}s for {num_of_file:d} files')
diff --git a/app/services/file_manager/file_upload/upload_client.py b/app/services/file_manager/file_upload/upload_client.py
index cbeaf7f1..66ec521b 100644
--- a/app/services/file_manager/file_upload/upload_client.py
+++ b/app/services/file_manager/file_upload/upload_client.py
@@ -28,10 +28,8 @@
 from app.services.user_authentication.token_manager import SrvTokenManager
 from app.utils.aggregated import get_file_info_by_geid
 from app.utils.aggregated import resilient_session
-from app.utils.aggregated import search_item
 
 from .exception import INVALID_CHUNK_ETAG
-from ..file_lineage import create_lineage
 
 
 class UploadClient:
@@ -39,7 +37,6 @@ class UploadClient:
     Summary:
         The upload client is per upload base. it stores some immutable.
         infomation of particular upload action:
-         - input_path: the path that user inputs. can be a folder or file.
          - project_code: the unique code of project.
          - zone: data zone. can be greenroom or core.
          - upload_message:
@@ -49,20 +46,18 @@ class UploadClient:
 
     def __init__(
         self,
-        input_path: str,
         project_code: str,
         parent_folder_id: str,
         zone: str = AppConfig.Env.green_zone,
         upload_message: str = 'cli straight upload',
         job_type: str = UploadType.AS_FILE,
-        process_pipeline: str = None,
         current_folder_node: str = '',
         regular_file: str = True,
         tags: list = None,
+        source_id: str = '',
     ):
         self.user = UserConfig()
         self.operator = self.user.username
-        self.input_path = input_path
         self.upload_message = upload_message
         self.chunk_size = AppConfig.Env.chunk_size  # remove
         self.base_url = {
@@ -79,11 +74,12 @@ def __init__(
         self.zone = zone
         self.job_type = job_type
         self.project_code = project_code
-        self.process_pipeline = process_pipeline
         self.current_folder_node = current_folder_node
         self.parent_folder_id = parent_folder_id
         self.regular_file = regular_file
+        # tags and souce_id are only allowed in file uplaod
         self.tags = tags
+        self.source_id = source_id
 
         # the flag to indicate if all upload process finished
         # then the token refresh loop will end
@@ -94,7 +90,7 @@ def generate_meta(self, local_path: str) -> Tuple[int, int]:
         Summary:
             The function is to generate chunk upload meatedata for a file.
         Parameter:
-            - input_path: The path of the local file eg. a/b/c.txt.
+            - local_path: The path of the local file eg. a/b/c.txt.
         return:
             - total_size: the size of file.
             - total_chunks: the number of chunks will be uploaded.
@@ -176,13 +172,13 @@ def pre_upload(self, file_objects: List[FileObject], output_path: str) -> List[F
             'current_folder_node': self.current_folder_node,
             'parent_folder_id': self.parent_folder_id,
             'folder_tags': self.tags,
+            'source_id': self.source_id,
             'data': [
                 {'resumable_filename': x.file_name, 'resumable_relative_path': x.parent_path} for x in file_objects
             ],
         }
 
         response = resilient_session().post(url, json=payload, headers=headers, timeout=None)
-
         if response.status_code == 200:
             result = response.json().get('result')
             file_mapping = {x.object_path: x for x in file_objects}
@@ -319,7 +315,8 @@ def upload_chunk(self, file_object: FileObject, chunk_number: int, chunk: str) -
             }
             headers = {'Authorization': 'Bearer ' + self.user.access_token, 'Session-ID': self.user.session_id}
             response = httpx.get(
-                self.base_url + '/v1/files/chunks/presigned',
+                # self.base_url + '/v1/files/chunks/presigned',
+                'http://localhost:5079/v1/files/chunks/presigned',
                 params=params,
                 headers=headers,
                 timeout=None,
@@ -369,14 +366,13 @@ def on_succeed(self, file_object: FileObject, tags: List[str], chunk_result: Lis
                 time.sleep(1)
 
         for i in range(AppConfig.Env.resilient_retry):
-            url = self.base_url + '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/v1/files'
+            # url = self.base_url + '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/v1/files'
+            url = 'http://localhost:5079/v1/files'
             payload = uf.generate_on_success_form(
                 self.project_code,
                 self.operator,
                 file_object,
-                tags,
                 [],
-                process_pipeline=self.process_pipeline,
                 upload_message=self.upload_message,
             )
             headers = {
@@ -399,36 +395,6 @@ def on_succeed(self, file_object: FileObject, tags: List[str], chunk_result: Lis
 
             time.sleep(AppConfig.Env.resilient_retry_interval * (i + 1))
 
-    @require_valid_token()
-    def create_file_lineage(self, source_file: dict, new_file_object: FileObject):
-        """
-        Summary:
-            The function is to create a lineage with source file.
-        Parameter:
-            - source_file(str): the file object that indicate the exist data to link with.
-            - new_file_object(FileObject): the new object just uploaded.
-        return:
-            - bool: if job success or not.
-        """
-
-        if source_file and self.zone == AppConfig.Env.core_zone:
-            child_rel_path = new_file_object.object_path
-            child_item = search_item(self.project_code, self.zone, child_rel_path, 'file')
-            child_file = child_item['result']
-            parent_file_geid = source_file['id']
-            child_file_geid = child_file['id']
-            lineage_event = {
-                'input_id': parent_file_geid,
-                'output_id': child_file_geid,
-                'input_path': os.path.join(source_file['parent_path'], source_file['name']),
-                'output_path': os.path.join(child_file['parent_path'], child_file['name']),
-                'project_code': self.project_code,
-                'action_type': self.process_pipeline,
-                'operator': self.operator,
-                'token': self.user.access_token,
-            }
-            create_lineage(lineage_event)
-
     def check_status(self, file_object: FileObject) -> bool:
         """
         Summary:
diff --git a/app/services/file_manager/file_upload/upload_validator.py b/app/services/file_manager/file_upload/upload_validator.py
index 15c2ba5d..33210e12 100644
--- a/app/services/file_manager/file_upload/upload_validator.py
+++ b/app/services/file_manager/file_upload/upload_validator.py
@@ -13,12 +13,11 @@
 
 
 class UploadEventValidator:
-    def __init__(self, project_code, zone, upload_message, source, process_pipeline, token, attribute, tag):
+    def __init__(self, project_code, zone, upload_message, source, token, attribute, tag):
         self.project_code = project_code
         self.zone = zone
         self.upload_message = upload_message
         self.source = source
-        self.process_pipeline = process_pipeline
         self.token = token
         self.attribute = attribute
         self.tag = tag
@@ -30,15 +29,10 @@ def validate_zone(self):
                 ECustomizedError.INVALID_UPLOAD_REQUEST, True, value='upload-message is required'
             )
         if self.source:
-            if not self.process_pipeline:
-                SrvErrorHandler.customized_handle(
-                    ECustomizedError.INVALID_UPLOAD_REQUEST, True, value='process pipeline name required'
-                )
-            else:
-                source_file_info = search_item(self.project_code, AppConfig.Env.green_zone.lower(), self.source, 'file')
-                source_file_info = source_file_info['result']
-                if not source_file_info:
-                    SrvErrorHandler.customized_handle(ECustomizedError.INVALID_SOURCE_FILE, True, value=self.source)
+            source_file_info = search_item(self.project_code, AppConfig.Env.green_zone.lower(), self.source, 'file')
+            source_file_info = source_file_info['result']
+            if not source_file_info:
+                SrvErrorHandler.customized_handle(ECustomizedError.INVALID_SOURCE_FILE, True, value=self.source)
         return source_file_info
 
     def validate_attribute(self):

From 4dc94d2cc9dea17de230aa01b6cf5ea5597a28b5 Mon Sep 17 00:00:00 2001
From: zhiren <zzhan@indocresearch.org>
Date: Mon, 17 Apr 2023 13:39:47 -0400
Subject: [PATCH 57/62] remove the test url

---
 .../file_manager/file_upload/file_upload.py        |  1 -
 .../file_manager/file_upload/upload_client.py      |  6 ++----
 .../file_manager/file_upload/test_upload_client.py | 14 +++++++-------
 3 files changed, 9 insertions(+), 12 deletions(-)

diff --git a/app/services/file_manager/file_upload/file_upload.py b/app/services/file_manager/file_upload/file_upload.py
index c85e7c15..02e6ed18 100644
--- a/app/services/file_manager/file_upload/file_upload.py
+++ b/app/services/file_manager/file_upload/file_upload.py
@@ -236,7 +236,6 @@ def resume_upload(
     upload_start_time = time.time()
 
     upload_client = UploadClient(
-        input_path=manifest_json.get('file'),
         project_code=manifest_json.get('project_code'),
         zone=manifest_json.get('zone'),
         job_type='AS_FOLDER',
diff --git a/app/services/file_manager/file_upload/upload_client.py b/app/services/file_manager/file_upload/upload_client.py
index 66ec521b..e200298d 100644
--- a/app/services/file_manager/file_upload/upload_client.py
+++ b/app/services/file_manager/file_upload/upload_client.py
@@ -315,8 +315,7 @@ def upload_chunk(self, file_object: FileObject, chunk_number: int, chunk: str) -
             }
             headers = {'Authorization': 'Bearer ' + self.user.access_token, 'Session-ID': self.user.session_id}
             response = httpx.get(
-                # self.base_url + '/v1/files/chunks/presigned',
-                'http://localhost:5079/v1/files/chunks/presigned',
+                self.base_url + '/v1/files/chunks/presigned',
                 params=params,
                 headers=headers,
                 timeout=None,
@@ -366,8 +365,7 @@ def on_succeed(self, file_object: FileObject, tags: List[str], chunk_result: Lis
                 time.sleep(1)
 
         for i in range(AppConfig.Env.resilient_retry):
-            # url = self.base_url + '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/v1/files'
-            url = 'http://localhost:5079/v1/files'
+            url = self.base_url + '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/v1/files'
             payload = uf.generate_on_success_form(
                 self.project_code,
                 self.operator,
diff --git a/tests/app/services/file_manager/file_upload/test_upload_client.py b/tests/app/services/file_manager/file_upload/test_upload_client.py
index 03ce331a..9a534b0d 100644
--- a/tests/app/services/file_manager/file_upload/test_upload_client.py
+++ b/tests/app/services/file_manager/file_upload/test_upload_client.py
@@ -26,7 +26,7 @@ def decorated_function(*args, **kwargs):
 
 
 def test_check_status_success(httpx_mock, mocker):
-    upload_client = UploadClient('test', 'test', 'test')
+    upload_client = UploadClient('project_code', 'parent_folder_id')
 
     mocker.patch('app.services.file_manager.file_upload.models.FileObject.generate_meta', return_value=(1, 1))
     mocker.patch('app.services.user_authentication.token_manager.SrvTokenManager.check_valid', return_value=0)
@@ -45,7 +45,7 @@ def test_check_status_success(httpx_mock, mocker):
 
 
 def test_check_status_fail(httpx_mock, mocker):
-    upload_client = UploadClient('test', 'test', 'test')
+    upload_client = UploadClient('project_code', 'parent_folder_id')
 
     mocker.patch('app.services.file_manager.file_upload.models.FileObject.generate_meta', return_value=(1, 1))
     mocker.patch('app.services.user_authentication.token_manager.SrvTokenManager.check_valid', return_value=0)
@@ -64,7 +64,7 @@ def test_check_status_fail(httpx_mock, mocker):
 
 
 def test_chunk_upload(httpx_mock, mocker):
-    upload_client = UploadClient('test', 'test', 'test')
+    upload_client = UploadClient('project_code', 'parent_folder_id')
 
     test_presigned_url = 'http://test/presigned'
     url = re.compile('^' + upload_client.base_url + '/v1/files/chunks/presigned.*$')
@@ -86,7 +86,7 @@ def test_token_refresh_auto(mocker):
         'app.services.user_authentication.token_manager.SrvTokenManager.refresh', return_value=None
     )
 
-    upload_client = UploadClient('test', 'test', 'test')
+    upload_client = UploadClient('project_code', 'parent_folder_id')
     pool = ThreadPool(2)
     async_fun = pool.apply_async(upload_client.upload_token_refresh)
     sleep(3)
@@ -111,7 +111,7 @@ def test_resumable_pre_upload_success(httpx_mock, mocker):
         return_value=decoded_token(),
     )
 
-    upload_client = UploadClient('test', 'project_code', 'parent_folder_id')
+    upload_client = UploadClient('project_code', 'parent_folder_id')
     mocker.patch('app.services.file_manager.file_upload.models.FileObject.generate_meta', return_value=(1, 1))
     test_obj = FileObject('object/path', 'local_path', 'resumable_id', 'job_id', 'item_id')
 
@@ -133,7 +133,7 @@ def test_resumable_pre_upload_failed_with_404(httpx_mock, mocker):
         return_value=decoded_token(),
     )
 
-    upload_client = UploadClient('test', 'project_code', 'parent_folder_id')
+    upload_client = UploadClient('project_code', 'parent_folder_id')
     mocker.patch('app.services.file_manager.file_upload.models.FileObject.generate_meta', return_value=(1, 1))
     test_obj = FileObject('object/path', 'local_path', 'resumable_id', 'job_id', 'item_id')
 
@@ -154,7 +154,7 @@ def test_resumable_pre_upload_failed_with_404(httpx_mock, mocker):
 
 
 def test_output_manifest_success(mocker):
-    upload_client = UploadClient('test', 'project_code', 'parent_folder_id')
+    upload_client = UploadClient('project_code', 'parent_folder_id')
     json_dump_mocker = mocker.patch('json.dump', return_value=None)
     mocker.patch('app.services.file_manager.file_upload.models.FileObject.generate_meta', return_value=(1, 1))
     test_obj = FileObject('object/path', 'local_path', 'resumable_id', 'job_id', 'item_id')

From 69314007f158209ab1b71fbfc942882dfbf4188d Mon Sep 17 00:00:00 2001
From: zhiren <zzhan@indocresearch.org>
Date: Tue, 18 Apr 2023 11:15:48 -0400
Subject: [PATCH 58/62] fixup the upload only output the manifest of last batch

---
 app/configs/app_config.py                              | 4 ++--
 app/services/file_manager/file_upload/file_upload.py   | 3 +++
 app/services/file_manager/file_upload/upload_client.py | 3 ---
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/app/configs/app_config.py b/app/configs/app_config.py
index 6abd7013..e2ba4b12 100644
--- a/app/configs/app_config.py
+++ b/app/configs/app_config.py
@@ -12,8 +12,8 @@ class Env(object):
         user_config_path = ConfigClass.config_path
         msg_path = ConfigClass.custom_path
         user_config_file = f'{user_config_path}/config.ini'
-        token_warn_need_refresh = 30  # refresh token if token is about to expire
-        token_refresh_interval = 120  # auto refresh token every 2 minutes
+        token_warn_need_refresh = 120  # refresh token if token is about to expire
+        token_refresh_interval = 90  # auto refresh token every 40 seconds
 
         # NOTE: there is a limitation on minio that
         # the multipart number is 10000. so we set
diff --git a/app/services/file_manager/file_upload/file_upload.py b/app/services/file_manager/file_upload/file_upload.py
index 02e6ed18..5d916024 100644
--- a/app/services/file_manager/file_upload/file_upload.py
+++ b/app/services/file_manager/file_upload/file_upload.py
@@ -181,6 +181,9 @@ def simple_upload(  # noqa: C901
         # the placeholder in object storage
         pre_upload_infos.extend(upload_client.pre_upload(file_batchs, output_path))
 
+    # then output manifest file to the output path
+    upload_client.output_manifest(pre_upload_infos, output_path)
+
     # now loop over each file under the folder and start
     # the chunk upload
 
diff --git a/app/services/file_manager/file_upload/upload_client.py b/app/services/file_manager/file_upload/upload_client.py
index e200298d..26f83b04 100644
--- a/app/services/file_manager/file_upload/upload_client.py
+++ b/app/services/file_manager/file_upload/upload_client.py
@@ -192,9 +192,6 @@ def pre_upload(self, file_objects: List[FileObject], output_path: str) -> List[F
                 file_object.job_id = job.get('job_id')
                 file_objets.append(file_object)
 
-            # then output manifest file to the output path
-            self.output_manifest(file_objets, output_path)
-
             mhandler.SrvOutPutHandler.preupload_success()
             return file_objets
         elif response.status_code == 403:

From bb0591337e6f5b53d25ff515689254fdae16e321 Mon Sep 17 00:00:00 2001
From: zhiren <zzhan@indocresearch.org>
Date: Tue, 18 Apr 2023 11:34:38 -0400
Subject: [PATCH 59/62] fixup the test

---
 tests/app/services/file_manager/file_upload/test_file_upload.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/app/services/file_manager/file_upload/test_file_upload.py b/tests/app/services/file_manager/file_upload/test_file_upload.py
index efb2a3e0..726ac845 100644
--- a/tests/app/services/file_manager/file_upload/test_file_upload.py
+++ b/tests/app/services/file_manager/file_upload/test_file_upload.py
@@ -120,7 +120,7 @@ def test_file_upload_skip_empty_file(mocker):
     mocker.patch('os.path.isdir', return_value=False)
     mocker.patch('app.services.file_manager.file_upload.models.FileObject.generate_meta', return_value=(0, 0))
 
-    item_ids = simple_upload(upload_event)
+    item_ids = simple_upload(upload_event, output_path='./test')
     assert len(item_ids) == 0
 
 

From af359c6bf20f663d3f4471167e657367d4a2cd77 Mon Sep 17 00:00:00 2001
From: zhiren <zzhan@indocresearch.org>
Date: Tue, 18 Apr 2023 11:36:22 -0400
Subject: [PATCH 60/62] bumpup version

---
 pyproject.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pyproject.toml b/pyproject.toml
index cadb1909..99a24127 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "app"
-version = "2.2.2"
+version = "2.2.3"
 description = "This service is designed to support pilot platform"
 authors = ["Indoc Research"]
 

From 29b19af31c982bbdcd52de6097160bd464e3faac Mon Sep 17 00:00:00 2001
From: zhiren <zzhan@indocresearch.org>
Date: Tue, 18 Apr 2023 12:00:41 -0400
Subject: [PATCH 61/62] temporary disable the lineage in this stating release

---
 app/commands/file.py                         | 5 +++++
 app/resources/custom_error.py                | 1 +
 app/resources/custom_help.py                 | 8 +++++---
 app/services/output_manager/error_handler.py | 1 +
 pyproject.toml                               | 2 +-
 5 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/app/commands/file.py b/app/commands/file.py
index ee007762..db943fef 100644
--- a/app/commands/file.py
+++ b/app/commands/file.py
@@ -120,6 +120,11 @@ def file_put(**kwargs):  # noqa: C901
     thread = kwargs.get('thread')
     output_path = kwargs.get('output_path')
 
+    # for 20230418 staging temporary disable the attribute
+    # since the backend is not ready yet
+    if source_file:
+        SrvErrorHandler.customized_handle(ECustomizedError.LINEAGE_FEATURE_NOT_READY, True)
+
     user = UserConfig()
     # Check zone and upload-message
     zone = get_zone(zone) if zone else AppConfig.Env.green_zone.lower()
diff --git a/app/resources/custom_error.py b/app/resources/custom_error.py
index ed47b7be..6f44bd34 100644
--- a/app/resources/custom_error.py
+++ b/app/resources/custom_error.py
@@ -122,4 +122,5 @@ class Error:
         'CONTAINER_REGISTRY_NO_URL': (
             'Container registry has not yet been configured. Related commands cannot be used at this time.'
         ),
+        'LINEAGE_FEATURE_NOT_READY': 'Lineage is not support at v2.3.0',
     }
diff --git a/app/resources/custom_help.py b/app/resources/custom_help.py
index 938f5844..f3dd9f2c 100644
--- a/app/resources/custom_help.py
+++ b/app/resources/custom_help.py
@@ -6,9 +6,11 @@
 class HelpPage:
     page = {
         'update': {
-            'version': '2.2.0',
-            '1': 'CLI supports to perform multi-threading upload for file/folders',
-            '2': 'CLI supports to perform resumable upload for single file',
+            'version': '2.3.0',
+            '1': 'The logic of normal upload and resumble are splited. '
+            'add new command for resumable upload as `pilotcli file resume -r manifest.json`',
+            '2': 'The manifest file will be output for both file/folder upload',
+            '3': 'Optimize logic, input and error message',
         },
         'dataset': {
             'DATASET_DOWNLOAD': 'Download a dataset or a particular version of a dataset.',
diff --git a/app/services/output_manager/error_handler.py b/app/services/output_manager/error_handler.py
index 4f9ea3f4..a0344839 100644
--- a/app/services/output_manager/error_handler.py
+++ b/app/services/output_manager/error_handler.py
@@ -81,6 +81,7 @@ class ECustomizedError(enum.Enum):
     CONTAINER_REGISTRY_NO_URL = 'CONTAINER_REGISTRY_NO_URL'
     CONFIG_NOT_FOUND = 'CONFIG_NOT_FOUND'
     CONFIG_EXIST = 'CONFIG_EXIST'
+    LINEAGE_FEATURE_NOT_READY = 'LINEAGE_FEATURE_NOT_READY'
 
 
 def customized_error_msg(customized_error: ECustomizedError):
diff --git a/pyproject.toml b/pyproject.toml
index 99a24127..18304491 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "app"
-version = "2.2.3"
+version = "2.3.0"
 description = "This service is designed to support pilot platform"
 authors = ["Indoc Research"]
 

From 71f5b3e7224c2d2fbc10982b6fc099168b3aaeef Mon Sep 17 00:00:00 2001
From: zhiren <zzhan@indocresearch.org>
Date: Tue, 18 Apr 2023 16:04:18 -0400
Subject: [PATCH 62/62] update the resumable upload as per batch operation

---
 .../file_manager/file_upload/file_upload.py   | 48 ++++++++++++-------
 .../file_manager/file_upload/upload_client.py |  2 -
 .../output_manager/message_handler.py         |  5 ++
 3 files changed, 37 insertions(+), 18 deletions(-)

diff --git a/app/services/file_manager/file_upload/file_upload.py b/app/services/file_manager/file_upload/file_upload.py
index f2a4cdd0..3b16b89c 100644
--- a/app/services/file_manager/file_upload/file_upload.py
+++ b/app/services/file_manager/file_upload/file_upload.py
@@ -256,28 +256,44 @@ def resume_upload(
     )
 
     # check files in manifest if some of them are already uploaded
-    item_ids = []
+    unfinished_items = []
     all_files = manifest_json.get('file_objects')
+    item_ids = []
     for item_id in all_files:
         item_ids.append(item_id)
-    items = get_file_info_by_geid(item_ids)
 
-    unfinished_items = []
-    for x in items:
-        if x.get('result').get('status') == ItemStatus.REGISTERED:
-            file_info = all_files.get(x.get('result').get('id'))
-            unfinished_items.append(
-                FileObject(
-                    file_info.get('object_path'),
-                    file_info.get('local_path'),
-                    file_info.get('resumable_id'),
-                    file_info.get('job_id'),
-                    file_info.get('item_id'),
+    # here add the batch of 500 per loop, the pre upload api cannot
+    # process very large amount of file at same time. otherwise it will timeout
+    num_of_batchs = math.ceil(len(all_files) / AppConfig.Env.upload_batch_size)
+    # here is list of pre upload result. We decided to call pre upload api by batch
+    for batch in range(0, num_of_batchs):
+        start_index = batch * AppConfig.Env.upload_batch_size
+        end_index = (batch + 1) * AppConfig.Env.upload_batch_size
+        file_batchs = item_ids[start_index:end_index]
+        items = get_file_info_by_geid(file_batchs)
+
+        # get the detail of item to see if the file is already uploaded
+        unfinished_files = []
+        for x in items:
+            if x.get('result').get('status') == ItemStatus.REGISTERED:
+                file_info = all_files.get(x.get('result').get('id'))
+                unfinished_files.append(
+                    FileObject(
+                        file_info.get('object_path'),
+                        file_info.get('local_path'),
+                        file_info.get('resumable_id'),
+                        file_info.get('job_id'),
+                        file_info.get('item_id'),
+                    )
                 )
-            )
 
-    # then for the rest of the files, check if any chunks are already uploaded
-    unfinished_items = upload_client.resume_upload(unfinished_items)
+        # then for the rest of the files, check if any chunks are already uploaded
+        mhandler.SrvOutPutHandler.resume_check_in_progress()
+        if len(unfinished_files) > 0:
+            unfinished_items.extend(upload_client.resume_upload(unfinished_files))
+
+    mhandler.SrvOutPutHandler.resume_warning(len(unfinished_items))
+    mhandler.SrvOutPutHandler.resume_check_success()
 
     # lastly, start resumable upload for the rest of the chunks
     # thread number +1 reserve one thread to refresh token
diff --git a/app/services/file_manager/file_upload/upload_client.py b/app/services/file_manager/file_upload/upload_client.py
index c77a9c36..e189dd89 100644
--- a/app/services/file_manager/file_upload/upload_client.py
+++ b/app/services/file_manager/file_upload/upload_client.py
@@ -114,7 +114,6 @@ def resume_upload(self, unfinished_file_objects: List[FileObject]) -> List[FileO
                 - local_path(str): the local path of file.
                 - chunk_info(dict): the mapping for chunks that already been uploaded.
         """
-        mhandler.SrvOutPutHandler.resume_warning(len(unfinished_file_objects))
 
         headers = {'Authorization': 'Bearer ' + self.user.access_token, 'Session-ID': self.user.session_id}
         url = AppConfig.Connections.url_bff + f'/v1/project/{self.project_code}/files/resumable'
@@ -142,7 +141,6 @@ def resume_upload(self, unfinished_file_objects: List[FileObject]) -> List[FileO
             file_obj = rid_file_object_map.get(uploaded_info.get('resumable_id'))
             # update the chunk info
             file_obj.uploaded_chunks = uploaded_info.get('chunks_info')
-        mhandler.SrvOutPutHandler.resume_check_success()
 
         return unfinished_file_objects
 
diff --git a/app/services/output_manager/message_handler.py b/app/services/output_manager/message_handler.py
index 65217687..2ef1dd89 100644
--- a/app/services/output_manager/message_handler.py
+++ b/app/services/output_manager/message_handler.py
@@ -133,6 +133,11 @@ def resume_check_success():
         """e.g. notify the resumable check succeed."""
         return logger.info('Resumable upload check complete.')
 
+    @staticmethod
+    def resume_check_in_progress():
+        """e.g. notify the resumable check succeed."""
+        return logger.info('Resumable upload check in progress.')
+
     @staticmethod
     def resume_warning(num_of_files: int):
         """e.g. notify the user if they comfirm the resumable upload."""