From 5fd12f7e33698deaa6b8c20e5a9643b140e793e5 Mon Sep 17 00:00:00 2001
From: zhiren <zzhan@indocresearch.org>
Date: Wed, 5 Apr 2023 10:20:46 -0400
Subject: [PATCH 01/11] backup

---
 app/commands/file.py                                   | 2 ++
 app/services/file_manager/file_upload/upload_client.py | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/app/commands/file.py b/app/commands/file.py
index eb751d52..1e938f12 100644
--- a/app/commands/file.py
+++ b/app/commands/file.py
@@ -167,6 +167,8 @@ def file_put(**kwargs):  # noqa: C901
         if not upload_message:
             upload_message = AppConfig.Env.default_upload_message
 
+    raise Exception('test')
+
     # Unique Paths
     paths = set(paths)
     # the loop will read all input path(folder or files)
diff --git a/app/services/file_manager/file_upload/upload_client.py b/app/services/file_manager/file_upload/upload_client.py
index 41dcc768..d6657056 100644
--- a/app/services/file_manager/file_upload/upload_client.py
+++ b/app/services/file_manager/file_upload/upload_client.py
@@ -151,7 +151,7 @@ def resume_upload(self, unfinished_file_objects: List[FileObject]) -> List[FileO
 
         return unfinished_file_objects
 
-    # @require_valid_token()
+    @require_valid_token()
     def pre_upload(self, local_file_paths: List[str], output_path: str) -> List[FileObject]:
         """
         Summary:

From 6f4227e121d3167b39675aa4a74539aaaaa386ec Mon Sep 17 00:00:00 2001
From: zhiren <zzhan@indocresearch.org>
Date: Wed, 5 Apr 2023 11:06:08 -0400
Subject: [PATCH 02/11] testing on the logic

---
 app/commands/file.py                          | 18 ++++++++++++++-
 app/models/upload_form.py                     |  2 ++
 app/services/crypto/crypto.py                 |  3 ++-
 .../file_manager/file_upload/file_upload.py   |  8 +++++++
 .../file_manager/file_upload/upload_client.py | 23 ++++---------------
 5 files changed, 33 insertions(+), 21 deletions(-)

diff --git a/app/commands/file.py b/app/commands/file.py
index 1e938f12..bc8928e0 100644
--- a/app/commands/file.py
+++ b/app/commands/file.py
@@ -167,7 +167,16 @@ def file_put(**kwargs):  # noqa: C901
         if not upload_message:
             upload_message = AppConfig.Env.default_upload_message
 
-    raise Exception('test')
+    # for the path formating there will be following cases:
+    # - file:
+    #   1. the project path exist, then will be AS_FILE. nothing will be changed
+    #   2. the project path not exist, then will be AS_FOLDER. the current folder node will
+    #      be the parent folder node + parent folder id. (like one level up).
+    # - folder:
+    #   1. the project path exist, then will be AS_FOLDER. the current folder node will be
+    #      the one that user input.
+    #   2. the project path not exist, then will be AS_FOLDER. the current folder node will
+    #      be the parent folder node + parent folder id. (like one level up).
 
     # Unique Paths
     paths = set(paths)
@@ -181,6 +190,12 @@ def file_put(**kwargs):  # noqa: C901
             zone,
             zipping,
         )
+
+        current_folder_node = 'testproject/admin'
+        parent_folder = {'id': 'testproject'}
+        create_folder_flag = False
+        result_file = None
+
         upload_event = {
             'project_code': project_code,
             'file': f,
@@ -193,6 +208,7 @@ def file_put(**kwargs):  # noqa: C901
             'compress_zip': zipping,
             'attribute': attribute,
         }
+        # print(upload_event)
         if pipeline:
             upload_event['process_pipeline'] = pipeline
         if source_file:
diff --git a/app/models/upload_form.py b/app/models/upload_form.py
index fcbd22c5..358b4b19 100644
--- a/app/models/upload_form.py
+++ b/app/models/upload_form.py
@@ -129,6 +129,8 @@ def generate_pre_upload_form(
         - request_payload(dict): the payload for preupload api.
         - local_file_mapping(dict): the mapping from object path into local path.
     """
+    # print(project_code, operator, local_file_paths, input_path, zone, job_type, current_folder)
+
     data, local_file_mapping = [], {}
     for file_local_path in local_file_paths:
         # the rule here is:
diff --git a/app/services/crypto/crypto.py b/app/services/crypto/crypto.py
index 026d2d87..81fb6792 100644
--- a/app/services/crypto/crypto.py
+++ b/app/services/crypto/crypto.py
@@ -76,4 +76,5 @@ def decryption(encrypted_message, secret, interactive=True):
             else:
                 raise ex
     else:
-        ehandler.SrvErrorHandler.customized_handle(ehandler.ECustomizedError.LOGIN_SESSION_INVALID, True)
+        pass
+        # ehandler.SrvErrorHandler.customized_handle(ehandler.ECustomizedError.LOGIN_SESSION_INVALID, True)
diff --git a/app/services/file_manager/file_upload/file_upload.py b/app/services/file_manager/file_upload/file_upload.py
index e65f5c03..585996c5 100644
--- a/app/services/file_manager/file_upload/file_upload.py
+++ b/app/services/file_manager/file_upload/file_upload.py
@@ -146,6 +146,12 @@ def simple_upload(  # noqa: C901
         else:
             job_type = UploadType.AS_FILE
 
+    # print('upload_file_path:', upload_file_path)
+    # print('target_folder:', target_folder)
+    # print('my_file:', my_file)
+    # print('job_type:', job_type)
+    # print('zone:', zone)
+
     upload_client = UploadClient(
         input_path=my_file,
         project_code=project_code,
@@ -157,6 +163,8 @@ def simple_upload(  # noqa: C901
         tags=tags,
     )
 
+    # print()
+
     # here add the batch of 500 per loop, the pre upload api cannot
     # process very large amount of file at same time. otherwise it will timeout
     num_of_batchs = math.ceil(len(upload_file_path) / AppConfig.Env.upload_batch_size)
diff --git a/app/services/file_manager/file_upload/upload_client.py b/app/services/file_manager/file_upload/upload_client.py
index d6657056..df990d73 100644
--- a/app/services/file_manager/file_upload/upload_client.py
+++ b/app/services/file_manager/file_upload/upload_client.py
@@ -4,14 +4,12 @@
 
 import hashlib
 import json
-import math
 import os
 import time
 from multiprocessing.pool import ThreadPool
 from typing import Any
 from typing import Dict
 from typing import List
-from typing import Tuple
 
 import httpx
 
@@ -90,21 +88,6 @@ def __init__(
         # then the token refresh loop will end
         self.finish_upload = False
 
-    def generate_meta(self, local_path: str) -> Tuple[int, int]:
-        """
-        Summary:
-            The function is to generate chunk upload meatedata for a file.
-        Parameter:
-            - input_path: The path of the local file eg. a/b/c.txt.
-        return:
-            - total_size: the size of file.
-            - total_chunks: the number of chunks will be uploaded.
-        """
-        file_length_in_bytes = os.path.getsize(local_path)
-        total_size = file_length_in_bytes
-        total_chunks = math.ceil(total_size / self.chunk_size)
-        return total_size, total_chunks
-
     # @require_valid_token()
     def resume_upload(self, unfinished_file_objects: List[FileObject]) -> List[FileObject]:
         """
@@ -151,7 +134,7 @@ def resume_upload(self, unfinished_file_objects: List[FileObject]) -> List[FileO
 
         return unfinished_file_objects
 
-    @require_valid_token()
+    # @require_valid_token()
     def pre_upload(self, local_file_paths: List[str], output_path: str) -> List[FileObject]:
         """
         Summary:
@@ -166,7 +149,7 @@ def pre_upload(self, local_file_paths: List[str], output_path: str) -> List[File
                 - local_path(str): the local path of file.
                 - chunk_info(dict): the mapping for chunks that already been uploaded.
         """
-
+        # print('pre upload')
         headers = {'Authorization': 'Bearer ' + self.user.access_token, 'Session-ID': self.user.session_id}
         url = AppConfig.Connections.url_bff + '/v1/project/{}/files'.format(self.project_code)
         # the file mapping is a dictionary that present the map from object storage path
@@ -183,6 +166,8 @@ def pre_upload(self, local_file_paths: List[str], output_path: str) -> List[File
 
         payload.update({'parent_folder_id': self.parent_folder_id})
         payload.update({'folder_tags': self.tags})
+        # print('pre upload payload: ', payload)
+        # raise Exception('pre upload')
         response = resilient_session().post(url, json=payload, headers=headers, timeout=None)
         if response.status_code == 200:
             result = response.json().get('result')

From 15aeaea8693db7b1db93cda43ce246227c28f968 Mon Sep 17 00:00:00 2001
From: zhiren <zzhan@indocresearch.org>
Date: Wed, 5 Apr 2023 16:22:31 -0400
Subject: [PATCH 03/11] refactor with upload logic

---
 app/commands/file.py                          |   7 +-
 app/models/upload_form.py                     | 133 +++++++++---------
 .../file_manager/file_upload/file_upload.py   |  40 +++---
 .../file_manager/file_upload/models.py        |   8 +-
 .../file_manager/file_upload/upload_client.py |  25 ++--
 5 files changed, 112 insertions(+), 101 deletions(-)

diff --git a/app/commands/file.py b/app/commands/file.py
index bc8928e0..85dc0094 100644
--- a/app/commands/file.py
+++ b/app/commands/file.py
@@ -183,6 +183,7 @@ def file_put(**kwargs):  # noqa: C901
     # the loop will read all input path(folder or files)
     # and process them one by one
     for f in paths:
+        # so this function will always return the furthest folder node as current_folder_node+parent_folder_id
         current_folder_node, parent_folder, create_folder_flag, result_file = assemble_path(
             f,
             target_folder,
@@ -191,14 +192,14 @@ def file_put(**kwargs):  # noqa: C901
             zipping,
         )
 
-        current_folder_node = 'testproject/admin'
+        current_folder_node = 'testproject/admin/test11'
         parent_folder = {'id': 'testproject'}
-        create_folder_flag = False
+        create_folder_flag = True
         result_file = None
 
         upload_event = {
             'project_code': project_code,
-            'file': f,
+            'file': f.rstrip('/'),  # remove the ending slash
             'tags': tag if tag else [],
             'zone': zone,
             'upload_message': upload_message,
diff --git a/app/models/upload_form.py b/app/models/upload_form.py
index 358b4b19..85ecadde 100644
--- a/app/models/upload_form.py
+++ b/app/models/upload_form.py
@@ -2,13 +2,14 @@
 #
 # Contact Indoc Research for any questions regarding the use of this source code.
 
-from os.path import basename
-from os.path import dirname
-from os.path import join
+# from os.path import basename
+# from os.path import dirname
+# from os.path import join
 from typing import List
 
 from app.services.file_manager.file_upload.models import FileObject
-from app.services.file_manager.file_upload.models import UploadType
+
+# from app.services.file_manager.file_upload.models import UploadType
 
 
 class FileUploadForm:
@@ -103,68 +104,68 @@ def metadatas(self, metadatas):
         self._attribute_map['metadatas'] = metadatas
 
 
-def generate_pre_upload_form(
-    project_code: str,
-    operator: str,
-    local_file_paths: List[str],
-    input_path: str,
-    zone: str,
-    job_type: UploadType,
-    current_folder: str = '',
-) -> tuple[dict, dict]:
-    """
-    Summary:
-        The function is to generate the preupload payload for api. The operation
-        is per batch that it will try to generate one payload for all files.
-    Parameter:
-        - project_code(str): The unique identifier for project.
-        - operator(str): The name of operator.
-        - local_file_paths(list[str]): The list of name for input files.
-        - input_path: The path specified by user, if it is folder, it will be like
-            a/b . If it is a file it will be same as local_file_paths eg. a/b/c.txt.
-        - zone(str): The zone of user try to upload to.
-        - job_type(UploadType): the upload type, AS_FOLDER or AS_FILE.
-        - current_folder(str): the folder path on object storage that user specified.
-    return:
-        - request_payload(dict): the payload for preupload api.
-        - local_file_mapping(dict): the mapping from object path into local path.
-    """
-    # print(project_code, operator, local_file_paths, input_path, zone, job_type, current_folder)
-
-    data, local_file_mapping = [], {}
-    for file_local_path in local_file_paths:
-        # the rule here is:
-        # - if use input as a folder then <input_path> is the folder user key in
-        #   eg. a/b/ . the <local_file_paths> is files under eg a/b/c/d.txt. The
-        #   path in object storage will be <current_folder>/c/d.txt
-        # - if use input as a file then <input_path> is the file user key in eg.
-        #   a/b/c/d.txt. the <local_file_paths> will be same as it. The path in
-        #   object storage will be <current_folder>/d.txt
-        if job_type == UploadType.AS_FOLDER:
-            file_relative_path = file_local_path.replace(input_path + '/', '')
-            object_path = join(current_folder, file_relative_path)
-            parent_path, file_name = dirname(object_path), basename(object_path)
-
-        else:
-            file_name = basename(file_local_path)
-            parent_path = current_folder
-
-        data.append({'resumable_filename': file_name, 'resumable_relative_path': parent_path})
-        # make a mapping as <object_path>: <local_path>. This will be returned
-        # and used in chunk upload api.
-        object_path = join(parent_path, file_name)
-        local_file_mapping.update({object_path: file_local_path})
-
-    request_payload = {
-        'project_code': project_code,
-        'operator': operator,
-        'job_type': str(job_type),
-        'zone': zone,
-        'current_folder_node': current_folder,
-        'data': data,
-    }
-
-    return request_payload, local_file_mapping
+# def generate_pre_upload_form(
+#     project_code: str,
+#     operator: str,
+#     local_file_paths: List[str],
+#     input_path: str,
+#     zone: str,
+#     job_type: UploadType,
+#     current_folder: str = '',
+# ) -> tuple[dict, dict]:
+#     """
+#     Summary:
+#         The function is to generate the preupload payload for api. The operation
+#         is per batch that it will try to generate one payload for all files.
+#     Parameter:
+#         - project_code(str): The unique identifier for project.
+#         - operator(str): The name of operator.
+#         - local_file_paths(list[str]): The list of name for input files.
+#         - input_path: The path specified by user, if it is folder, it will be like
+#             a/b . If it is a file it will be same as local_file_paths eg. a/b/c.txt.
+#         - zone(str): The zone of user try to upload to.
+#         - job_type(UploadType): the upload type, AS_FOLDER or AS_FILE.
+#         - current_folder(str): the folder path on object storage that user specified.
+#     return:
+#         - request_payload(dict): the payload for preupload api.
+#         - local_file_mapping(dict): the mapping from object path into local path.
+#     """
+#     print(project_code, operator, local_file_paths, input_path, zone, job_type, current_folder)
+
+#     data, local_file_mapping = [], {}
+#     for file_local_path in local_file_paths:
+#         # the rule here is:
+#         # - if use input as a folder then <input_path> is the folder user key in
+#         #   eg. a/b/ . the <local_file_paths> is files under eg a/b/c/d.txt. The
+#         #   path in object storage will be <current_folder>/c/d.txt
+#         # - if use input as a file then <input_path> is the file user key in eg.
+#         #   a/b/c/d.txt. the <local_file_paths> will be same as it. The path in
+#         #   object storage will be <current_folder>/d.txt
+#         if job_type == UploadType.AS_FOLDER:
+#             file_relative_path = file_local_path.replace(input_path + '/', '')
+#             object_path = join(current_folder, file_relative_path)
+#             parent_path, file_name = dirname(object_path), basename(object_path)
+
+#         else:
+#             file_name = basename(file_local_path)
+#             parent_path = current_folder
+
+#         data.append({'resumable_filename': file_name, 'resumable_relative_path': parent_path})
+#         # make a mapping as <object_path>: <local_path>. This will be returned
+#         # and used in chunk upload api.
+#         object_path = join(parent_path, file_name)
+#         local_file_mapping.update({object_path: file_local_path})
+
+#     request_payload = {
+#         'project_code': project_code,
+#         'operator': operator,
+#         'job_type': str(job_type),
+#         'zone': zone,
+#         'current_folder_node': current_folder,
+#         'data': data,
+#     }
+
+#     return request_payload, local_file_mapping
 
 
 def generate_on_success_form(
diff --git a/app/services/file_manager/file_upload/file_upload.py b/app/services/file_manager/file_upload/file_upload.py
index 585996c5..de143610 100644
--- a/app/services/file_manager/file_upload/file_upload.py
+++ b/app/services/file_manager/file_upload/file_upload.py
@@ -107,7 +107,7 @@ def simple_upload(  # noqa: C901
     output_path: str = None,
 ):
     upload_start_time = time.time()
-    my_file = upload_event.get('file')
+    input_path = upload_event.get('file')
     project_code = upload_event.get('project_code')
     tags = upload_event.get('tags')
     zone = upload_event.get('zone')
@@ -121,39 +121,37 @@ def simple_upload(  # noqa: C901
     source_file = upload_event.get('valid_source')
     attribute = upload_event.get('attribute')
 
-    mhandler.SrvOutPutHandler.start_uploading(my_file)
+    mhandler.SrvOutPutHandler.start_uploading(input_path)
     # TODO: PILOT-2392 simplify the logic under
     # if the input request zip folder then process the path as single file
     # otherwise read throught the folder to get path underneath
-    if os.path.isdir(my_file):
+    if os.path.isdir(input_path):
         job_type = UploadType.AS_FILE if compress_zip else UploadType.AS_FOLDER
         if job_type == UploadType.AS_FILE:
-            upload_file_path = [my_file.rstrip('/').lstrip() + '.zip']
+            upload_file_path = [input_path.rstrip('/').lstrip() + '.zip']
             target_folder = '/'.join(target_folder.split('/')[:-1]).rstrip('/')
-            compress_folder_to_zip(my_file)
-        elif job_type == UploadType.AS_FOLDER:
-            SrvErrorHandler.customized_handle(ECustomizedError.UNSUPPORTED_PROJECT, True, project_code)
+            compress_folder_to_zip(input_path)
         else:
             logger.warning('Current version does not support folder tagging, ' 'any selected tags will be ignored')
-            upload_file_path = get_file_in_folder(my_file)
+            upload_file_path = get_file_in_folder(input_path)
     else:
-        upload_file_path = [my_file]
-        target_folder = '/'.join(target_folder.split('/')[:-1]).rstrip('/')
+        upload_file_path = [input_path]
 
         if create_folder_flag:
             job_type = UploadType.AS_FOLDER
-            my_file = os.path.dirname(my_file)  # update the path as folder
+            input_path = os.path.dirname(input_path)  # update the path as folder
+            target_folder = '/'.join(target_folder.split('/')[:-1]).rstrip('/')
         else:
             job_type = UploadType.AS_FILE
 
     # print('upload_file_path:', upload_file_path)
     # print('target_folder:', target_folder)
-    # print('my_file:', my_file)
+    # print('input_path:', input_path)
     # print('job_type:', job_type)
     # print('zone:', zone)
 
     upload_client = UploadClient(
-        input_path=my_file,
+        input_path=input_path,
         project_code=project_code,
         zone=zone,
         job_type=job_type,
@@ -163,19 +161,23 @@ def simple_upload(  # noqa: C901
         tags=tags,
     )
 
-    # print()
+    # format the local path into object storage path for preupload
+    file_objects = []
+    for file in upload_file_path:
+        # first remove the input path from the file path
+        file_path_sub = file.replace(input_path + '/', '')
+        object_path = os.path.join(target_folder, file_path_sub)
+        file_objects.append(FileObject(object_path, file, None))
 
     # here add the batch of 500 per loop, the pre upload api cannot
     # process very large amount of file at same time. otherwise it will timeout
-    num_of_batchs = math.ceil(len(upload_file_path) / AppConfig.Env.upload_batch_size)
+    num_of_batchs = math.ceil(len(file_objects) / AppConfig.Env.upload_batch_size)
     # here is list of pre upload result. We decided to call pre upload api by batch
-    # the result will store as (UploaderObject, preupload_id_mapping)
     pre_upload_infos = []
-
     for batch in range(0, num_of_batchs):
         start_index = batch * AppConfig.Env.upload_batch_size
         end_index = (batch + 1) * AppConfig.Env.upload_batch_size
-        file_batchs = upload_file_path[start_index:end_index]
+        file_batchs = file_objects[start_index:end_index]
 
         # sending the pre upload request to generate
         # the placeholder in object storage
@@ -212,7 +214,7 @@ def simple_upload(  # noqa: C901
             time.sleep(0.5)
         if source_file:
             upload_client.create_file_lineage(source_file)
-            os.remove(file_batchs[0]) if os.path.isdir(my_file) and job_type == UploadType.AS_FILE else None
+            os.remove(file_batchs[0]) if os.path.isdir(input_path) and job_type == UploadType.AS_FILE else None
 
     num_of_file = len(upload_file_path)
     logger.info(f'Upload Time: {time.time() - upload_start_time:.2f}s for {num_of_file:d} files')
diff --git a/app/services/file_manager/file_upload/models.py b/app/services/file_manager/file_upload/models.py
index e9ddf1be..b2e7e585 100644
--- a/app/services/file_manager/file_upload/models.py
+++ b/app/services/file_manager/file_upload/models.py
@@ -54,7 +54,13 @@ class FileObject:
     uploaded_chunks: List[dict]
 
     def __init__(
-        self, resumable_id: str, job_id: str, item_id: str, object_path: str, local_path: str, uploaded_chunks: List
+        self,
+        object_path: str,
+        local_path: str,
+        uploaded_chunks: List,
+        resumable_id: str = None,
+        job_id: str = None,
+        item_id: str = None,
     ) -> None:
         # object storage info
         self.resumable_id = resumable_id
diff --git a/app/services/file_manager/file_upload/upload_client.py b/app/services/file_manager/file_upload/upload_client.py
index df990d73..ace580f3 100644
--- a/app/services/file_manager/file_upload/upload_client.py
+++ b/app/services/file_manager/file_upload/upload_client.py
@@ -135,7 +135,7 @@ def resume_upload(self, unfinished_file_objects: List[FileObject]) -> List[FileO
         return unfinished_file_objects
 
     # @require_valid_token()
-    def pre_upload(self, local_file_paths: List[str], output_path: str) -> List[FileObject]:
+    def pre_upload(self, file_objects: List[FileObject], output_path: str) -> List[FileObject]:
         """
         Summary:
             The function is to initiate all the multipart upload.
@@ -152,17 +152,18 @@ def pre_upload(self, local_file_paths: List[str], output_path: str) -> List[File
         # print('pre upload')
         headers = {'Authorization': 'Bearer ' + self.user.access_token, 'Session-ID': self.user.session_id}
         url = AppConfig.Connections.url_bff + '/v1/project/{}/files'.format(self.project_code)
-        # the file mapping is a dictionary that present the map from object storage path
-        # with local file path. It will be used in chunk upload api.
-        payload, file_mapping = uf.generate_pre_upload_form(
-            self.project_code,
-            self.operator,
-            local_file_paths,
-            self.input_path,
-            zone=self.zone,
-            job_type=self.job_type,
-            current_folder=self.current_folder_node,
-        )
+
+        file_mapping = {x.object_path: x.local_path for x in file_objects}
+        payload = {
+            'project_code': self.project_code,
+            'operator': self.operator,
+            'job_type': str(self.job_type),
+            'zone': self.zone,
+            'current_folder_node': self.current_folder_node,
+            'data': [
+                {'resumable_filename': x.file_name, 'resumable_relative_path': x.parent_path} for x in file_objects
+            ],
+        }
 
         payload.update({'parent_folder_id': self.parent_folder_id})
         payload.update({'folder_tags': self.tags})

From c9463184866ba969e1ca667b3309ecdf3c93c093 Mon Sep 17 00:00:00 2001
From: zhiren <zzhan@indocresearch.org>
Date: Tue, 11 Apr 2023 10:31:45 -0400
Subject: [PATCH 04/11] fixup with test

---
 app/commands/file.py                          |  5 ----
 .../file_manager/file_upload/file_upload.py   | 14 +++--------
 .../file_manager/file_upload/models.py        |  3 +--
 .../file_manager/file_upload/upload_client.py | 25 ++++++++-----------
 .../file_upload/test_file_upload.py           |  2 +-
 .../file_manager/file_upload/test_model.py    |  6 ++---
 .../file_upload/test_upload_client.py         |  8 +++---
 7 files changed, 23 insertions(+), 40 deletions(-)

diff --git a/app/commands/file.py b/app/commands/file.py
index 61c92527..8e5dea35 100644
--- a/app/commands/file.py
+++ b/app/commands/file.py
@@ -196,11 +196,6 @@ def file_put(**kwargs):  # noqa: C901
             zipping,
         )
 
-        current_folder_node = 'testproject/admin/test11'
-        parent_folder = {'id': 'testproject'}
-        create_folder_flag = True
-        result_file = None
-
         upload_event = {
             'project_code': project_code,
             'file': f.rstrip('/'),  # remove the ending slash
diff --git a/app/services/file_manager/file_upload/file_upload.py b/app/services/file_manager/file_upload/file_upload.py
index 78eb7f78..b5379a0c 100644
--- a/app/services/file_manager/file_upload/file_upload.py
+++ b/app/services/file_manager/file_upload/file_upload.py
@@ -141,17 +141,10 @@ def simple_upload(  # noqa: C901
         if create_folder_flag:
             job_type = UploadType.AS_FOLDER
             input_path = os.path.dirname(input_path)  # update the path as folder
-            target_folder = '/'.join(target_folder.split('/')[:-1]).rstrip('/')
         else:
             target_folder = '/'.join(target_folder.split('/')[:-1]).rstrip('/')
             job_type = UploadType.AS_FILE
 
-    # print('upload_file_path:', upload_file_path)
-    # print('target_folder:', target_folder)
-    # print('input_path:', input_path)
-    # print('job_type:', job_type)
-    # print('zone:', zone)
-
     upload_client = UploadClient(
         input_path=input_path,
         project_code=project_code,
@@ -169,7 +162,7 @@ def simple_upload(  # noqa: C901
         # first remove the input path from the file path
         file_path_sub = file.replace(input_path + '/', '')
         object_path = os.path.join(target_folder, file_path_sub)
-        file_objects.append(FileObject(object_path, file, None))
+        file_objects.append(FileObject(object_path, file))
 
     # here add the batch of 500 per loop, the pre upload api cannot
     # process very large amount of file at same time. otherwise it will timeout
@@ -257,12 +250,11 @@ def resume_upload(
             file_info = all_files.get(x.get('result').get('id'))
             unfinished_items.append(
                 FileObject(
+                    file_info.get('object_path'),
+                    file_info.get('local_path'),
                     file_info.get('resumable_id'),
                     file_info.get('job_id'),
                     file_info.get('item_id'),
-                    file_info.get('object_path'),
-                    file_info.get('local_path'),
-                    [],
                 )
             )
 
diff --git a/app/services/file_manager/file_upload/models.py b/app/services/file_manager/file_upload/models.py
index f96e6ba6..334e0d7a 100644
--- a/app/services/file_manager/file_upload/models.py
+++ b/app/services/file_manager/file_upload/models.py
@@ -62,7 +62,6 @@ def __init__(
         self,
         object_path: str,
         local_path: str,
-        uploaded_chunks: List,
         resumable_id: str = None,
         job_id: str = None,
         item_id: str = None,
@@ -79,7 +78,7 @@ def __init__(
         self.total_size, self.total_chunks = self.generate_meta(local_path)
 
         # resumable info
-        self.uploaded_chunks = uploaded_chunks
+        self.uploaded_chunks = {}
 
     def generate_meta(self, local_path: str) -> Tuple[int, int]:
         """
diff --git a/app/services/file_manager/file_upload/upload_client.py b/app/services/file_manager/file_upload/upload_client.py
index 09238b7c..bc858115 100644
--- a/app/services/file_manager/file_upload/upload_client.py
+++ b/app/services/file_manager/file_upload/upload_client.py
@@ -164,38 +164,35 @@ def pre_upload(self, file_objects: List[FileObject], output_path: str) -> List[F
                 - local_path(str): the local path of file.
                 - chunk_info(dict): the mapping for chunks that already been uploaded.
         """
-        # print('pre upload')
+
         headers = {'Authorization': 'Bearer ' + self.user.access_token, 'Session-ID': self.user.session_id}
         url = AppConfig.Connections.url_bff + '/v1/project/{}/files'.format(self.project_code)
-
-        file_mapping = {x.object_path: x.local_path for x in file_objects}
         payload = {
             'project_code': self.project_code,
             'operator': self.operator,
             'job_type': str(self.job_type),
             'zone': self.zone,
             'current_folder_node': self.current_folder_node,
+            'parent_folder_id': self.parent_folder_id,
+            'folder_tags': self.tags,
             'data': [
                 {'resumable_filename': x.file_name, 'resumable_relative_path': x.parent_path} for x in file_objects
             ],
         }
-
-        payload.update({'parent_folder_id': self.parent_folder_id})
-        payload.update({'folder_tags': self.tags})
-        # print('pre upload payload: ', payload)
-        # raise Exception('pre upload')
         response = resilient_session().post(url, json=payload, headers=headers, timeout=None)
+
         if response.status_code == 200:
             result = response.json().get('result')
+            file_mapping = {x.object_path: x for x in file_objects}
             file_objets = []
             for job in result:
                 object_path = job.get('target_names')[0]
-                resumable_id = job.get('payload').get('resumable_identifier')
-                item_id = job.get('payload').get('item_id')
-                job_id = job.get('job_id')
-                file_objets.append(
-                    FileObject(resumable_id, job_id, item_id, object_path, file_mapping.get(object_path), {})
-                )
+                # get the file object from mapping and update the attribute
+                file_object = file_mapping.get(object_path)
+                file_object.resumable_id = job.get('payload').get('resumable_identifier')
+                file_object.item_id = job.get('payload').get('item_id')
+                file_object.job_id = job.get('job_id')
+                file_objets.append(file_object)
 
             # then output manifest file to the output path
             self.output_manifest(file_objets, output_path)
diff --git a/tests/app/services/file_manager/file_upload/test_file_upload.py b/tests/app/services/file_manager/file_upload/test_file_upload.py
index 7e016b84..3f33298a 100644
--- a/tests/app/services/file_manager/file_upload/test_file_upload.py
+++ b/tests/app/services/file_manager/file_upload/test_file_upload.py
@@ -108,7 +108,7 @@ def test_assemble_path_at_non_existing_folder(mocker):
 
 def test_resume_upload(mocker):
     mocker.patch('app.services.file_manager.file_upload.models.FileObject.generate_meta', return_value=(1, 1))
-    test_obj = FileObject('resumable_id', 'job_id', 'item_id', 'object/path', 'local_path', [])
+    test_obj = FileObject('object/path', 'local_path', 'resumable_id', 'job_id', 'item_id')
 
     manifest_json = {
         'project_code': 'project_code',
diff --git a/tests/app/services/file_manager/file_upload/test_model.py b/tests/app/services/file_manager/file_upload/test_model.py
index da612bf7..7b9924ab 100644
--- a/tests/app/services/file_manager/file_upload/test_model.py
+++ b/tests/app/services/file_manager/file_upload/test_model.py
@@ -9,7 +9,7 @@
 def test_file_upload_model_update_progress_bar(mocker):
     mocker.patch('app.services.file_manager.file_upload.models.getsize', return_value=100)
 
-    file_obj = FileObject('test', 'test', 'test', 'test', 'test', [])
+    file_obj = FileObject('test', 'test', 'test', 'test', 'test')
     file_obj.update_progress(1)
 
     assert file_obj.progress_bar is not None
@@ -19,7 +19,7 @@ def test_file_upload_model_update_progress_bar(mocker):
 def test_file_upload_model_close_progress_bar(mocker):
     mocker.patch('app.services.file_manager.file_upload.models.getsize', return_value=100)
 
-    file_obj = FileObject('test', 'test', 'test', 'test', 'test', [])
+    file_obj = FileObject('test', 'test', 'test', 'test', 'test')
     file_obj.close_progress()
 
     assert file_obj.progress_bar is None
@@ -29,7 +29,7 @@ def test_file_upload_model_generate_meta(mocker):
     AppConfig.Env.chunk_size = 10
     mocker.patch('app.services.file_manager.file_upload.models.getsize', return_value=100)
 
-    file_obj = FileObject('test', 'test', 'test', 'test', 'test', [])
+    file_obj = FileObject('test', 'test', 'test', 'test', 'test')
     total_size, total_chunks = file_obj.generate_meta('test')
 
     assert total_size == 100
diff --git a/tests/app/services/file_manager/file_upload/test_upload_client.py b/tests/app/services/file_manager/file_upload/test_upload_client.py
index e01ce291..cec7523f 100644
--- a/tests/app/services/file_manager/file_upload/test_upload_client.py
+++ b/tests/app/services/file_manager/file_upload/test_upload_client.py
@@ -22,7 +22,7 @@ def test_chunk_upload(httpx_mock, mocker):
     httpx_mock.add_response(method='PUT', url=test_presigned_url, json={'result': ''})
     mocker.patch('app.services.file_manager.file_upload.models.FileObject.generate_meta', return_value=(1, 1))
 
-    test_obj = FileObject('test', 'test', 'test', 'test', 'test', [])
+    test_obj = FileObject('test', 'test', 'test', 'test', 'test')
     res = upload_client.upload_chunk(test_obj, 0, b'1')
 
     assert test_obj.progress_bar.n == 1
@@ -63,7 +63,7 @@ def test_resumable_pre_upload_success(httpx_mock, mocker):
 
     upload_client = UploadClient('test', 'project_code', 'parent_folder_id')
     mocker.patch('app.services.file_manager.file_upload.models.FileObject.generate_meta', return_value=(1, 1))
-    test_obj = FileObject('resumable_id', 'job_id', 'item_id', 'object/path', 'local_path', [])
+    test_obj = FileObject('object/path', 'local_path', 'resumable_id', 'job_id', 'item_id')
 
     url = AppConfig.Connections.url_bff + f'/v1/project/{upload_client.project_code}/files/resumable'
     httpx_mock.add_response(
@@ -85,7 +85,7 @@ def test_resumable_pre_upload_failed_with_404(httpx_mock, mocker):
 
     upload_client = UploadClient('test', 'project_code', 'parent_folder_id')
     mocker.patch('app.services.file_manager.file_upload.models.FileObject.generate_meta', return_value=(1, 1))
-    test_obj = FileObject('resumable_id', 'job_id', 'item_id', 'object/path', 'local_path', [])
+    test_obj = FileObject('object/path', 'local_path', 'resumable_id', 'job_id', 'item_id')
 
     url = AppConfig.Connections.url_bff + f'/v1/project/{upload_client.project_code}/files/resumable'
     httpx_mock.add_response(
@@ -107,7 +107,7 @@ def test_output_manifest_success(mocker):
     upload_client = UploadClient('test', 'project_code', 'parent_folder_id')
     json_dump_mocker = mocker.patch('json.dump', return_value=None)
     mocker.patch('app.services.file_manager.file_upload.models.FileObject.generate_meta', return_value=(1, 1))
-    test_obj = FileObject('resumable_id', 'job_id', 'item_id', 'object/path', 'local_path', [])
+    test_obj = FileObject('object/path', 'local_path', 'resumable_id', 'job_id', 'item_id')
 
     res = upload_client.output_manifest([test_obj], 'test')
 

From 8f918d44149f349ca1942f18aadcd0c3ffedcd72 Mon Sep 17 00:00:00 2001
From: zhiren <zzhan@indocresearch.org>
Date: Tue, 11 Apr 2023 11:01:58 -0400
Subject: [PATCH 05/11] test

---
 app/services/file_manager/file_upload/upload_client.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/app/services/file_manager/file_upload/upload_client.py b/app/services/file_manager/file_upload/upload_client.py
index bc858115..82a3ee5c 100644
--- a/app/services/file_manager/file_upload/upload_client.py
+++ b/app/services/file_manager/file_upload/upload_client.py
@@ -179,6 +179,9 @@ def pre_upload(self, file_objects: List[FileObject], output_path: str) -> List[F
                 {'resumable_filename': x.file_name, 'resumable_relative_path': x.parent_path} for x in file_objects
             ],
         }
+
+        # print('pre upload payload', payload)
+        raise Exception('stop here')
         response = resilient_session().post(url, json=payload, headers=headers, timeout=None)
 
         if response.status_code == 200:

From e849cff8cbdc08078160d2b27a760545da9cfec4 Mon Sep 17 00:00:00 2001
From: zhiren <zzhan@indocresearch.org>
Date: Tue, 11 Apr 2023 11:48:21 -0400
Subject: [PATCH 06/11] fixup the path formatting issue

---
 app/commands/file.py                          | 11 +++++----
 .../file_manager/file_upload/file_upload.py   | 24 +++++++++++--------
 .../file_manager/file_upload/upload_client.py |  2 --
 3 files changed, 20 insertions(+), 17 deletions(-)

diff --git a/app/commands/file.py b/app/commands/file.py
index 8e5dea35..8dd95926 100644
--- a/app/commands/file.py
+++ b/app/commands/file.py
@@ -173,14 +173,15 @@ def file_put(**kwargs):  # noqa: C901
 
     # for the path formating there will be following cases:
     # - file:
-    #   1. the project path exist, then will be AS_FILE. nothing will be changed
-    #   2. the project path not exist, then will be AS_FOLDER. the current folder node will
-    #      be the parent folder node + parent folder id. (like one level up).
+    #   1. the project path exist, then will be AS_FILE. nothing will be changed.
+    #      current_folder_node will be empty string.
+    #   2. the project path not exist, then will be AS_FOLDER. the current_folder_node will
+    #      be the parent folder node + the shortest non-exist folder. (like one level down).
     # - folder:
     #   1. the project path exist, then will be AS_FOLDER. the current folder node will be
     #      the one that user input.
     #   2. the project path not exist, then will be AS_FOLDER. the current folder node will
-    #      be the parent folder node + parent folder id. (like one level up).
+    #      be the parent folder node + the shortest non-exist folder. (like one level down).
 
     # Unique Paths
     paths = set(paths)
@@ -198,6 +199,7 @@ def file_put(**kwargs):  # noqa: C901
 
         upload_event = {
             'project_code': project_code,
+            'target_folder': target_folder,
             'file': f.rstrip('/'),  # remove the ending slash
             'tags': tag if tag else [],
             'zone': zone,
@@ -208,7 +210,6 @@ def file_put(**kwargs):  # noqa: C901
             'compress_zip': zipping,
             'attribute': attribute,
         }
-        # print(upload_event)
         if pipeline:
             upload_event['process_pipeline'] = pipeline
         if source_file:
diff --git a/app/services/file_manager/file_upload/file_upload.py b/app/services/file_manager/file_upload/file_upload.py
index b5379a0c..f13e6d25 100644
--- a/app/services/file_manager/file_upload/file_upload.py
+++ b/app/services/file_manager/file_upload/file_upload.py
@@ -69,13 +69,18 @@ def assemble_path(
 
     current_file_path = target_folder + '/' + f.rstrip('/').split('/')[-1]
     result_file = current_file_path
+    if zipping:
+        result_file = result_file + '.zip'
 
     # set name folder as first parent folder
-    name_folder = current_file_path.split('/')[0]
+    name_folder = target_folder.split('/')[0]
     parent_folder = search_item(project_code, zone, name_folder, 'name_folder')
     parent_folder = parent_folder.get('result')
-    create_folder_flag = False
 
+    # if f input is a file then current_folder_node is target_folder
+    # otherwise it is target_folder + f input name
+    current_folder_node = target_folder if os.path.isfile(f) else current_file_path
+    create_folder_flag = False
     if len(current_file_path.split('/')) > 2:
         sub_path = target_folder.split('/')
         for index in range(len(sub_path) - 1):
@@ -85,7 +90,7 @@ def assemble_path(
             # find the longest existing folder as parent folder
             # if user input a path that need to create some folders
             if not res.get('result'):
-                current_file_path = folder_path
+                current_folder_node = folder_path
                 click.confirm(customized_error_msg(ECustomizedError.CREATE_FOLDER_IF_NOT_EXIST), abort=True)
                 create_folder_flag = True
                 break
@@ -97,9 +102,7 @@ def assemble_path(
     if not parent_folder:
         SrvErrorHandler.customized_handle(ECustomizedError.PERMISSION_DENIED, True)
 
-    if zipping:
-        result_file = result_file + '.zip'
-    return current_file_path, parent_folder, create_folder_flag, result_file
+    return current_folder_node, parent_folder, create_folder_flag, result_file
 
 
 def simple_upload(  # noqa: C901
@@ -114,7 +117,7 @@ def simple_upload(  # noqa: C901
     zone = upload_event.get('zone')
     # process_pipeline = upload_event.get('process_pipeline', None)
     # upload_message = upload_event.get('upload_message')
-    target_folder = upload_event.get('current_folder_node', '')
+    current_folder_node = upload_event.get('current_folder_node', '')
     parent_folder_id = upload_event.get('parent_folder_id', '')
     create_folder_flag = upload_event.get('create_folder_flag', False)
     compress_zip = upload_event.get('compress_zip', False)
@@ -130,7 +133,7 @@ def simple_upload(  # noqa: C901
         job_type = UploadType.AS_FILE if compress_zip else UploadType.AS_FOLDER
         if job_type == UploadType.AS_FILE:
             upload_file_path = [input_path.rstrip('/').lstrip() + '.zip']
-            target_folder = '/'.join(target_folder.split('/')[:-1]).rstrip('/')
+            # target_folder = '/'.join(target_folder.split('/')[:-1]).rstrip('/')
             compress_folder_to_zip(input_path)
         else:
             logger.warning('Current version does not support folder tagging, ' 'any selected tags will be ignored')
@@ -142,7 +145,7 @@ def simple_upload(  # noqa: C901
             job_type = UploadType.AS_FOLDER
             input_path = os.path.dirname(input_path)  # update the path as folder
         else:
-            target_folder = '/'.join(target_folder.split('/')[:-1]).rstrip('/')
+            # target_folder = '/'.join(target_folder.split('/')[:-1]).rstrip('/')
             job_type = UploadType.AS_FILE
 
     upload_client = UploadClient(
@@ -150,7 +153,7 @@ def simple_upload(  # noqa: C901
         project_code=project_code,
         zone=zone,
         job_type=job_type,
-        current_folder_node=target_folder,
+        current_folder_node=current_folder_node,
         parent_folder_id=parent_folder_id,
         regular_file=regular_file,
         tags=tags,
@@ -158,6 +161,7 @@ def simple_upload(  # noqa: C901
 
     # format the local path into object storage path for preupload
     file_objects = []
+    target_folder = upload_event.get('target_folder', '')
     for file in upload_file_path:
         # first remove the input path from the file path
         file_path_sub = file.replace(input_path + '/', '')
diff --git a/app/services/file_manager/file_upload/upload_client.py b/app/services/file_manager/file_upload/upload_client.py
index 82a3ee5c..d74b4948 100644
--- a/app/services/file_manager/file_upload/upload_client.py
+++ b/app/services/file_manager/file_upload/upload_client.py
@@ -180,8 +180,6 @@ def pre_upload(self, file_objects: List[FileObject], output_path: str) -> List[F
             ],
         }
 
-        # print('pre upload payload', payload)
-        raise Exception('stop here')
         response = resilient_session().post(url, json=payload, headers=headers, timeout=None)
 
         if response.status_code == 200:

From 44f7cce12d6fd436024d99355c7142f4fb3e162e Mon Sep 17 00:00:00 2001
From: zhiren <zzhan@indocresearch.org>
Date: Tue, 11 Apr 2023 11:52:30 -0400
Subject: [PATCH 07/11] add the combine check to make sure the thread stop
 after all finished

---
 app/services/file_manager/file_upload/file_upload.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/app/services/file_manager/file_upload/file_upload.py b/app/services/file_manager/file_upload/file_upload.py
index f13e6d25..347edef7 100644
--- a/app/services/file_manager/file_upload/file_upload.py
+++ b/app/services/file_manager/file_upload/file_upload.py
@@ -126,14 +126,12 @@ def simple_upload(  # noqa: C901
     attribute = upload_event.get('attribute')
 
     mhandler.SrvOutPutHandler.start_uploading(input_path)
-    # TODO: PILOT-2392 simplify the logic under
     # if the input request zip folder then process the path as single file
     # otherwise read throught the folder to get path underneath
     if os.path.isdir(input_path):
         job_type = UploadType.AS_FILE if compress_zip else UploadType.AS_FOLDER
         if job_type == UploadType.AS_FILE:
             upload_file_path = [input_path.rstrip('/').lstrip() + '.zip']
-            # target_folder = '/'.join(target_folder.split('/')[:-1]).rstrip('/')
             compress_folder_to_zip(input_path)
         else:
             logger.warning('Current version does not support folder tagging, ' 'any selected tags will be ignored')
@@ -145,7 +143,6 @@ def simple_upload(  # noqa: C901
             job_type = UploadType.AS_FOLDER
             input_path = os.path.dirname(input_path)  # update the path as folder
         else:
-            # target_folder = '/'.join(target_folder.split('/')[:-1]).rstrip('/')
             job_type = UploadType.AS_FILE
 
     upload_client = UploadClient(
@@ -190,14 +187,21 @@ def simple_upload(  # noqa: C901
 
     pool = ThreadPool(num_of_thread + 1)
     pool.apply_async(upload_client.upload_token_refresh)
+    on_succeed_res = []
     for file_object in pre_upload_infos:
         chunk_res = upload_client.stream_upload(file_object, pool)
         # NOTE: if there is some racing error make the combine chunks
         # out of thread pool.
-        pool.apply_async(
+        res = pool.apply_async(
             upload_client.on_succeed,
             args=(file_object, tags, chunk_res),
         )
+        on_succeed_res.append(res)
+
+    # wait for all the chunk combination to finish
+    for res in on_succeed_res:
+        while res.get() is None:
+            time.sleep(0.5)
     upload_client.set_finish_upload()
 
     pool.close()

From 474866dfb991c96fc169faa5f80551748ca30131 Mon Sep 17 00:00:00 2001
From: zhiren <zzhan@indocresearch.org>
Date: Tue, 11 Apr 2023 11:54:52 -0400
Subject: [PATCH 08/11] remove the chunk combine check

---
 app/services/file_manager/file_upload/file_upload.py | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/app/services/file_manager/file_upload/file_upload.py b/app/services/file_manager/file_upload/file_upload.py
index 347edef7..ca1bfeac 100644
--- a/app/services/file_manager/file_upload/file_upload.py
+++ b/app/services/file_manager/file_upload/file_upload.py
@@ -197,11 +197,6 @@ def simple_upload(  # noqa: C901
             args=(file_object, tags, chunk_res),
         )
         on_succeed_res.append(res)
-
-    # wait for all the chunk combination to finish
-    for res in on_succeed_res:
-        while res.get() is None:
-            time.sleep(0.5)
     upload_client.set_finish_upload()
 
     pool.close()

From dd6b4798e09efe0f8a9ef6684d46ebde2b4c333f Mon Sep 17 00:00:00 2001
From: zhiren <zzhan@indocresearch.org>
Date: Wed, 12 Apr 2023 10:41:42 -0400
Subject: [PATCH 09/11] fixup the test case

---
 .../services/file_manager/file_upload/test_upload_client.py   | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/app/services/file_manager/file_upload/test_upload_client.py b/tests/app/services/file_manager/file_upload/test_upload_client.py
index 3f181e4b..03ce331a 100644
--- a/tests/app/services/file_manager/file_upload/test_upload_client.py
+++ b/tests/app/services/file_manager/file_upload/test_upload_client.py
@@ -38,7 +38,7 @@ def test_check_status_success(httpx_mock, mocker):
         status_code=200,
     )
 
-    test_obj = FileObject('test', 'test', 'test', 'test', 'test', [])
+    test_obj = FileObject('test', 'test', 'test', 'test', 'test')
     result = upload_client.check_status(test_obj)
 
     assert result is True
@@ -57,7 +57,7 @@ def test_check_status_fail(httpx_mock, mocker):
         status_code=200,
     )
 
-    test_obj = FileObject('test', 'test', 'test', 'test', 'test', [])
+    test_obj = FileObject('test', 'test', 'test', 'test', 'test')
     result = upload_client.check_status(test_obj)
 
     assert result is False

From 5f6ca44a6dad075addfc7e9847087d9e141061ee Mon Sep 17 00:00:00 2001
From: zhiren <zzhan@indocresearch.org>
Date: Wed, 12 Apr 2023 10:43:28 -0400
Subject: [PATCH 10/11] fixup the test case

---
 app/services/crypto/crypto.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/app/services/crypto/crypto.py b/app/services/crypto/crypto.py
index 81fb6792..026d2d87 100644
--- a/app/services/crypto/crypto.py
+++ b/app/services/crypto/crypto.py
@@ -76,5 +76,4 @@ def decryption(encrypted_message, secret, interactive=True):
             else:
                 raise ex
     else:
-        pass
-        # ehandler.SrvErrorHandler.customized_handle(ehandler.ECustomizedError.LOGIN_SESSION_INVALID, True)
+        ehandler.SrvErrorHandler.customized_handle(ehandler.ECustomizedError.LOGIN_SESSION_INVALID, True)

From 9d13ac1406ecff1d9f99b650768ba236f42e49a2 Mon Sep 17 00:00:00 2001
From: zhiren <zzhan@indocresearch.org>
Date: Thu, 13 Apr 2023 10:50:13 -0400
Subject: [PATCH 11/11] manually merged

---
 app/services/file_manager/file_upload/file_upload.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/app/services/file_manager/file_upload/file_upload.py b/app/services/file_manager/file_upload/file_upload.py
index fd640dce..aded3707 100644
--- a/app/services/file_manager/file_upload/file_upload.py
+++ b/app/services/file_manager/file_upload/file_upload.py
@@ -131,13 +131,12 @@ def simple_upload(  # noqa: C901
     if os.path.isdir(input_path):
         job_type = UploadType.AS_FILE if compress_zip else UploadType.AS_FOLDER
         if job_type == UploadType.AS_FILE:
-            upload_file_path = [my_file.rstrip('/').lstrip() + '.zip']
-            target_folder = '/'.join(target_folder.split('/')[:-1]).rstrip('/')
-            compress_folder_to_zip(my_file)
+            upload_file_path = [input_path.rstrip('/').lstrip() + '.zip']
+            compress_folder_to_zip(input_path)
         elif tags or attribute:
             SrvErrorHandler.customized_handle(ECustomizedError.UNSUPPORT_TAG_MANIFEST, True)
         else:
-            upload_file_path = get_file_in_folder(my_file)
+            upload_file_path = get_file_in_folder(input_path)
     else:
         upload_file_path = [input_path]
 
@@ -189,7 +188,7 @@ def simple_upload(  # noqa: C901
 
     pool = ThreadPool(num_of_thread + 1)
     pool.apply_async(upload_client.upload_token_refresh)
-    on_succeed_res = []
+    on_success_res = []
     for file_object in pre_upload_infos:
         chunk_res = upload_client.stream_upload(file_object, pool)
         # NOTE: if there is some racing error make the combine chunks