Pilot 4102: update file upload command to read tag/attribute file

app/commands/file.py

@@ -12,7 +12,6 @@
 import app.services.output_manager.help_page as file_help
 import app.services.output_manager.message_handler as message_handler
 from app.configs.app_config import AppConfig
-from app.configs.user_config import UserConfig
 from app.services.file_manager.file_download.download_client import SrvFileDownload
 from app.services.file_manager.file_list import SrvFileList
 from app.services.file_manager.file_manifests import SrvFileManifests
@@ -40,15 +39,21 @@ def cli():
 
 
 @click.command(name='upload')
-@click.argument('paths', type=click.Path(exists=True), nargs=-1)
-@click.option('-p', '--project-path', required=True, help=file_help.file_help_page(file_help.FileHELP.FILE_UPLOAD_P))
+@click.argument('files', type=click.Path(exists=True), nargs=-1)
+@click.option(
+    '-p',
+    '--project-path',
+    required=True,
+    type=click.Path(),
+    help=file_help.file_help_page(file_help.FileHELP.FILE_UPLOAD_P),
+)
 @click.option(
     '-a',
     '--attribute',
     default=None,
     required=False,
     help=file_help.file_help_page(file_help.FileHELP.FILE_UPLOAD_A),
-    # type=click.Path(exists=True),
+    type=click.File('rb'),
     show_default=True,
 )
 @click.option(
@@ -58,6 +63,7 @@ def cli():
     required=False,
     multiple=True,
     help=file_help.file_help_page(file_help.FileHELP.FILE_UPLOAD_T),
+    type=click.File('rb'),
     show_default=True,
 )
 @click.option(
@@ -112,21 +118,25 @@ def cli():
 def file_put(**kwargs):  # noqa: C901
     """"""
 
-    paths = kwargs.get('paths')
+    files = kwargs.get('files')
     project_path = kwargs.get('project_path')
-    tag = kwargs.get('tag')
+    tag_files = kwargs.get('tag')
     zone = kwargs.get('zone')
     upload_message = kwargs.get('upload_message')
     source_file = kwargs.get('source_file')
     zipping = kwargs.get('zip')
-    attribute = kwargs.get('attribute')
+    attribute_file = kwargs.get('attribute')
     thread = kwargs.get('thread')
     output_path = kwargs.get('output_path')
 
-    user = UserConfig()
+    # load tag json file to list, and attribute file to dict
+    tag = []
+    for t_f in tag_files:
+        tag.extend(json.load(t_f))
+    attribute = json.load(attribute_file) if attribute_file else None
+
     # Check zone and upload-message
     zone = get_zone(zone) if zone else AppConfig.Env.green_zone.lower()
-
     toc = customized_error_msg(ECustomizedError.TOU_CONTENT).replace(' ', '...')
     try:
         if zone.lower() == AppConfig.Env.core_zone.lower() and click.confirm(fit_terminal_width(toc), abort=True):
@@ -136,7 +146,7 @@ def file_put(**kwargs):  # noqa: C901
         exit(1)
 
     # check if user input at least one file/folder
-    if len(paths) == 0:
+    if len(files) == 0:
         SrvErrorHandler.customized_handle(ECustomizedError.INVALID_PATHS, True)
 
     # check if the manifest file exists
@@ -157,7 +167,6 @@ def file_put(**kwargs):  # noqa: C901
         'upload_message': upload_message,
         'source': source_file,
         'project_code': project_code,
-        'token': user.access_token,
         'attribute': attribute,
         'tag': tag,
     }
@@ -181,10 +190,10 @@ def file_put(**kwargs):  # noqa: C901
     #      be the parent folder node + the shortest non-exist folder. (like one level down).
 
     # Unique Paths
-    paths = set(paths)
+    files = set(files)
     # the loop will read all input path(folder or files)
     # and process them one by one
-    for f in paths:
+    for f in files:
         # so this function will always return the furthest folder node as current_folder_node+parent_folder_id
         current_folder_node, parent_folder, create_folder_flag, result_file = assemble_path(
             f,
@@ -275,10 +284,9 @@ def validate_upload_event(event):
     upload_message = event.get('upload_message')
     source = event.get('source')
     project_code = event.get('project_code')
-    token = event.get('token')
     attribute = event.get('attribute')
     tag = event.get('tag')
-    validator = UploadEventValidator(project_code, zone, upload_message, source, token, attribute, tag)
+    validator = UploadEventValidator(project_code, zone, upload_message, source, attribute, tag)
     converted_content = validator.validate_upload_event()
     return converted_content
 
@@ -486,7 +494,6 @@ def file_metadata_download(**kwargs):
     attribute_folder = kwargs.get('attribute').rstrip('/')
     tag_folder = kwargs.get('tag').rstrip('/')
 
-    # user = UserConfig()
     # Check zone and upload-message
     zone = get_zone(zone) if zone else AppConfig.Env.green_zone.lower()
     file_meta_client = FileMetaClient(zone, file_path, general_folder, attribute_folder, tag_folder)

app/services/file_manager/file_upload/upload_validator.py

@@ -2,7 +2,9 @@
 #
 # Contact Indoc Systems for any questions regarding the use of this source code.
 
-import os
+from typing import Any
+from typing import Dict
+from typing import List
 
 from app.configs.app_config import AppConfig
 from app.services.file_manager.file_manifests import SrvFileManifests
@@ -13,12 +15,13 @@
 
 
 class UploadEventValidator:
-    def __init__(self, project_code, zone, upload_message, source, token, attribute, tag):
+    def __init__(
+        self, project_code: str, zone: str, upload_message: str, source: str, attribute: Dict[str, Any], tag: List[str]
+    ):
         self.project_code = project_code
         self.zone = zone
         self.upload_message = upload_message
         self.source = source
-        self.token = token
         self.attribute = attribute
         self.tag = tag
 
@@ -37,11 +40,8 @@ def validate_zone(self):
 
     def validate_attribute(self):
         srv_manifest = SrvFileManifests()
-        if not os.path.isfile(self.attribute):
-            raise Exception('Attribute not exist in the given path')
         try:
-            attribute = srv_manifest.read_manifest_template(self.attribute)
-            attribute = srv_manifest.convert_import(attribute, self.project_code)
+            attribute = srv_manifest.convert_import(self.attribute, self.project_code)
             srv_manifest.validate_manifest(attribute)
             return attribute
         except Exception:

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "app"
-version = "2.9.0"
+version = "2.9.1"
 description = "This service is designed to support pilot platform"
 authors = ["Indoc Systems"]
 

tests/app/commands/test_file.py

@@ -2,6 +2,7 @@
 #
 # Contact Indoc Systems for any questions regarding the use of this source code.
 
+import json
 from os import makedirs
 from os.path import dirname
 
@@ -40,9 +41,11 @@ def test_file_upload_command_success_with_attribute(mocker, cli_runner):
     with runner.isolated_filesystem():
         with open('test.txt', 'w') as f:
             f.write('test.txt')
+        with open('template.json', 'w') as f:
+            json.dump({'template': {'attr1': 'value'}}, f)
 
         result = cli_runner.invoke(
-            file_put, ['--project-path', 'test', '--thread', 1, '--attribute', 'test.json', 'test.txt']
+            file_put, ['--project-path', 'test', '--thread', 1, '--attribute', 'template.json', 'test.txt']
         )
     assert result.exit_code == 0
     simple_upload_mock.assert_called_once()