From c5f8a845d467c4480cb04322e5e20788618fcd73 Mon Sep 17 00:00:00 2001 From: zhiren Date: Mon, 27 Nov 2023 14:49:17 -0500 Subject: [PATCH 1/4] add new command metadata to download file metadata/tags/attributes --- app/commands/file.py | 66 ++++++++++++++++++++++++ app/resources/custom_help.py | 5 ++ app/services/output_manager/help_page.py | 6 +++ 3 files changed, 77 insertions(+) diff --git a/app/commands/file.py b/app/commands/file.py index c3bacfaf..f25add83 100644 --- a/app/commands/file.py +++ b/app/commands/file.py @@ -435,3 +435,69 @@ def file_download(**kwargs): for item in item_res: srv_download = SrvFileDownload(zone, interactive) srv_download.simple_download_file(output_path, [item]) + + +@click.command(name='metadata') +@click.argument('file_path', type=click.STRING, nargs=-1) +@click.option( + '-z', + '--zone', + default=AppConfig.Env.green_zone, + required=True, + help=file_help.file_help_page(file_help.FileHELP.FILE_META_Z), + show_default=False, +) +@click.option( + '-g', + '--general', + default=None, + required=True, + help=file_help.file_help_page(file_help.FileHELP.FILE_META_G), + show_default=True, +) +@click.option( + '-a', + '--attribute', + default=None, + required=True, + help=file_help.file_help_page(file_help.FileHELP.FILE_META_A), + show_default=True, +) +@click.option( + '-t', + '--tag', + default=None, + required=True, + help=file_help.file_help_page(file_help.FileHELP.FILE_META_T), + show_default=True, +) +# @require_valid_token() +@doc(file_help.file_help_page(file_help.FileHELP.FILE_META)) +def file_metadata_download(**kwargs): + ''' + Summary: + Download metadata of a file including general, attribute and tag. + ''' + + # file_path = kwargs.get('file_path') + zone = kwargs.get('zone') + general_location = kwargs.get('general') + attribute_location = kwargs.get('attribute') + tag_location = kwargs.get('tag') + + # user = UserConfig() + # Check zone and upload-message + zone = get_zone(zone) if zone else AppConfig.Env.green_zone.lower() + + # check if the manifest file exists + try: + _ = {os.path.exists(x) for x in [general_location, attribute_location, tag_location]} + # print(files_exist) + + # if os.path.exists(output_path): + # click.confirm( + # customized_error_msg(ECustomizedError.MANIFEST_OF_FOLDER_FILE_EXIST) % (output_path), abort=True + # ) + except Abort: + message_handler.SrvOutPutHandler.cancel_upload() + exit(1) diff --git a/app/resources/custom_help.py b/app/resources/custom_help.py index 65b6aec3..a3d3122f 100644 --- a/app/resources/custom_help.py +++ b/app/resources/custom_help.py @@ -54,6 +54,11 @@ class HelpPage: "The processed pipeline of your processed files. [only used with '--source' option]" ), 'FILE_UPLOAD_ZIP': 'Upload folder as a compressed zip file.', + 'FILE_META': 'Download metadata file of a given file in target zone.', + 'FILE_META_Z': 'Target Zone (i.e., core/greenroom)', + 'FILE_META_G': 'The location of general metadata file', + 'FILE_META_A': 'The location of attribute metadata file', + 'FILE_META_T': 'The location of tag metadata file', }, 'config': { 'SET_CONFIG': 'Chose config file and set for cli.', diff --git a/app/services/output_manager/help_page.py b/app/services/output_manager/help_page.py index d48d0e2b..18b0702e 100644 --- a/app/services/output_manager/help_page.py +++ b/app/services/output_manager/help_page.py @@ -73,6 +73,12 @@ class FileHELP(enum.Enum): FILE_UPLOAD_PIPELINE = 'FILE_UPLOAD_PIPELINE' FILE_UPLOAD_ZIP = 'FILE_UPLOAD_ZIP' + FILE_META = 'FILE_META' + FILE_META_Z = 'FILE_META_Z' + FILE_META_G = 'FILE_META_G' + FILE_META_A = 'FILE_META_A' + FILE_META_T = 'FILE_META_T' + def file_help_page(FileHELP: FileHELP): helps = help_msg.get('file', 'default file help') From 46aeeb10f81c9948e7201a56da145c630d33b38f Mon Sep 17 00:00:00 2001 From: zhiren Date: Wed, 29 Nov 2023 16:08:52 -0500 Subject: [PATCH 2/4] update the upload command to read tag/attribute file when upload --- app/commands/file.py | 38 +++++++++++-------- .../file_upload/upload_validator.py | 14 +++---- 2 files changed, 30 insertions(+), 22 deletions(-) diff --git a/app/commands/file.py b/app/commands/file.py index f25add83..63bb1181 100644 --- a/app/commands/file.py +++ b/app/commands/file.py @@ -12,7 +12,6 @@ import app.services.output_manager.help_page as file_help import app.services.output_manager.message_handler as message_handler from app.configs.app_config import AppConfig -from app.configs.user_config import UserConfig from app.services.file_manager.file_download.download_client import SrvFileDownload from app.services.file_manager.file_list import SrvFileList from app.services.file_manager.file_manifests import SrvFileManifests @@ -39,15 +38,21 @@ def cli(): @click.command(name='upload') -@click.argument('paths', type=click.Path(exists=True), nargs=-1) -@click.option('-p', '--project-path', required=True, help=file_help.file_help_page(file_help.FileHELP.FILE_UPLOAD_P)) +@click.argument('files', type=click.Path(exists=True), nargs=-1) +@click.option( + '-p', + '--project-path', + required=True, + type=click.Path(), + help=file_help.file_help_page(file_help.FileHELP.FILE_UPLOAD_P), +) @click.option( '-a', '--attribute', default=None, required=False, help=file_help.file_help_page(file_help.FileHELP.FILE_UPLOAD_A), - # type=click.Path(exists=True), + type=click.File('rb'), show_default=True, ) @click.option( @@ -57,6 +62,7 @@ def cli(): required=False, multiple=True, help=file_help.file_help_page(file_help.FileHELP.FILE_UPLOAD_T), + type=click.File('rb'), show_default=True, ) @click.option( @@ -111,21 +117,25 @@ def cli(): def file_put(**kwargs): # noqa: C901 """""" - paths = kwargs.get('paths') + files = kwargs.get('files') project_path = kwargs.get('project_path') - tag = kwargs.get('tag') + tag_files = kwargs.get('tag') zone = kwargs.get('zone') upload_message = kwargs.get('upload_message') source_file = kwargs.get('source_file') zipping = kwargs.get('zip') - attribute = kwargs.get('attribute') + attribute_file = kwargs.get('attribute') thread = kwargs.get('thread') output_path = kwargs.get('output_path') - user = UserConfig() + # load tag json file to list, and attribute file to dict + tag = [] + for t_f in tag_files: + tag.extend(json.load(t_f)) + attribute = json.load(attribute_file) if attribute_file else None + # Check zone and upload-message zone = get_zone(zone) if zone else AppConfig.Env.green_zone.lower() - toc = customized_error_msg(ECustomizedError.TOU_CONTENT).replace(' ', '...') try: if zone.lower() == AppConfig.Env.core_zone.lower() and click.confirm(fit_terminal_width(toc), abort=True): @@ -135,7 +145,7 @@ def file_put(**kwargs): # noqa: C901 exit(1) # check if user input at least one file/folder - if len(paths) == 0: + if len(files) == 0: SrvErrorHandler.customized_handle(ECustomizedError.INVALID_PATHS, True) # check if the manifest file exists @@ -156,7 +166,6 @@ def file_put(**kwargs): # noqa: C901 'upload_message': upload_message, 'source': source_file, 'project_code': project_code, - 'token': user.access_token, 'attribute': attribute, 'tag': tag, } @@ -180,10 +189,10 @@ def file_put(**kwargs): # noqa: C901 # be the parent folder node + the shortest non-exist folder. (like one level down). # Unique Paths - paths = set(paths) + files = set(files) # the loop will read all input path(folder or files) # and process them one by one - for f in paths: + for f in files: # so this function will always return the furthest folder node as current_folder_node+parent_folder_id current_folder_node, parent_folder, create_folder_flag, result_file = assemble_path( f, @@ -274,10 +283,9 @@ def validate_upload_event(event): upload_message = event.get('upload_message') source = event.get('source') project_code = event.get('project_code') - token = event.get('token') attribute = event.get('attribute') tag = event.get('tag') - validator = UploadEventValidator(project_code, zone, upload_message, source, token, attribute, tag) + validator = UploadEventValidator(project_code, zone, upload_message, source, attribute, tag) converted_content = validator.validate_upload_event() return converted_content diff --git a/app/services/file_manager/file_upload/upload_validator.py b/app/services/file_manager/file_upload/upload_validator.py index 3e48dcde..51500b40 100644 --- a/app/services/file_manager/file_upload/upload_validator.py +++ b/app/services/file_manager/file_upload/upload_validator.py @@ -2,7 +2,9 @@ # # Contact Indoc Systems for any questions regarding the use of this source code. -import os +from typing import Any +from typing import Dict +from typing import List from app.configs.app_config import AppConfig from app.services.file_manager.file_manifests import SrvFileManifests @@ -13,12 +15,13 @@ class UploadEventValidator: - def __init__(self, project_code, zone, upload_message, source, token, attribute, tag): + def __init__( + self, project_code: str, zone: str, upload_message: str, source: str, attribute: Dict[str, Any], tag: List[str] + ): self.project_code = project_code self.zone = zone self.upload_message = upload_message self.source = source - self.token = token self.attribute = attribute self.tag = tag @@ -37,11 +40,8 @@ def validate_zone(self): def validate_attribute(self): srv_manifest = SrvFileManifests() - if not os.path.isfile(self.attribute): - raise Exception('Attribute not exist in the given path') try: - attribute = srv_manifest.read_manifest_template(self.attribute) - attribute = srv_manifest.convert_import(attribute, self.project_code) + attribute = srv_manifest.convert_import(self.attribute, self.project_code) srv_manifest.validate_manifest(attribute) return attribute except Exception: From 9446e45c5d18a40e4d63f1c2d19fce95153bfe14 Mon Sep 17 00:00:00 2001 From: zhiren Date: Wed, 29 Nov 2023 17:04:06 -0500 Subject: [PATCH 3/4] fixup test cases --- tests/app/commands/test_file.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/app/commands/test_file.py b/tests/app/commands/test_file.py index 0e01a49b..4200e9fe 100644 --- a/tests/app/commands/test_file.py +++ b/tests/app/commands/test_file.py @@ -2,6 +2,8 @@ # # Contact Indoc Systems for any questions regarding the use of this source code. +import json + import click import questionary @@ -35,9 +37,11 @@ def test_file_upload_command_success_with_attribute(mocker, cli_runner): with runner.isolated_filesystem(): with open('test.txt', 'w') as f: f.write('test.txt') + with open('template.json', 'w') as f: + json.dump({'template': {'attr1': 'value'}}, f) result = cli_runner.invoke( - file_put, ['--project-path', 'test', '--thread', 1, '--attribute', 'test.json', 'test.txt'] + file_put, ['--project-path', 'test', '--thread', 1, '--attribute', 'template.json', 'test.txt'] ) assert result.exit_code == 0 simple_upload_mock.assert_called_once() From 8fdd466dae6747d7fd6c825f2dcc17c9b3414d72 Mon Sep 17 00:00:00 2001 From: zhiren Date: Tue, 5 Dec 2023 16:23:03 -0500 Subject: [PATCH 4/4] bumpup to next version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index ce7a52f1..08678608 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "app" -version = "2.9.0" +version = "2.9.1" description = "This service is designed to support pilot platform" authors = ["Indoc Systems"]