From c964438e0b5e14bab9a257fb040b120053525673 Mon Sep 17 00:00:00 2001 From: Vadym Moshynskyi Date: Mon, 10 Jul 2023 18:13:08 +0200 Subject: [PATCH 1/2] PILOT-3305: Remove kg_resource command functionality --- app/commands/entry_point.py | 14 -- app/commands/kg_resource.py | 23 --- app/resources/custom_help.py | 4 - app/services/kg_manager/kg_resource.py | 89 --------- app/services/output_manager/error_handler.py | 5 - app/services/output_manager/help_page.py | 12 +- .../services/kg_manager/test_kg_manager.py | 172 ------------------ 7 files changed, 1 insertion(+), 318 deletions(-) delete mode 100644 app/commands/kg_resource.py delete mode 100644 app/services/kg_manager/kg_resource.py delete mode 100644 tests/app/services/kg_manager/test_kg_manager.py diff --git a/app/commands/entry_point.py b/app/commands/entry_point.py index a16da236..094d6d81 100644 --- a/app/commands/entry_point.py +++ b/app/commands/entry_point.py @@ -2,8 +2,6 @@ # # Contact Indoc Research for any questions regarding the use of this source code. -import os - import click from app.services.user_authentication.decorator import require_config @@ -25,19 +23,14 @@ from .file import file_resume # Import custom commands -from .kg_resource import kg_resource from .project import project_list_all from .use_config import set_env from .user import login from .user import logout -kg_enabled = os.environ.get('PILOT_CLI_KG_ENABLED', 'false') == 'true' - def command_groups(): commands = ['file', 'user', 'use_config', 'project', 'dataset', 'container_registry'] - if kg_enabled: - commands.append('kg_resource') return commands @@ -104,10 +97,3 @@ def cr_group(): config_group.add_command(set_env) # Custom commands -if kg_enabled: - - @entry_point.group(name='kg_resource') - def kg_resource_group(): - pass - - kg_resource_group.add_command(kg_resource) diff --git a/app/commands/kg_resource.py b/app/commands/kg_resource.py deleted file mode 100644 index 5ddb43f4..00000000 --- a/app/commands/kg_resource.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright (C) 2022-2023 Indoc Research -# -# Contact Indoc Research for any questions regarding the use of this source code. - -import click - -import app.services.output_manager.help_page as kg_help -from app.services.kg_manager.kg_resource import SrvKGResourceMgr -from app.utils.aggregated import doc - - -@click.command() -def cli(): - """KnowledgeGraph Actions.""" - pass - - -@click.command(name='import') -@click.argument('paths', type=click.Path(exists=True), nargs=-1) -@doc(kg_help.kg_resource_help_page(kg_help.KgResourceHELP.KG_IMPORT)) -def kg_resource(paths): - kg = SrvKGResourceMgr(paths) - kg.import_resource() diff --git a/app/resources/custom_help.py b/app/resources/custom_help.py index 4c35b293..6c154ae5 100644 --- a/app/resources/custom_help.py +++ b/app/resources/custom_help.py @@ -58,10 +58,6 @@ class HelpPage: 'SET_CONFIG': 'Chose config file and set for cli.', 'CONFIG_DESTINATION': 'The destination the config file goes to, default will be current cli directory.', }, - 'knowledge_graph': { - 'KG_IMPORT': 'Import dataset schema into BlueBrainNexus ', - 'KG_DATASET_CODE': 'The dataset code', - }, 'container_registry': { 'LIST_PROJECTS': 'List all projects', 'LIST_REPOSITORIES': 'List all repositories (optionally: in a given project)', diff --git a/app/services/kg_manager/kg_resource.py b/app/services/kg_manager/kg_resource.py deleted file mode 100644 index b2adc0f4..00000000 --- a/app/services/kg_manager/kg_resource.py +++ /dev/null @@ -1,89 +0,0 @@ -# Copyright (C) 2022-2023 Indoc Research -# -# Contact Indoc Research for any questions regarding the use of this source code. - -import collections -import json -import os - -import app.services.logger_services.log_functions as logger -from app.configs.app_config import AppConfig -from app.configs.user_config import UserConfig -from app.models.service_meta_class import MetaService -from app.services.output_manager.error_handler import ECustomizedError -from app.services.output_manager.error_handler import OverSizeError -from app.services.output_manager.error_handler import SrvErrorHandler -from app.services.output_manager.error_handler import customized_error_msg -from app.services.user_authentication.decorator import require_valid_token -from app.utils.aggregated import get_file_in_folder -from app.utils.aggregated import resilient_session - - -class SrvKGResourceMgr(metaclass=MetaService): - def __init__(self, paths): - self.user = UserConfig() - self.paths = paths - - def pre_load_data(self, paths): - json_data = {} - for path in paths: - invalid_json_msg = f'{path} is an invalid json file' - try: - self.validate_file_size(path) - with open(path) as f: - json_data[path] = json.load(f) - except json.decoder.JSONDecodeError: - SrvErrorHandler.customized_handle(ECustomizedError.INVALID_ACTION, False, invalid_json_msg) - continue - except OverSizeError as e: - SrvErrorHandler.default_handle(str(e), False) - continue - except Exception: - SrvErrorHandler.customized_handle(ECustomizedError.INVALID_ACTION, False, invalid_json_msg) - continue - return json_data - - def validate_file_size(self, path): - size = os.path.getsize(path) - if size > 1000000: - raise OverSizeError(customized_error_msg(ECustomizedError.OVER_SIZE) % path) - - @require_valid_token() - def import_resource(self): - url = AppConfig.Connections.url_bff + '/v1/kg/resources' - file_to_process = [] - try: - for path in self.paths: - path = os.path.relpath(path) - if os.path.isdir(path): - files = get_file_in_folder(path) - file_to_process = file_to_process + files - else: - file_to_process.append(path) - duplicate_file_list = [f for f, count in collections.Counter(file_to_process).items() if count > 1] - if duplicate_file_list: - duplicate_files = ', \n'.join(duplicate_file_list) - logger.warning(f'Following files have multiple input, it will process one time: \n{duplicate_files}') - json_data = self.pre_load_data(file_to_process) - if not json_data: - return - - payload = {'dataset_code': [], 'data': json_data} - headers = {'Authorization': 'Bearer ' + self.user.access_token} - res = resilient_session().post(url, headers=headers, json=payload) - response = res.json() - code = response.get('code') - result = response.get('result') - if code == 200: - ignored = result.get('ignored') - processed = result.get('processing') - if ignored: - ignored_files = ', \n'.join(list(ignored.keys())) - logger.warning(f'File skipped: \n{ignored_files}') - if processed: - processed_files = ', \n'.join(list(processed.keys())) - logger.succeed(f'File imported: \n{processed_files}') - else: - SrvErrorHandler.customized_handle(ECustomizedError.ERROR_CONNECTION, True) - except Exception as e: - SrvErrorHandler.customized_handle(ECustomizedError.INVALID_ACTION, False, str(e)) diff --git a/app/services/output_manager/error_handler.py b/app/services/output_manager/error_handler.py index 3d71cf9c..452b812e 100644 --- a/app/services/output_manager/error_handler.py +++ b/app/services/output_manager/error_handler.py @@ -110,8 +110,3 @@ def customized_handle(customized_error: ECustomizedError, if_exit=False, value=N logger.error(customized_error_msg(customized_error)) if if_exit: sys.exit(0) - - -class OverSizeError(Exception): - def __init__(self, message='File size is too large'): - super().__init__(message) diff --git a/app/services/output_manager/help_page.py b/app/services/output_manager/help_page.py index a75789da..033c16ce 100644 --- a/app/services/output_manager/help_page.py +++ b/app/services/output_manager/help_page.py @@ -89,16 +89,6 @@ def file_help_page(FileHELP: FileHELP): return helps.get(FileHELP.name) -class KgResourceHELP(enum.Enum): - KG_IMPORT = 'KG_IMPORT' - KG_DATASET_CODE = 'KG_DATASET_CODE' - - -def kg_resource_help_page(KgResourceHELP: KgResourceHELP): - helps = help_msg.get('knowledge_graph', 'default kg help') - return helps.get(KgResourceHELP.name) - - class ContainerRegistryHELP(enum.Enum): LIST_PROJECTS = 'LIST_PROJECTS' LIST_REPOSITORIES = 'LIST_REPOSITORIES' @@ -108,5 +98,5 @@ class ContainerRegistryHELP(enum.Enum): def cr_help_page(ContainerRegistryHELP: ContainerRegistryHELP): - helps = help_msg.get('container_registry', 'default kg help') + helps = help_msg.get('container_registry', 'default container_registry help') return helps.get(ContainerRegistryHELP.name) diff --git a/tests/app/services/kg_manager/test_kg_manager.py b/tests/app/services/kg_manager/test_kg_manager.py deleted file mode 100644 index cf422069..00000000 --- a/tests/app/services/kg_manager/test_kg_manager.py +++ /dev/null @@ -1,172 +0,0 @@ -# Copyright (C) 2022-2023 Indoc Research -# -# Contact Indoc Research for any questions regarding the use of this source code. - -from app.services.kg_manager.kg_resource import SrvKGResourceMgr - - -def test_import_kg(httpx_mock, mocker, capsys): - mocker.patch('app.services.user_authentication.token_manager.SrvTokenManager.check_valid', return_value=0) - mocker.patch('os.path.isdir', return_value=False) - mocker.patch( - 'app.services.kg_manager.kg_resource.SrvKGResourceMgr.pre_load_data', - return_value={'json1': 'data1', 'json2': 'data2'}, - ) - httpx_mock.add_response( - method='POST', - url='http://bff_cli/v1/kg/resources', - json={ - 'code': 200, - 'error_msg': '', - 'result': { - 'processing': { - 'sample_file.json': { - '@context': 'https://bluebrain.github.io/nexus/contexts/metadata.json', - '@id': 'http://test_domain/kg/v1/resources/dataset/_/d09d9594-e495-425b-a6a6-68e230ee139e', - '@type': 'http://test_domain/kg/v1/vocabs/dataset/Not_Specified', - '_constrainedBy': 'https://bluebrain.github.io/nexus/schemas/unconstrained.json', - '_createdAt': '2022-03-23T13:55:51.897Z', - '_createdBy': 'http://test_domain/kg/v1/realms/users/test_user', - '_deprecated': False, - '_incoming': ( - 'http://test_domain/kg/v1/resources/dataset/_/' - 'd09d9594-e495-425b-a6a6-68e230ee139e/incoming' - ), - '_outgoing': ( - 'http://test_domain/kg/v1/resources/dataset/_/' - 'd09d9594-e495-425b-a6a6-68e230ee139e/outgoing' - ), - '_project': 'http://test_domain/kg/v1/projects/dataset', - '_rev': 1, - '_schemaProject': 'http://test_domain/kg/v1/projects/dataset', - '_self': 'http://test_domain/kg/v1/resources/dataset/_/d09d9594-e495-425b-a6a6-68e230ee139e', - '_updatedAt': '2022-03-23T13:55:51.897Z', - '_updatedBy': 'http://test_domain/kg/v1/realms/users/test_user', - } - }, - 'ignored': {}, - }, - }, - ) - - kg_mgr = SrvKGResourceMgr(('fake_file.json',)) - kg_mgr.import_resource() - out, _ = capsys.readouterr() - assert out == 'File imported: \nsample_file.json\n' - - -def test_import_kg_invalid_json(mocker, capsys): - mocker.patch('app.services.user_authentication.token_manager.SrvTokenManager.check_valid', return_value=0) - mocker.patch('os.path.isdir', return_value=False) - kg_mgr = SrvKGResourceMgr(('fake_file.csv',)) - kg_mgr.import_resource() - out, err = capsys.readouterr() - assert out == 'Invalid action: fake_file.csv is an invalid json file\n' - - -def test_import_kg_folder(httpx_mock, mocker, capsys): - mocker.patch('app.services.user_authentication.token_manager.SrvTokenManager.check_valid', return_value=0) - mocker.patch('os.path.isdir', return_value=True) - mocker.patch( - 'app.services.kg_manager.kg_resource.SrvKGResourceMgr.pre_load_data', - return_value={'json1': 'data1', 'json2': 'data2'}, - ) - httpx_mock.add_response( - method='POST', - url='http://bff_cli/v1/kg/resources', - json={ - 'code': 200, - 'error_msg': '', - 'result': { - 'processing': { - 'json_test/fake_file1.json': { - '@context': 'https://bluebrain.github.io/nexus/contexts/metadata.json', - '@id': 'http://test_domain/kg/v1/resources/dataset/_/6b3d6b08-921a-4e73-9f53-93bdb77d76e1', - '@type': 'http://test_domain/kg/v1/vocabs/dataset/Not_Specified', - '_constrainedBy': 'https://bluebrain.github.io/nexus/schemas/unconstrained.json', - '_createdAt': '2022-03-23T18:40:23.698Z', - '_createdBy': 'http://test_domain/kg/v1/realms/users/test_user', - '_deprecated': False, - '_incoming': ( - 'http://test_domain/kg/v1/resources/dataset/_/' - '6b3d6b08-921a-4e73-9f53-93bdb77d76e1/incoming' - ), - '_outgoing': ( - 'http://test_domain/kg/v1/resources/dataset/_/' - '6b3d6b08-921a-4e73-9f53-93bdb77d76e1/outgoing' - ), - '_project': 'http://test_domain/kg/v1/projects/dataset', - '_rev': 1, - '_schemaProject': 'http://test_domain/kg/v1/projects/dataset', - '_self': 'http://test_domain/kg/v1/resources/dataset/_/6b3d6b08-921a-4e73-9f53-93bdb77d76e1', - '_updatedAt': '2022-03-23T18:40:23.698Z', - '_updatedBy': 'http://test_domain/kg/v1/realms/users/test_user', - }, - 'json_test/fake_file2.json': { - '@context': 'https://bluebrain.github.io/nexus/contexts/metadata.json', - '@id': 'http://test_domain/kg/v1/resources/dataset/_/b3088a27-4a44-4bf0-a952-942f9bf8a4f4', - '@type': 'http://test_domain/kg/v1/vocabs/dataset/Not_Specified', - '_constrainedBy': 'https://bluebrain.github.io/nexus/schemas/unconstrained.json', - '_createdAt': '2022-03-23T18:40:23.882Z', - '_createdBy': 'http://test_domain/kg/v1/realms/users/test_user', - '_deprecated': False, - '_incoming': ( - 'http://test_domain/kg/v1/resources/dataset/_/' - 'b3088a27-4a44-4bf0-a952-942f9bf8a4f4/incoming' - ), - '_outgoing': ( - 'http://test_domain/kg/v1/resources/dataset/_/' - 'b3088a27-4a44-4bf0-a952-942f9bf8a4f4/outgoing' - ), - '_project': 'http://test_domain/kg/v1/projects/dataset', - '_rev': 1, - '_schemaProject': 'http://test_domain/kg/v1/projects/dataset', - '_self': 'http://test_domain/kg/v1/resources/dataset/_/b3088a27-4a44-4bf0-a952-942f9bf8a4f4', - '_updatedAt': '2022-03-23T18:40:23.882Z', - '_updatedBy': 'http://test_domain/kg/v1/realms/users/test_user', - }, - 'json_test/fake_file3.json': { - '@context': 'https://bluebrain.github.io/nexus/contexts/metadata.json', - '@id': 'http://test_domain/kg/v1/resources/dataset/_/c45b0998-65e0-4436-a6fe-0a5b4fb1c854', - '@type': 'http://test_domain/kg/v1/vocabs/dataset/Not_Specified', - '_constrainedBy': 'https://bluebrain.github.io/nexus/schemas/unconstrained.json', - '_createdAt': '2022-03-23T18:40:24.071Z', - '_createdBy': 'http://test_domain/kg/v1/realms/users/test_user', - '_deprecated': False, - '_incoming': ( - 'http://test_domain/kg/v1/resources/dataset/_/' - 'c45b0998-65e0-4436-a6fe-0a5b4fb1c854/incoming' - ), - '_outgoing': ( - 'http://test_domain/kg/v1/resources/dataset/_/' - 'c45b0998-65e0-4436-a6fe-0a5b4fb1c854/outgoing' - ), - '_project': 'http://test_domain/kg/v1/projects/dataset', - '_rev': 1, - '_schemaProject': 'http://test_domain/kg/v1/projects/dataset', - '_self': 'http://test_domain/kg/v1/resources/dataset/_/c45b0998-65e0-4436-a6fe-0a5b4fb1c854', - '_updatedAt': '2022-03-23T18:40:24.071Z', - '_updatedBy': 'http://test_domain/kg/v1/realms/users/test_user', - }, - }, - 'ignored': {}, - }, - }, - ) - - kg_mgr = SrvKGResourceMgr(('./json_folder',)) - kg_mgr.import_resource() - out, err = capsys.readouterr() - assert out == ( - 'File imported: \njson_test/fake_file1.json, \n' 'json_test/fake_file2.json, \njson_test/fake_file3.json\n' - ) - - -def test_import_kg_invalid_json_too_large(mocker, capsys): - mocker.patch('app.services.user_authentication.token_manager.SrvTokenManager.check_valid', return_value=0) - mocker.patch('os.path.isdir', return_value=False) - mocker.patch('os.path.getsize', return_value=1000001) - kg_mgr = SrvKGResourceMgr(('fake_file.json',)) - kg_mgr.import_resource() - out, err = capsys.readouterr() - assert out == 'fake_file.json is too large\n' From abb4ea04f3f4468a78433360e982c99c91822798 Mon Sep 17 00:00:00 2001 From: Vadym Moshynskyi Date: Mon, 10 Jul 2023 18:25:31 +0200 Subject: [PATCH 2/2] PILOT-3305: Make container_registry disabled by default --- app/commands/entry_point.py | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/app/commands/entry_point.py b/app/commands/entry_point.py index 094d6d81..e9ed194c 100644 --- a/app/commands/entry_point.py +++ b/app/commands/entry_point.py @@ -2,6 +2,8 @@ # # Contact Indoc Research for any questions regarding the use of this source code. +import os + import click from app.services.user_authentication.decorator import require_config @@ -28,9 +30,13 @@ from .user import login from .user import logout +container_registry_enabled = os.environ.get('PILOT_CLI_CONTAINER_REGISTRY_ENABLED', 'false') == 'true' + def command_groups(): - commands = ['file', 'user', 'use_config', 'project', 'dataset', 'container_registry'] + commands = ['file', 'user', 'use_config', 'project', 'dataset'] + if container_registry_enabled: + commands.append('container_registry') return commands @@ -71,12 +77,6 @@ def config_group(): pass -@entry_point.group(name='container_registry') -@require_config -def cr_group(): - pass - - file_group.add_command(file_put) file_group.add_command(file_check_manifest) file_group.add_command(file_export_manifest) @@ -89,11 +89,18 @@ def cr_group(): dataset_group.add_command(dataset_list) dataset_group.add_command(dataset_show_detail) dataset_group.add_command(dataset_download) -cr_group.add_command(list_projects) -cr_group.add_command(list_repositories) -cr_group.add_command(create_project) -cr_group.add_command(get_secret) -cr_group.add_command(invite_member) config_group.add_command(set_env) # Custom commands +if container_registry_enabled: + + @entry_point.group(name='container_registry') + @require_config + def cr_group(): + pass + + cr_group.add_command(list_projects) + cr_group.add_command(list_repositories) + cr_group.add_command(create_project) + cr_group.add_command(get_secret) + cr_group.add_command(invite_member)