diff --git a/app/commands/entry_point.py b/app/commands/entry_point.py index 0d65a6e4..a16da236 100644 --- a/app/commands/entry_point.py +++ b/app/commands/entry_point.py @@ -25,27 +25,17 @@ from .file import file_resume # Import custom commands -from .hpc import hpc_auth -from .hpc import hpc_get_node -from .hpc import hpc_get_partition -from .hpc import hpc_job_info -from .hpc import hpc_job_submit -from .hpc import hpc_list_nodes -from .hpc import hpc_list_partitions from .kg_resource import kg_resource from .project import project_list_all from .use_config import set_env from .user import login from .user import logout -hpc_enabled = os.environ.get('PILOT_CLI_HPC_ENABLED', 'false') == 'true' kg_enabled = os.environ.get('PILOT_CLI_KG_ENABLED', 'false') == 'true' def command_groups(): commands = ['file', 'user', 'use_config', 'project', 'dataset', 'container_registry'] - if hpc_enabled: - commands.append('hpc') if kg_enabled: commands.append('kg_resource') return commands @@ -114,20 +104,6 @@ def cr_group(): config_group.add_command(set_env) # Custom commands -if hpc_enabled: - - @entry_point.group(name='hpc') - def hpc_group(): - pass - - hpc_group.add_command(hpc_auth) - hpc_group.add_command(hpc_job_submit) - hpc_group.add_command(hpc_job_info) - hpc_group.add_command(hpc_list_nodes) - hpc_group.add_command(hpc_get_node) - hpc_group.add_command(hpc_list_partitions) - hpc_group.add_command(hpc_get_partition) - if kg_enabled: @entry_point.group(name='kg_resource') diff --git a/app/commands/hpc.py b/app/commands/hpc.py deleted file mode 100644 index d53f4577..00000000 --- a/app/commands/hpc.py +++ /dev/null @@ -1,157 +0,0 @@ -# Copyright (C) 2022-2023 Indoc Research -# -# Contact Indoc Research for any questions regarding the use of this source code. - -import click - -import app.services.logger_services.log_functions as logger -import app.services.output_manager.help_page as hpc_help -from app.configs.user_config import UserConfig -from app.services.hpc_manager.hpc_auth import HPCTokenManager -from app.services.hpc_manager.hpc_cluster import HPCNodeManager -from app.services.hpc_manager.hpc_cluster import HPCPartitionManager -from app.services.hpc_manager.hpc_jobs import HPCJobManager -from app.services.user_authentication.user_login_logout import check_is_active -from app.services.user_authentication.user_login_logout import check_is_login -from app.services.user_authentication.user_login_logout import get_tokens -from app.utils.aggregated import doc - - -@click.command() -def cli(): - """HPC Actions.""" - pass - - -@click.command(name='token') -@click.option('-h', '--host', prompt='Host', help=(hpc_help.hpc_help_page(hpc_help.HpcHELP.HPC_LOGIN_HOST))) -@click.option('-U', '--username', prompt='Username', help=(hpc_help.hpc_help_page(hpc_help.HpcHELP.HPC_LOGIN_USERNAME))) -@click.option( - '-P', - '--password', - prompt='Password', - help=(hpc_help.hpc_help_page(hpc_help.HpcHELP.HPC_LOGIN_PASSWORD)), - hide_input=True, -) -@doc(hpc_help.hpc_help_page(hpc_help.HpcHELP.HPC_AUTH)) -def hpc_auth(host, username, password): - user = UserConfig() - is_login = check_is_login(False) - is_active = check_is_active(False) - # No login session and no input username, password - if not (username and password) and not (is_login and is_active): - username = click.prompt('Username') - password = click.prompt('Password', hide_input=True) - token = get_tokens(username, password)[0] - # Input username and password - elif username and password: - if is_login and is_active: - token = user.access_token - else: - token = get_tokens(username, password)[0] - # No Input username and password, but has login session - elif not (username and password) and (is_login and is_active): - username = user.username - password = user.password - token = user.access_token - hpc_mgr = HPCTokenManager(token) - hpc_token = hpc_mgr.auth_user(host, username, password) - logger.succeed('Authenticated successfully, token saved') - user.hpc_token = hpc_token - user.save() - - -@click.command(name='submit') -@click.option('-h', '--host', prompt='Host', help=(hpc_help.hpc_help_page(hpc_help.HpcHELP.HPC_LOGIN_HOST))) -@click.argument('path', type=click.Path(exists=True), nargs=1) -@doc(hpc_help.hpc_help_page(hpc_help.HpcHELP.HPC_SUBMIT)) -def hpc_job_submit(host, path): - hpc_mgr = HPCJobManager() - submit_job = hpc_mgr.submit_job(host, path) - for k, v in submit_job.items(): - logger.succeed(f'{k}: {v}') - - -@click.command(name='get-job') -@click.option('-h', '--host', prompt='Host', help=(hpc_help.hpc_help_page(hpc_help.HpcHELP.HPC_LOGIN_HOST))) -@click.argument('job_id', type=click.STRING, nargs=1) -@doc(hpc_help.hpc_help_page(hpc_help.HpcHELP.HPC_JOB_INFO)) -def hpc_job_info(host, job_id): - hpc_mgr = HPCJobManager() - job_info = hpc_mgr.get_job(host, job_id) - for k, v in job_info.items(): - if k not in ['standard_input']: - logger.succeed(f'{k}: {v}') - - -@click.command(name='list-nodes') -@click.option('-h', '--host', prompt='Host', help=(hpc_help.hpc_help_page(hpc_help.HpcHELP.HPC_LOGIN_HOST))) -@doc(hpc_help.hpc_help_page(hpc_help.HpcHELP.HPC_NODES)) -def hpc_list_nodes(host): - hpc_mgr = HPCNodeManager() - nodes = hpc_mgr.list_nodes(host) - for node in nodes: - for node_name, node_info in node.items(): - logger.succeed(f'\nNode name: {node_name}') - row_value = '' - for k, v in node_info.items(): - row_value = row_value + k + ': ' + str(v) + ' , ' - logger.info(row_value.rstrip(', ')) - logger.succeed('\nAll nodes are listed') - - -@click.command(name='get-node') -@click.option('-h', '--host', prompt='Host', help=(hpc_help.hpc_help_page(hpc_help.HpcHELP.HPC_LOGIN_HOST))) -@click.argument('node_name', type=click.STRING, nargs=1) -@doc(hpc_help.hpc_help_page(hpc_help.HpcHELP.HPC_GET_NODE)) -def hpc_get_node(host, node_name): - hpc_mgr = HPCNodeManager() - nodes = hpc_mgr.get_node(host, node_name) - for node in nodes: - for node_name, node_info in node.items(): - logger.succeed(f'\nNode name: {node_name}') - row_value = '' - for k, v in node_info.items(): - row_value = row_value + k + ': ' + str(v) + ' , ' - logger.info(row_value.rstrip(', ')) - logger.succeed('\n') - - -@click.command(name='list-partitions') -@click.option('-h', '--host', prompt='Host', help=(hpc_help.hpc_help_page(hpc_help.HpcHELP.HPC_LOGIN_HOST))) -@doc(hpc_help.hpc_help_page(hpc_help.HpcHELP.HPC_PARTITIONS)) -def hpc_list_partitions(host): - hpc_mgr = HPCPartitionManager() - partitions = hpc_mgr.list_partitions(host) - for partition in partitions: - for partition_name, partition_info in partition.items(): - logger.succeed(f'\nPartition name: {partition_name}') - row_value = '' - for k, v in partition_info.items(): - if k == 'tres': - value = str(v).replace(',', ', ') - else: - value = ', '.join(v) - row_value = row_value + k + ': ' + value + ' \n' - logger.info(row_value.rstrip(', ')) - logger.succeed('\nAll partitions are listed') - - -@click.command(name='get-partition') -@click.option('-h', '--host', prompt='Host', help=(hpc_help.hpc_help_page(hpc_help.HpcHELP.HPC_LOGIN_HOST))) -@click.argument('partition_name', type=click.STRING, nargs=1) -@doc(hpc_help.hpc_help_page(hpc_help.HpcHELP.HPC_GET_PARTITION)) -def hpc_get_partition(host, partition_name): - hpc_mgr = HPCPartitionManager() - partitions = hpc_mgr.get_partition(host, partition_name) - for partition in partitions: - for partition_name, partition_info in partition.items(): - logger.succeed(f'\nPartition name: {partition_name}') - row_value = '' - for k, v in partition_info.items(): - if k == 'tres': - value = str(v).replace(',', ', ') - else: - value = ', '.join(v) - row_value = row_value + k + ': ' + value + ' \n' - logger.info(row_value) diff --git a/app/configs/user_config.py b/app/configs/user_config.py index 7b417553..93c7170b 100644 --- a/app/configs/user_config.py +++ b/app/configs/user_config.py @@ -63,7 +63,6 @@ def __init__(self, config_path: Union[str, Path, None] = None, config_filename: 'access_token': '', 'refresh_token': '', 'secret': generate_secret(), - 'hpc_token': '', 'last_active': int(time.time()), 'session_id': '', } @@ -100,7 +99,6 @@ def clear(self): 'api_key': '', 'access_token': '', 'refresh_token': '', - 'hpc_token': '', 'secret': generate_secret(), 'last_active': 0, 'session_id': '', @@ -158,14 +156,6 @@ def secret(self): def secret(self, val): self.config['USER']['secret'] = val - @property - def hpc_token(self): - return decryption(self.config['USER']['hpc_token'], self.secret) - - @hpc_token.setter - def hpc_token(self, val): - self.config['USER']['hpc_token'] = encryption(val, self.secret) - @property def last_active(self): return self.config['USER']['last_active'] diff --git a/app/models/convert_type.py b/app/models/convert_type.py deleted file mode 100644 index 21177330..00000000 --- a/app/models/convert_type.py +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright (C) 2022-2023 Indoc Research -# -# Contact Indoc Research for any questions regarding the use of this source code. - -import ast - -import click - - -class PythonLiteralOption(click.Option): - def type_cast_value(self, ctx, value): - try: - return ast.literal_eval(value) - except Exception: - raise click.BadParameter(value) diff --git a/app/models/service_meta_class.py b/app/models/service_meta_class.py index 534b0f54..8557c3f9 100644 --- a/app/models/service_meta_class.py +++ b/app/models/service_meta_class.py @@ -8,10 +8,3 @@ def __new__(cls, name: str, bases, namespace, **kwargs): if not name.startswith('Srv'): raise TypeError('[Fatal] Invalid Service Statement: class name should start with "Srv"', name) return super().__new__(cls, name, bases, namespace, **kwargs) - - -class HPCMetaService(type): - def __new__(cls, name: str, bases, namespace, **kwargs): - if not name.startswith('HPC'): - raise TypeError('[Fatal] Invalid Service Statement: class name should start with "Srv"', name) - return super().__new__(cls, name, bases, namespace, **kwargs) diff --git a/app/models/upload_form.py b/app/models/upload_form.py index 032d7692..70945d4a 100644 --- a/app/models/upload_form.py +++ b/app/models/upload_form.py @@ -5,98 +5,6 @@ from app.services.file_manager.file_upload.models import FileObject -class FileUploadForm: - def __init__(self): - self._attribute_map = { - 'resumable_identifier': '', - 'resumable_filename': '', - 'resumable_chunk_number': -1, - 'resumable_total_chunks': -1, - 'resumable_total_size': -1, - 'resumable_relative_path': '', - 'tags': [], - 'uploader': '', - 'metadatas': None, - 'container_id': '', - } - - @property - def to_dict(self): - return self._attribute_map - - @property - def resumable_identifier(self): - return self._attribute_map['resumable_identifier'] - - @resumable_identifier.setter - def resumable_identifier(self, resumable_identifier): - self._attribute_map['resumable_identifier'] = resumable_identifier - - @property - def resumable_filename(self): - return self._attribute_map['resumable_filename'] - - @resumable_filename.setter - def resumable_filename(self, resumable_filename): - self._attribute_map['resumable_filename'] = resumable_filename - - @property - def resumable_chunk_number(self): - return self._attribute_map['resumable_chunk_number'] - - @resumable_chunk_number.setter - def resumable_chunk_number(self, resumable_chunk_number): - self._attribute_map['resumable_chunk_number'] = resumable_chunk_number - - @property - def resumable_total_chunks(self): - return self._attribute_map['resumable_total_chunks'] - - @resumable_total_chunks.setter - def resumable_total_chunks(self, resumable_total_chunks): - self._attribute_map['resumable_total_chunks'] = resumable_total_chunks - - @property - def resumable_relative_path(self): - return self._attribute_map['resumable_relative_path'] - - @resumable_relative_path.setter - def resumable_relative_path(self, resumable_relative_path): - self._attribute_map['resumable_relative_path'] = resumable_relative_path.rstrip('/') - - @property - def resumable_total_size(self): - return self._attribute_map['resumable_total_size'] - - @resumable_total_size.setter - def resumable_total_size(self, resumable_total_size): - self._attribute_map['resumable_total_size'] = resumable_total_size - - @property - def tags(self): - return self._attribute_map['tags'] - - @tags.setter - def tags(self, tags): - self._attribute_map['tags'] = tags - - @property - def uploader(self): - return self._attribute_map['uploader'] - - @uploader.setter - def uploader(self, uploader): - self._attribute_map['uploader'] = uploader - - @property - def metadatas(self): - return self._attribute_map['metadatas'] - - @metadatas.setter - def metadatas(self, metadatas): - self._attribute_map['metadatas'] = metadatas - - def generate_on_success_form( project_code: str, operator: str, diff --git a/app/resources/custom_error.py b/app/resources/custom_error.py index bb093d1c..afb6b512 100644 --- a/app/resources/custom_error.py +++ b/app/resources/custom_error.py @@ -94,8 +94,6 @@ class Error: 'DATASET_PERMISSION': 'You do not have permission to access this dataset', 'USER_DISABLED': 'User may not exist or has been disabled', 'OVER_SIZE': '%s is too large', - 'CANNOT_AUTH_HPC': 'Cannot proceed with HPC authorization request', - 'CANNOT_PROCESS_HPC_JOB': 'Cannot process with HPC: %s', 'CONTAINER_REGISTRY_HOST_INVALID': "Invalid host URL. Ensure host begins with 'http://' or 'https://'.", 'CONTAINER_REGISTRY_401': 'You lack valid authentication credentials for the requested resource.', 'CONTAINER_REGISTRY_403': 'You do not have permission to access this host or resource.', diff --git a/app/resources/custom_help.py b/app/resources/custom_help.py index 0da765ba..4c35b293 100644 --- a/app/resources/custom_help.py +++ b/app/resources/custom_help.py @@ -58,19 +58,6 @@ class HelpPage: 'SET_CONFIG': 'Chose config file and set for cli.', 'CONFIG_DESTINATION': 'The destination the config file goes to, default will be current cli directory.', }, - 'hpc': { - 'HPC_AUTH': 'Authorize user to HPC with access token.', - 'HPC_LOGIN_HOST': 'The host address for login HPC.', - 'HPC_LOGIN_USERNAME': 'The username for login HPC.', - 'HPC_LOGIN_PASSWORD': 'The password for login HPC.', - 'HPC_TOKEN': 'The HPC token', - 'HPC_SUBMIT': 'Submit a job to HPC', - 'HPC_JOB_INFO': 'Get a job information', - 'HPC_NODES': 'Get a list of nodes', - 'HPC_GET_NODE': 'Get node information by node name', - 'HPC_PARTITIONS': 'Get a list of partitions', - 'HPC_GET_PARTITION': 'Get partition information by partition name', - }, 'knowledge_graph': { 'KG_IMPORT': 'Import dataset schema into BlueBrainNexus ', 'KG_DATASET_CODE': 'The dataset code', diff --git a/app/services/file_manager/file_upload/upload_client.py b/app/services/file_manager/file_upload/upload_client.py index f4c42b88..f1e7ffb8 100644 --- a/app/services/file_manager/file_upload/upload_client.py +++ b/app/services/file_manager/file_upload/upload_client.py @@ -16,10 +16,10 @@ import httpx -import app.models.upload_form as uf import app.services.output_manager.message_handler as mhandler from app.configs.app_config import AppConfig from app.configs.user_config import UserConfig +from app.models.upload_form import generate_on_success_form from app.services.file_manager.file_upload.models import FileObject from app.services.file_manager.file_upload.models import UploadType from app.services.output_manager.error_handler import ECustomizedError @@ -363,7 +363,7 @@ def on_succeed(self, file_object: FileObject, tags: List[str], chunk_result: Lis for i in range(AppConfig.Env.resilient_retry): url = self.base_url + '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/v1/files' - payload = uf.generate_on_success_form( + payload = generate_on_success_form( self.project_code, self.operator, file_object, diff --git a/app/services/hpc_manager/hpc_auth.py b/app/services/hpc_manager/hpc_auth.py deleted file mode 100644 index dd1300e5..00000000 --- a/app/services/hpc_manager/hpc_auth.py +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright (C) 2022-2023 Indoc Research -# -# Contact Indoc Research for any questions regarding the use of this source code. - -from app.configs.app_config import AppConfig -from app.models.service_meta_class import HPCMetaService -from app.services.output_manager.error_handler import ECustomizedError -from app.services.output_manager.error_handler import SrvErrorHandler -from app.services.user_authentication.decorator import require_valid_token -from app.utils.aggregated import resilient_session - - -class HPCTokenManager(metaclass=HPCMetaService): - def __init__(self, token): - self.token = token - - @require_valid_token() - def auth_user(self, host, username, password): - url = AppConfig.Connections.url_bff + '/v1/hpc/auth' - payload = {'token_issuer': host, 'username': username, 'password': password} - headers = {'Authorization': 'Bearer ' + self.token} - res = resilient_session().post(url, headers=headers, json=payload) - _res = res.json() - code = _res.get('code') - if code == 200: - token = _res.get('result') - return token - else: - SrvErrorHandler.customized_handle(ECustomizedError.CANNOT_AUTH_HPC, True) diff --git a/app/services/hpc_manager/hpc_cluster.py b/app/services/hpc_manager/hpc_cluster.py deleted file mode 100644 index af89d4d0..00000000 --- a/app/services/hpc_manager/hpc_cluster.py +++ /dev/null @@ -1,123 +0,0 @@ -# Copyright (C) 2022-2023 Indoc Research -# -# Contact Indoc Research for any questions regarding the use of this source code. - -from app.configs.app_config import AppConfig -from app.configs.user_config import UserConfig -from app.models.service_meta_class import HPCMetaService -from app.services.output_manager.error_handler import ECustomizedError -from app.services.output_manager.error_handler import SrvErrorHandler -from app.services.user_authentication.decorator import require_valid_token -from app.utils.aggregated import resilient_session - - -class HPCPartitionManager(metaclass=HPCMetaService): - def __init__(self): - self.user = UserConfig() - if self.user.hpc_token: - self.token = self.user.hpc_token - else: - SrvErrorHandler.customized_handle(ECustomizedError.CANNOT_PROCESS_HPC_JOB, True, value='Invalid HPC token') - self.username = self.user.username - - @require_valid_token('kong') - def list_partitions(self, host): - url = AppConfig.Connections.url_bff + '/v1/hpc/partitions' - paramas = {'host': host, 'username': self.username, 'token': self.token} - headers = {'Authorization': 'Bearer ' + self.user.access_token} - res = resilient_session().get(url, headers=headers, params=paramas) - _res = res.json() - code = _res.get('code') - if code == 200: - _info = _res.get('result') - return _info - elif code == 400: - error_msg = _res.get('error_msg') - if 'HPC protocal required' in error_msg: - error_detail = f'missing protocol in the host, try http://{host} or https://{host}' - else: - error_detail = 'Cannot list partitions, please verify your host and try again later' - SrvErrorHandler.customized_handle(ECustomizedError.CANNOT_PROCESS_HPC_JOB, True, value=error_detail) - else: - SrvErrorHandler.customized_handle(ECustomizedError.CANNOT_PROCESS_HPC_JOB, True, alue='List partitions') - - @require_valid_token() - def get_partition(self, host, partition_name): - url = AppConfig.Connections.url_bff + f'/v1/hpc/partitions/{partition_name}' - params = {'host': host, 'username': self.username, 'token': self.token, 'partition_name': partition_name} - headers = {'Authorization': 'Bearer ' + self.user.access_token} - res = resilient_session().get(url, headers=headers, params=params) - _res = res.json() - code = _res.get('code') - if code == 200: - result = _res.get('result') - return result - elif code == 400: - error_msg = _res.get('error_msg') - if 'HPC protocal required' in error_msg: - error_detail = f'missing protocol in the host, try http://{host} or https://{host}' - else: - error_detail = 'Cannot get partition, please verify your partition name and try again later' - SrvErrorHandler.customized_handle(ECustomizedError.CANNOT_PROCESS_HPC_JOB, True, value=error_detail) - elif code == 404: - error_detail = f'Partition {partition_name} may not exist' - SrvErrorHandler.customized_handle(ECustomizedError.CANNOT_PROCESS_HPC_JOB, True, value=error_detail) - else: - error_detail = f'Get partition {partition_name}' - SrvErrorHandler.customized_handle(ECustomizedError.CANNOT_PROCESS_HPC_JOB, True, value=error_detail) - - -class HPCNodeManager(metaclass=HPCMetaService): - def __init__(self): - self.user = UserConfig() - if self.user.hpc_token: - self.token = self.user.hpc_token - else: - SrvErrorHandler.customized_handle(ECustomizedError.CANNOT_PROCESS_HPC_JOB, True, value='Invalid HPC token') - self.username = self.user.username - - @require_valid_token() - def get_node(self, host, node_name): - url = AppConfig.Connections.url_bff + f'/v1/hpc/nodes/{node_name}' - params = {'host': host, 'username': self.username, 'token': self.token, 'node_name': node_name} - headers = {'Authorization': 'Bearer ' + self.user.access_token} - res = resilient_session().get(url, headers=headers, params=params) - _res = res.json() - code = _res.get('code') - if code == 200: - result = _res.get('result') - return result - elif code == 400: - error_msg = _res.get('error_msg') - if 'HPC protocal required' in error_msg: - error_detail = f'missing protocol in the host, try http://{host} or https://{host}' - else: - error_detail = 'Cannot get node information, please verify your node name and try again later' - SrvErrorHandler.customized_handle(ECustomizedError.CANNOT_PROCESS_HPC_JOB, True, value=error_detail) - elif code == 404: - error_detail = f'Node {node_name} may not exist' - SrvErrorHandler.customized_handle(ECustomizedError.CANNOT_PROCESS_HPC_JOB, True, value=error_detail) - else: - error_detail = f'Get node {node_name}' - SrvErrorHandler.customized_handle(ECustomizedError.CANNOT_PROCESS_HPC_JOB, True, value=error_detail) - - @require_valid_token() - def list_nodes(self, host): - url = AppConfig.Connections.url_bff + '/v1/hpc/nodes' - paramas = {'host': host, 'username': self.username, 'token': self.token} - headers = {'Authorization': 'Bearer ' + self.user.access_token} - res = resilient_session().get(url, headers=headers, params=paramas) - _res = res.json() - code = _res.get('code') - if code == 200: - _info = _res.get('result') - return _info - elif code == 400: - error_msg = _res.get('error_msg') - if 'HPC protocal required' in error_msg: - error_detail = f'missing protocol in the host, try http://{host} or https://{host}' - else: - error_detail = 'Cannot list nodes, please verify your host and try again later' - SrvErrorHandler.customized_handle(ECustomizedError.CANNOT_PROCESS_HPC_JOB, True, value=error_detail) - else: - SrvErrorHandler.customized_handle(ECustomizedError.CANNOT_PROCESS_HPC_JOB, True, value='List nodes') diff --git a/app/services/hpc_manager/hpc_jobs.py b/app/services/hpc_manager/hpc_jobs.py deleted file mode 100644 index ed89f511..00000000 --- a/app/services/hpc_manager/hpc_jobs.py +++ /dev/null @@ -1,66 +0,0 @@ -# Copyright (C) 2022-2023 Indoc Research -# -# Contact Indoc Research for any questions regarding the use of this source code. - -import json - -from app.configs.app_config import AppConfig -from app.configs.user_config import UserConfig -from app.models.service_meta_class import HPCMetaService -from app.services.output_manager.error_handler import ECustomizedError -from app.services.output_manager.error_handler import SrvErrorHandler -from app.services.output_manager.response_handler import HPCJobInfoResponse -from app.services.output_manager.response_handler import HPCJobSubmitResponse -from app.services.user_authentication.decorator import require_valid_token -from app.utils.aggregated import resilient_session - - -class HPCJobManager(metaclass=HPCMetaService): - def __init__(self): - self.user = UserConfig() - self.token = ( - self.user.hpc_token - if self.user.hpc_token - else SrvErrorHandler.customized_handle( - ECustomizedError.CANNOT_PROCESS_HPC_JOB, True, value='Invalid HPC token' - ) - ) - self.username = self.user.username - - def pre_load_data(self, path): - json_data = {} - try: - with open(path) as f: - json_data = json.load(f) - f.close() - except json.decoder.JSONDecodeError: - SrvErrorHandler.customized_handle(ECustomizedError.INVALID_ACTION, False, f'{path} is an invalid json file') - except Exception: - SrvErrorHandler.customized_handle(ECustomizedError.INVALID_ACTION, False, f'{path} is an invalid json file') - return json_data - - @require_valid_token() - def submit_job(self, host, path): - url = AppConfig.Connections.url_bff + '/v1/hpc/job' - job_info = self.pre_load_data(path) - payload = {'host': host, 'username': self.username, 'token': self.token, 'job_info': job_info} - headers = {'Authorization': 'Bearer ' + self.user.access_token} - res = resilient_session().post(url, headers=headers, json=payload) - _res = res.json() - code = _res.get('code') - result = _res.get('result') - _ = getattr(HPCJobSubmitResponse(payload, _res), f'return_{code}_response')() - return result - - @require_valid_token() - def get_job(self, host, job_id): - url = AppConfig.Connections.url_bff + f'/v1/hpc/job/{job_id}' - params = {'host': host, 'username': self.username, 'token': self.token} - headers = {'Authorization': 'Bearer ' + self.user.access_token} - response = resilient_session().get(url, headers=headers, params=params) - _res = response.json() - code = _res.get('code') - result = _res.get('result') - params['job_id'] = job_id - _ = getattr(HPCJobInfoResponse(params, _res), f'return_{code}_response')() - return result diff --git a/app/services/output_manager/error_handler.py b/app/services/output_manager/error_handler.py index bf1d02ae..3d71cf9c 100644 --- a/app/services/output_manager/error_handler.py +++ b/app/services/output_manager/error_handler.py @@ -67,8 +67,6 @@ class ECustomizedError(enum.Enum): DATASET_NOT_EXIST = 'DATASET_NOT_EXIST' DATASET_PERMISSION = 'DATASET_PERMISSION' USER_DISABLED = 'USER_DISABLED' - CANNOT_AUTH_HPC = 'CANNOT_AUTH_HPC' - CANNOT_PROCESS_HPC_JOB = 'CANNOT_PROCESS_HPC_JOB' OVER_SIZE = 'OVER_SIZE' CONTAINER_REGISTRY_HOST_INVALID = 'CONTAINER_REGISTRY_HOST_INVALID' CONTAINER_REGISTRY_401 = 'CONTAINER_REGISTRY_401' diff --git a/app/services/output_manager/help_page.py b/app/services/output_manager/help_page.py index 68102f70..a75789da 100644 --- a/app/services/output_manager/help_page.py +++ b/app/services/output_manager/help_page.py @@ -89,25 +89,6 @@ def file_help_page(FileHELP: FileHELP): return helps.get(FileHELP.name) -class HpcHELP(enum.Enum): - HPC_AUTH = 'HPC_AUTH' - HPC_LOGIN_HOST = 'HPC_LOGIN_HOST' - HPC_LOGIN_USERNAME = 'HPC_LOGIN_USERNAME' - HPC_LOGIN_PASSWORD = 'HPC_LOGIN_PASSWORD' - HPC_TOKEN = 'HPC_TOKEN' - HPC_SUBMIT = 'HPC_SUBMIT' - HPC_JOB_INFO = 'HPC_JOB_INFO' - HPC_NODES = 'HPC_NODES' - HPC_GET_NODE = 'HPC_GET_NODE' - HPC_PARTITIONS = 'HPC_PARTITIONS' - HPC_GET_PARTITION = 'HPC_GET_PARTITION' - - -def hpc_help_page(HpcHELP: HpcHELP): - helps = help_msg.get('hpc', 'default hpc help') - return helps.get(HpcHELP.name) - - class KgResourceHELP(enum.Enum): KG_IMPORT = 'KG_IMPORT' KG_DATASET_CODE = 'KG_DATASET_CODE' diff --git a/app/services/output_manager/response_handler.py b/app/services/output_manager/response_handler.py deleted file mode 100644 index 8d394f31..00000000 --- a/app/services/output_manager/response_handler.py +++ /dev/null @@ -1,73 +0,0 @@ -# Copyright (C) 2022-2023 Indoc Research -# -# Contact Indoc Research for any questions regarding the use of this source code. - -from app.services.output_manager.error_handler import ECustomizedError -from app.services.output_manager.error_handler import SrvErrorHandler - - -class HPCJobInfoResponse: - def __init__(self, payload: dict, response: dict): - self.payload = payload - self.res = response - - def return_200_response(self): - pass - - def return_400_response(self): - error_msg = self.res.get('error_msg') - host = self.payload.get('host') - if 'HPC protocal required' in error_msg: - error_detail = f'missing protocol in the host, try http://{host} or https://{host}' - else: - error_detail = 'Cannot get job, please verify your job ID and try again later' - SrvErrorHandler.customized_handle(ECustomizedError.CANNOT_PROCESS_HPC_JOB, True, value=error_detail) - - def return_404_response(self): - error_msg = self.res.get('error_msg') - job_id = self.payload.get('job_id') - host = self.payload.get('host') - if 'Job ID' in error_msg: - error_detail = f'job {job_id} may not exist' - elif 'Host not found' in error_msg: - error_detail = f'host {host} may not exist' - SrvErrorHandler.customized_handle(ECustomizedError.CANNOT_PROCESS_HPC_JOB, True, value=error_detail) - - def return_500_response(self): - job_id = self.payload.get('job_id') - error_detail = f'Job ID {job_id}' - SrvErrorHandler.customized_handle(ECustomizedError.CANNOT_PROCESS_HPC_JOB, True, value=error_detail) - - -class HPCJobSubmitResponse: - def __init__(self, payload: dict, response: dict) -> None: - self.payload = payload - self.res = response - - def return_200_response(self): - pass - - def return_400_response(self): - error_msg = self.res.get('error_msg') - host = self.payload.get('host') - if 'Missing script' in error_msg: - error_detail = f'{error_msg} in the job json file' - elif 'HPC protocal required' in error_msg: - error_detail = f'missing protocol in the host, try http://{host} or https://{host}' - else: - error_detail = 'Cannot submit job, please verify your json file and try again later' - SrvErrorHandler.customized_handle(ECustomizedError.CANNOT_PROCESS_HPC_JOB, True, value=error_detail) - - def return_403_response(self): - error_detail = self.res.get('error_msg') - SrvErrorHandler.customized_handle(ECustomizedError.CANNOT_PROCESS_HPC_JOB, True, value=error_detail) - - def return_500_response(self): - path = self.payload.get('path') - error_detail = f'submit job {path}' - SrvErrorHandler.customized_handle(ECustomizedError.CANNOT_PROCESS_HPC_JOB, True, value=error_detail) - - -class HPCListPartitionsResponse: - def __init__(self) -> None: - pass diff --git a/app/services/user_authentication/user_login_logout.py b/app/services/user_authentication/user_login_logout.py index b6e8882f..0cad5c5f 100644 --- a/app/services/user_authentication/user_login_logout.py +++ b/app/services/user_authentication/user_login_logout.py @@ -48,7 +48,6 @@ def login_using_api_key(api_key: str) -> bool: user_config.refresh_token = '' user_config.username = username user_config.last_active = str(int(time.time())) - user_config.hpc_token = '' user_config.session_id = 'cli-' + str(uuid4()) user_config.save() @@ -107,7 +106,6 @@ def validate_user_device_login(device_code: str, expires: int, interval: int) -> user_config.refresh_token = resp_dict['refresh_token'] user_config.username = decode_token['preferred_username'] user_config.last_active = str(int(time.time())) - user_config.hpc_token = '' user_config.session_id = 'cli-' + str(uuid4()) user_config.save() diff --git a/app/utils/aggregated.py b/app/utils/aggregated.py index 281eee63..a5c48e53 100644 --- a/app/utils/aggregated.py +++ b/app/utils/aggregated.py @@ -2,7 +2,6 @@ # # Contact Indoc Research for any questions regarding the use of this source code. -import datetime import os import re import shutil @@ -18,10 +17,6 @@ from env import ConfigClass -def get_current_datetime(): - return datetime.datetime.now().isoformat() - - def resilient_session(): # each resilient session will headers = {'VM-Info': ConfigClass.VM_INFO} diff --git a/tests/app/services/hpc_manager/test_hpc_auth.py b/tests/app/services/hpc_manager/test_hpc_auth.py deleted file mode 100644 index e31dae13..00000000 --- a/tests/app/services/hpc_manager/test_hpc_auth.py +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright (C) 2022-2023 Indoc Research -# -# Contact Indoc Research for any questions regarding the use of this source code. - -import pytest - -from app.services.hpc_manager.hpc_auth import HPCTokenManager - - -def test_hpc_auth(httpx_mock, mocker): - mocker.patch('app.services.user_authentication.token_manager.SrvTokenManager.check_valid', return_value=0) - httpx_mock.add_response( - method='POST', - url='http://bff_cli' + '/v1/hpc/auth', - json={'code': 200, 'error_msg': '', 'result': 'fake-token'}, - status_code=200, - ) - hpc_mgr = HPCTokenManager('fake_token') - token = hpc_mgr.auth_user('test_host', 'test_user', 'test_password') - assert token == 'fake-token' - - -def test_hpc_auth_failed(httpx_mock, mocker, capsys): - mocker.patch('app.services.user_authentication.token_manager.SrvTokenManager.check_valid', return_value=0) - httpx_mock.add_response( - method='POST', - url='http://bff_cli' + '/v1/hpc/auth', - json={'code': 500, 'error_msg': 'User authorization failed: Authentication failed.', 'result': []}, - ) - hpc_mgr = HPCTokenManager('fake_token') - with pytest.raises(SystemExit): - hpc_mgr.auth_user('test_host', 'test_user', 'test_password') - out, err = capsys.readouterr() - assert out == 'Cannot proceed with HPC authorization request\n' diff --git a/tests/app/services/hpc_manager/test_hpc_cluster.py b/tests/app/services/hpc_manager/test_hpc_cluster.py deleted file mode 100644 index 986c7efc..00000000 --- a/tests/app/services/hpc_manager/test_hpc_cluster.py +++ /dev/null @@ -1,55 +0,0 @@ -# Copyright (C) 2022-2023 Indoc Research -# -# Contact Indoc Research for any questions regarding the use of this source code. - -import pytest - -from app.configs.user_config import UserConfig -from app.resources.custom_error import Error -from app.services.hpc_manager.hpc_cluster import HPCPartitionManager -from app.services.output_manager.error_handler import ECustomizedError -from tests.conftest import decoded_token - - -def test_hpc_list_partitions(httpx_mock, mocker): - mocker.patch( - 'app.services.user_authentication.token_manager.SrvTokenManager.decode_access_token', - return_value=decoded_token(), - ) - user_config = UserConfig() - user_config.username = 'test-user' - user_config.hpc_token = 'test-hpc-token' - httpx_mock.add_response( - method='GET', - url='http://bff_cli/v1/hpc/partitions?host=test_host&username=test-user&token=test-hpc-token', - json={ - 'code': 200, - 'error_msg': '', - 'result': [ - {'partition1': {'nodes': ['hpc-node1'], 'tres': 'cpu=2,mem=4G,node=1,billing=2'}}, - {'partition2': {'nodes': ['hpc-node2'], 'tres': 'cpu=1,mem=8G,node=1,billing=1'}}, - ], - }, - ) - - expected_partitions = [ - {'partition1': {'nodes': ['hpc-node1'], 'tres': 'cpu=2,mem=4G,node=1,billing=2'}}, - {'partition2': {'nodes': ['hpc-node2'], 'tres': 'cpu=1,mem=8G,node=1,billing=1'}}, - ] - hpc_mgr = HPCPartitionManager() - partion = hpc_mgr.list_partitions('test_host') - assert partion == expected_partitions - - -def test_hpc_list_partitions_no_token(mocker, capsys): - mocker.patch( - 'app.services.user_authentication.token_manager.SrvTokenManager.decode_access_token', - return_value=decoded_token(), - ) - user_config = UserConfig() - user_config.hpc_token = '' - with pytest.raises(SystemExit): - hpc_mgr = HPCPartitionManager() - _ = hpc_mgr.list_partitions('test_host') - out, err = capsys.readouterr() - assert out == Error.error_msg.get(ECustomizedError.CANNOT_PROCESS_HPC_JOB.name) % 'Invalid HPC token\n' diff --git a/tests/conftest.py b/tests/conftest.py index ea0c6b85..ec2653c9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -31,7 +31,6 @@ def mock_settings(monkeypatch, mocker): monkeypatch.setattr(UserConfig, 'api_key', 'test-api-key') monkeypatch.setattr(UserConfig, 'access_token', 'test-access-token') monkeypatch.setattr(UserConfig, 'refresh_token', 'test-refresh-token') - monkeypatch.setattr(UserConfig, 'hpc_token', 'test-hpc-token') mocker.patch('app.configs.user_config.UserConfig.save') # Do not save config when running tests