From 9c4b3bff3ea998a57729ea611cf00e9fe37d9e7c Mon Sep 17 00:00:00 2001 From: zhiren Date: Tue, 19 Dec 2023 16:53:15 -0500 Subject: [PATCH 1/6] update config file to allow user customize the api_url and keycloak_url instead of domain only --- README.md | 26 +++++++++++++++++-------- app/configs/app_config.py | 6 +++--- app/configs/config.py | 41 ++++++++++++++------------------------- pyproject.toml | 2 +- 4 files changed, 37 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index 0fb3f2ef..a43e3926 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,24 @@ Command line tool that allows the user to execute data operations on the platfor - Python - [Click](https://click.palletsprojects.com/en/8.0.x/) +## Getting Started + +### Prerequisites +- Python 3.7+ +- [Poetry](https://python-poetry.org/docs/#installation) + +#### Run with Python +1. Install dependencies (optional: run in edit mode). + ``` + poetry install + poetry run python app/pilotcli.py --help + ``` +2. Add environment variables if needed. + + 1. Create a `.env` file in the root directory of the project. + 2. Sdd following two environmental varibles to the `.env` file. + - `api_url`: the url that the api server is hosted on. default is `https://api.pilot.indocresearch.com/pilot` + - `keycloak_url`: thr url that the keycloak server is hosted on. default is `https://iam.pilot.indocresearch.com/realms/pilot/protocol/openid-connect` #### Run from bundled application 1. Navigate to the appropriate directory for your system. @@ -17,14 +35,6 @@ Command line tool that allows the user to execute data operations on the platfor ./app/bundled_app/mac/ ./app/bundled_app/mac_arm/ -#### Run with Python -1. Install dependencies (optional: run in edit mode). - - poetry install - poetry run pilotcli - -2. Add environment variables if needed. - ## Usage ./app/bundled_app/linux/pilotcli --help diff --git a/app/configs/app_config.py b/app/configs/app_config.py index 79e4158f..d1c77ef9 100644 --- a/app/configs/app_config.py +++ b/app/configs/app_config.py @@ -41,8 +41,8 @@ class Connections: url_dataset_v2download = ConfigClass.url_dataset_v2download url_dataset = ConfigClass.url_dataset url_validation = ConfigClass.url_validation - url_keycloak = ConfigClass.url_keycloak - url_keycloak_token = f'{ConfigClass.url_keycloak}/token' + url_keycloak = ConfigClass.keycloak_url + url_keycloak_token = f'{ConfigClass.keycloak_url}/token' url_bff = ConfigClass.url_bff - url_base = ConfigClass.base_url + url_base = ConfigClass.api_url url_portal = ConfigClass.url_portal diff --git a/app/configs/config.py b/app/configs/config.py index 04bbf98c..2d478319 100644 --- a/app/configs/config.py +++ b/app/configs/config.py @@ -28,59 +28,48 @@ class Settings(BaseSettings): harbor_client_secret: str = '' url_harbor: str = '' - domain: str = 'pilot.indocresearch.com' - - @computed_field - def base_url(self) -> str: - return f'https://api.{self.domain}/pilot' + api_url: str = 'https://api.pilot.indocresearch.com/pilot' + keycloak_url: str = 'https://iam.pilot.indocresearch.com/realms/pilot/protocol/openid-connect' @computed_field def url_bff(self) -> str: - return f'{self.base_url}/cli' + return f'{self.api_url}/cli' @computed_field def url_portal(self) -> str: - return f'{self.base_url}/portal' - - @computed_field - def url_keycloak_realm(self) -> str: - return f'https://iam.{self.domain}/realms/pilot' - - @computed_field - def url_keycloak(self) -> str: - return f'{self.url_keycloak_realm}/protocol/openid-connect' + return f'{self.api_url}/portal' @computed_field def url_authn(self) -> str: - return f'{self.base_url}/portal/users/auth' + return f'{self.api_url}/portal/users/auth' @computed_field def url_refresh_token(self) -> str: - return f'{self.base_url}/portal/users/refresh' + return f'{self.api_url}/portal/users/refresh' @computed_field def url_file_tag(self) -> str: - return f'{self.base_url}/portal/v2/%s/tags' + return f'{self.api_url}/portal/v2/%s/tags' @computed_field def url_upload_greenroom(self) -> str: - return f'{self.base_url}/upload/gr' + return f'{self.api_url}/upload/gr' @computed_field def url_upload_core(self) -> str: - return f'{self.base_url}/upload/core' + return f'{self.api_url}/upload/core' @computed_field def url_status(self) -> str: - return f'{self.base_url}/portal/v1/files/actions/tasks' + return f'{self.api_url}/portal/v1/files/actions/tasks' @computed_field def url_download_greenroom(self) -> str: - return f'{self.base_url}/portal/download/gr/' + return f'{self.api_url}/portal/download/gr/' @computed_field def url_download_core(self) -> str: - return f'{self.base_url}/portal/download/core/' + return f'{self.api_url}/portal/download/core/' @computed_field def url_v2_download_pre(self) -> str: @@ -88,15 +77,15 @@ def url_v2_download_pre(self) -> str: @computed_field def url_dataset_v2download(self) -> str: - return f'{self.base_url}/portal/download/core/v2/dataset' + return f'{self.api_url}/portal/download/core/v2/dataset' @computed_field def url_dataset(self) -> str: - return f'{self.base_url}/portal/v1/dataset' + return f'{self.api_url}/portal/v1/dataset' @computed_field def url_validation(self) -> str: - return f'{self.base_url}/v1/files/validation' + return f'{self.api_url}/v1/files/validation' @lru_cache(1) diff --git a/pyproject.toml b/pyproject.toml index 9154869f..5ec41f62 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "app" -version = "2.9.3" +version = "2.9.4" description = "This service is designed to support pilot platform" authors = ["Indoc Systems"] From c8c738101552da94c4fb582322d853b77e84ac4c Mon Sep 17 00:00:00 2001 From: zhiren Date: Tue, 19 Dec 2023 17:06:55 -0500 Subject: [PATCH 2/6] fixup the test cases --- app/services/user_authentication/user_login_logout.py | 2 +- tests/app/commands/test_user.py | 2 +- tests/app/services/user_authentication/test_token_manager.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/services/user_authentication/user_login_logout.py b/app/services/user_authentication/user_login_logout.py index 33d71838..fd3ce5e3 100644 --- a/app/services/user_authentication/user_login_logout.py +++ b/app/services/user_authentication/user_login_logout.py @@ -23,7 +23,7 @@ def exchange_api_key(api_key: str) -> Union[str, None]: """Exchange API Key with JWT token using Keycloak.""" - url = f'{ConfigClass.url_keycloak_realm}/api-key/{api_key}' + url = f'{ConfigClass.keycloak_url}/api-key/{api_key}' try: response = requests.get(url, timeout=5) response.raise_for_status() diff --git a/tests/app/commands/test_user.py b/tests/app/commands/test_user.py index 3d2ff335..bc06e552 100644 --- a/tests/app/commands/test_user.py +++ b/tests/app/commands/test_user.py @@ -14,7 +14,7 @@ def test_login_command_with_api_key_option_calls_keycloak_and_stores_response_in username = fake.user_name() api_key = fake.pystr(20) access_token = jwt.encode({'preferred_username': username}, key='').decode() - requests_mock.get(f'{settings.url_keycloak_realm}/api-key/{api_key}', json={'access_token': access_token}) + requests_mock.get(f'{settings.keycloak_url}/api-key/{api_key}', json={'access_token': access_token}) result = cli_runner.invoke(login, ['--api-key', api_key]) diff --git a/tests/app/services/user_authentication/test_token_manager.py b/tests/app/services/user_authentication/test_token_manager.py index 6846ae20..e0e5fc10 100644 --- a/tests/app/services/user_authentication/test_token_manager.py +++ b/tests/app/services/user_authentication/test_token_manager.py @@ -33,7 +33,7 @@ def test_refresh_api_key_calls_keycloak_and_stores_access_token_in_config(self, manager = SrvTokenManager() access_token = jwt.encode({}, key='').decode() requests_mock.get( - f'{settings.url_keycloak_realm}/api-key/{manager.config.api_key}', + f'{settings.keycloak_url}/api-key/{manager.config.api_key}', json={'access_token': access_token}, ) From 4487ad5315a90262eec1e9e7ea10c721123da0a6 Mon Sep 17 00:00:00 2001 From: zhiren Date: Tue, 19 Dec 2023 17:16:42 -0500 Subject: [PATCH 3/6] use the keycloak realm url instead of keycloak token in config --- app/configs/app_config.py | 5 +++-- app/configs/config.py | 6 +++++- app/services/user_authentication/user_login_logout.py | 2 +- tests/app/commands/test_user.py | 2 +- .../app/services/user_authentication/test_token_manager.py | 2 +- 5 files changed, 11 insertions(+), 6 deletions(-) diff --git a/app/configs/app_config.py b/app/configs/app_config.py index d1c77ef9..85e59928 100644 --- a/app/configs/app_config.py +++ b/app/configs/app_config.py @@ -41,8 +41,9 @@ class Connections: url_dataset_v2download = ConfigClass.url_dataset_v2download url_dataset = ConfigClass.url_dataset url_validation = ConfigClass.url_validation - url_keycloak = ConfigClass.keycloak_url - url_keycloak_token = f'{ConfigClass.keycloak_url}/token' + url_keycloak = ConfigClass.url_keycloak + url_keycloak_token = f'{ConfigClass.url_keycloak}/token' + url_keycloak_realm = ConfigClass.keycloak_realm_url url_bff = ConfigClass.url_bff url_base = ConfigClass.api_url url_portal = ConfigClass.url_portal diff --git a/app/configs/config.py b/app/configs/config.py index 2d478319..9b65018d 100644 --- a/app/configs/config.py +++ b/app/configs/config.py @@ -29,7 +29,7 @@ class Settings(BaseSettings): url_harbor: str = '' api_url: str = 'https://api.pilot.indocresearch.com/pilot' - keycloak_url: str = 'https://iam.pilot.indocresearch.com/realms/pilot/protocol/openid-connect' + keycloak_realm_url: str = 'https://iam.pilot.indocresearch.com/realms/pilot' @computed_field def url_bff(self) -> str: @@ -39,6 +39,10 @@ def url_bff(self) -> str: def url_portal(self) -> str: return f'{self.api_url}/portal' + @computed_field + def url_keycloak(self) -> str: + return f'{self.keycloak_realm_url}/protocol/openid-connect' + @computed_field def url_authn(self) -> str: return f'{self.api_url}/portal/users/auth' diff --git a/app/services/user_authentication/user_login_logout.py b/app/services/user_authentication/user_login_logout.py index fd3ce5e3..4b27f5bc 100644 --- a/app/services/user_authentication/user_login_logout.py +++ b/app/services/user_authentication/user_login_logout.py @@ -23,7 +23,7 @@ def exchange_api_key(api_key: str) -> Union[str, None]: """Exchange API Key with JWT token using Keycloak.""" - url = f'{ConfigClass.keycloak_url}/api-key/{api_key}' + url = f'{ConfigClass.keycloak_realm_url}/api-key/{api_key}' try: response = requests.get(url, timeout=5) response.raise_for_status() diff --git a/tests/app/commands/test_user.py b/tests/app/commands/test_user.py index bc06e552..3d2ff335 100644 --- a/tests/app/commands/test_user.py +++ b/tests/app/commands/test_user.py @@ -14,7 +14,7 @@ def test_login_command_with_api_key_option_calls_keycloak_and_stores_response_in username = fake.user_name() api_key = fake.pystr(20) access_token = jwt.encode({'preferred_username': username}, key='').decode() - requests_mock.get(f'{settings.keycloak_url}/api-key/{api_key}', json={'access_token': access_token}) + requests_mock.get(f'{settings.url_keycloak_realm}/api-key/{api_key}', json={'access_token': access_token}) result = cli_runner.invoke(login, ['--api-key', api_key]) diff --git a/tests/app/services/user_authentication/test_token_manager.py b/tests/app/services/user_authentication/test_token_manager.py index e0e5fc10..6846ae20 100644 --- a/tests/app/services/user_authentication/test_token_manager.py +++ b/tests/app/services/user_authentication/test_token_manager.py @@ -33,7 +33,7 @@ def test_refresh_api_key_calls_keycloak_and_stores_access_token_in_config(self, manager = SrvTokenManager() access_token = jwt.encode({}, key='').decode() requests_mock.get( - f'{settings.keycloak_url}/api-key/{manager.config.api_key}', + f'{settings.url_keycloak_realm}/api-key/{manager.config.api_key}', json={'access_token': access_token}, ) From 4726cf052471df7eb438bba4d449cf89e79dfe11 Mon Sep 17 00:00:00 2001 From: zhiren Date: Tue, 19 Dec 2023 17:17:20 -0500 Subject: [PATCH 4/6] update README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a43e3926..b469ea7e 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ Command line tool that allows the user to execute data operations on the platfor 1. Create a `.env` file in the root directory of the project. 2. Sdd following two environmental varibles to the `.env` file. - `api_url`: the url that the api server is hosted on. default is `https://api.pilot.indocresearch.com/pilot` - - `keycloak_url`: thr url that the keycloak server is hosted on. default is `https://iam.pilot.indocresearch.com/realms/pilot/protocol/openid-connect` + - `keycloak_realm_url`: thr url that the keycloak server is hosted on. default is `https://iam.pilot.indocresearch.com/realms/pilot` #### Run from bundled application 1. Navigate to the appropriate directory for your system. From e788074412f79d828f00a6a2cee52b364f93e989 Mon Sep 17 00:00:00 2001 From: zhiren Date: Tue, 19 Dec 2023 17:22:57 -0500 Subject: [PATCH 5/6] unify the config usage --- app/services/user_authentication/user_login_logout.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/services/user_authentication/user_login_logout.py b/app/services/user_authentication/user_login_logout.py index 4b27f5bc..b4ca2bef 100644 --- a/app/services/user_authentication/user_login_logout.py +++ b/app/services/user_authentication/user_login_logout.py @@ -23,7 +23,7 @@ def exchange_api_key(api_key: str) -> Union[str, None]: """Exchange API Key with JWT token using Keycloak.""" - url = f'{ConfigClass.keycloak_realm_url}/api-key/{api_key}' + url = f'{AppConfig.Connections.url_keycloak_realm}/api-key/{api_key}' try: response = requests.get(url, timeout=5) response.raise_for_status() From 2d532c43c881474c6e3409501c54c0e6ab22a2d1 Mon Sep 17 00:00:00 2001 From: zhiren Date: Tue, 19 Dec 2023 17:29:44 -0500 Subject: [PATCH 6/6] fixup test cases --- tests/app/commands/test_user.py | 5 ++++- tests/app/services/user_authentication/test_token_manager.py | 2 +- tests/conftest.py | 1 + 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/app/commands/test_user.py b/tests/app/commands/test_user.py index 3d2ff335..77b4b76d 100644 --- a/tests/app/commands/test_user.py +++ b/tests/app/commands/test_user.py @@ -5,6 +5,7 @@ import jwt from app.commands.user import login +from app.configs.app_config import AppConfig from app.configs.user_config import UserConfig @@ -14,7 +15,9 @@ def test_login_command_with_api_key_option_calls_keycloak_and_stores_response_in username = fake.user_name() api_key = fake.pystr(20) access_token = jwt.encode({'preferred_username': username}, key='').decode() - requests_mock.get(f'{settings.url_keycloak_realm}/api-key/{api_key}', json={'access_token': access_token}) + requests_mock.get( + f'{AppConfig.Connections.url_keycloak_realm}/api-key/{api_key}', json={'access_token': access_token} + ) result = cli_runner.invoke(login, ['--api-key', api_key]) diff --git a/tests/app/services/user_authentication/test_token_manager.py b/tests/app/services/user_authentication/test_token_manager.py index 6846ae20..049a26d5 100644 --- a/tests/app/services/user_authentication/test_token_manager.py +++ b/tests/app/services/user_authentication/test_token_manager.py @@ -33,7 +33,7 @@ def test_refresh_api_key_calls_keycloak_and_stores_access_token_in_config(self, manager = SrvTokenManager() access_token = jwt.encode({}, key='').decode() requests_mock.get( - f'{settings.url_keycloak_realm}/api-key/{manager.config.api_key}', + f'{AppConfig.Connections.url_keycloak_realm}/api-key/{manager.config.api_key}', json={'access_token': access_token}, ) diff --git a/tests/conftest.py b/tests/conftest.py index 8c74fa46..6755b434 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -27,6 +27,7 @@ def mock_settings(monkeypatch, mocker): monkeypatch.setattr(AppConfig.Connections, 'url_download_core', 'http://url_dataset_download_core') monkeypatch.setattr(AppConfig.Connections, 'url_upload_greenroom', 'http://upload_gr') monkeypatch.setattr(AppConfig.Connections, 'url_upload_core', 'http://upload_core') + monkeypatch.setattr(AppConfig.Connections, 'url_keycloak_realm', 'http://url_keycloak_realm') monkeypatch.setattr(UserConfig, 'username', 'test-user') monkeypatch.setattr(UserConfig, 'password', 'test-password') monkeypatch.setattr(UserConfig, 'api_key', 'test-api-key')