From db05538420fd8621a6ef0aa66c1e49563c808d83 Mon Sep 17 00:00:00 2001 From: Daniel Date: Tue, 16 May 2023 14:45:08 -0400 Subject: [PATCH 01/13] PILOT-2587: Condense deprecating github actions into a single, active, and maintained one --- .github/workflows/build-and-publish.yml | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index 7af02d06..8cf1f6ea 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -61,28 +61,21 @@ jobs: - name: Build binary run: poetry run pyinstaller -F --distpath ./app/bundled_app/linux --specpath ./app/build/linux --workpath ./app/build/linux --paths=./.venv/lib/python3.9/site-packages ./app/pilotcli.py -n ${{ github.sha }} + - name: Rename output file + run: mv "./app/bundled_app/linux/${{ github.sha }}" "./app/bundled_app/linux/pilotcli" + - name: Set version in env run: poetry run echo "TAG_VERSION=`poetry version --short`" >> $GITHUB_ENV - name: Create Release id: create_release - uses: actions/create-release@v1 + uses: softprops/action-gh-release@v1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # This token is provided by Actions, you do not need to create your own token with: tag_name: ${{ env.TAG_VERSION }} - release_name: Release ${{ needs.extract-branch-name.outputs.branch }} ${{ env.TAG_VERSION }} + name: Release ${{ needs.extract-branch-name.outputs.branch }} ${{ env.TAG_VERSION }} body: ${{ github.event.head_commit.message }} draft: false prerelease: false - - - name: Upload Release Binary - id: upload-release-binary - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ steps.create_release.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps - asset_path: ./app/bundled_app/linux/${{ github.sha }} - asset_name: pilotcli - asset_content_type: application/octet-stream + files: ./app/bundled_app/linux/pilotcli \ No newline at end of file From 97bf68453a2d7f767daf7e895c8091e665c61eaf Mon Sep 17 00:00:00 2001 From: Daniel Date: Tue, 16 May 2023 14:57:23 -0400 Subject: [PATCH 02/13] fix no newline at end of file issue with the build and publish workflow --- .github/workflows/build-and-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index 8cf1f6ea..7d0a1262 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -78,4 +78,4 @@ jobs: body: ${{ github.event.head_commit.message }} draft: false prerelease: false - files: ./app/bundled_app/linux/pilotcli \ No newline at end of file + files: ./app/bundled_app/linux/pilotcli From b651dd5b6b090ea31599e46d28e73d2e0ac37e33 Mon Sep 17 00:00:00 2001 From: zhiren Date: Mon, 29 May 2023 12:41:32 -0400 Subject: [PATCH 03/13] fixup the dataset download issue. need more test cases --- .../dataset_manager/dataset_download.py | 52 +++++++++++-------- 1 file changed, 30 insertions(+), 22 deletions(-) diff --git a/app/services/dataset_manager/dataset_download.py b/app/services/dataset_manager/dataset_download.py index de36cb1d..979d0c9e 100644 --- a/app/services/dataset_manager/dataset_download.py +++ b/app/services/dataset_manager/dataset_download.py @@ -5,7 +5,6 @@ import datetime import os import time -from urllib.parse import unquote import requests from tqdm import tqdm @@ -69,23 +68,25 @@ def pre_dataset_download(self): except Exception: SrvErrorHandler.default_handle(response.content, True) - def generate_download_url(self): - if self.version: - download_url = AppConfig.Connections.url_dataset_v2download + f'/download/{self.hash_code}' - else: - download_url = AppConfig.Connections.url_download_core + f'v1/download/{self.hash_code}' - headers = { - 'Authorization': 'Bearer ' + self.user.access_token, - } - res = requests.get(download_url, headers=headers) - res_json = res.json() - if self.version: - self.download_url = self.hash_code - default_filename = self.download_url.split('/')[-1].split('?')[0] - self.default_filename = unquote(default_filename) - else: - self.download_url = download_url - self.default_filename = res_json.get('error_msg').split('/')[-1].rstrip('.') + # def generate_download_url(self): + # if self.version: + # download_url = AppConfig.Connections.url_dataset_v2download + f'/download/{self.hash_code}' + # else: + # download_url = AppConfig.Connections.url_download_core + f'v1/download/{self.hash_code}' + # headers = { + # 'Authorization': 'Bearer ' + self.user.access_token, + # } + # print(download_url) + # res = requests.get(download_url, headers=headers) + # res_json = res.json() + # print(res_json) + # if self.version: + # self.download_url = self.hash_code + # default_filename = self.download_url.split('/')[-1].split('?')[0] + # self.default_filename = unquote(default_filename) + # else: + # self.download_url = download_url + # self.default_filename = res_json.get('error_msg').split('/')[-1].rstrip('.') @require_valid_token() def download_status(self) -> EFileStatus: @@ -109,13 +110,15 @@ def check_download_preparing_status(self) -> EFileStatus: @require_valid_token() def send_download_request(self): logger.info('start downloading...') + with requests.get(self.download_url, stream=True, allow_redirects=True) as r: r.raise_for_status() # Since version zip file was created by our system, thus no need to consider filename contain '?' if not self.default_filename: - filename = f'{self.dataset_code}_{self.version}_{str(datetime.datetime.now())}' + filename = f'{self.dataset_code}_{self.version}_{str(datetime.datetime.now())}.zip' else: filename = self.default_filename + output_path = self.avoid_duplicate_file_name(self.output.rstrip('/') + '/' + filename) self.total_size = int(r.headers.get('Content-length')) with open(output_path, 'wb') as file, tqdm( @@ -151,7 +154,12 @@ def avoid_duplicate_file_name(self, filename): def download_dataset(self): pre_result = self.pre_dataset_download() self.hash_code = pre_result.get('result').get('payload').get('hash_code') - self.generate_download_url() + self.download_url = AppConfig.Connections.url_download_core + f'v1/download/{self.hash_code}' + # format the naming for the default filename + self.default_filename = pre_result.get('result').get('target_names')[0] + self.default_filename = self.default_filename.split('/')[-1] + + # wait the download status to be ready status = self.check_download_preparing_status() SrvOutPutHandler.download_status(status) saved_filename = self.send_download_request() @@ -164,8 +172,8 @@ def download_dataset(self): def download_dataset_version(self, version): self.version = version pre_result = self.pre_dataset_version_download() - self.hash_code = pre_result.get('result').get('source') - self.generate_download_url() + self.download_url = pre_result.get('result').get('source') + saved_filename = self.send_download_request() if os.path.isfile(saved_filename): SrvOutPutHandler.download_success(saved_filename) From cf9b86641aac0d0d959f52a1b25f481b5352e627 Mon Sep 17 00:00:00 2001 From: Color Zhan Date: Thu, 1 Jun 2023 09:15:38 -0400 Subject: [PATCH 04/13] merge back/staging release 15052023 (#62) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * update the lineage to use core zone file instead of greenroom * bumpup version * bumpup version to 2.5.0 * fixes versioning * bumpup version * Pilot 2936: add an action to create the cli with mac version (#59) * add new action for create mac binary * add the tag name * add the python and poetry * output the release url in the linux step not in a dedicate step * output the release url in the linux step not in a dedicate step * output the release url in the linux step not in a dedicate step * bumpup version * bumpup version * bumpup version * fixup the upload_url * fixup the upload_url * fixup the upload_url * use correct naming * add back build step * remove the test branch * bumpup version --------- Co-authored-by: zhiren * bumpup to next minor * cleanup the code --------- Co-authored-by: zhiren Co-authored-by: Dušan Andrić --- .github/workflows/build-and-publish.yml | 77 +++++++++++++++++-- app/resources/custom_help.py | 2 +- .../file_upload/upload_validator.py | 2 +- pyproject.toml | 2 +- 4 files changed, 72 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index 7d0a1262..6554e157 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -21,10 +21,13 @@ jobs: id: extract_branch shell: bash run: echo "branch=${GITHUB_REF#refs/heads/}" >> $GITHUB_OUTPUT - push-binary: + + push-binary-linux: needs: [ extract-branch-name ] - if: ${{ needs.extract-branch-name.outputs.branch == 'main' || needs.extract-branch-name.outputs.branch == 'develop' }} + if: ${{ needs.extract-branch-name.outputs.branch == 'main' || needs.extract-branch-name.outputs.branch == 'develop'}} runs-on: ubuntu-20.04 + outputs: + upload_url: ${{steps.create_release.outputs.upload_url}} steps: - name: Checkout repository uses: actions/checkout@v3 @@ -61,21 +64,79 @@ jobs: - name: Build binary run: poetry run pyinstaller -F --distpath ./app/bundled_app/linux --specpath ./app/build/linux --workpath ./app/build/linux --paths=./.venv/lib/python3.9/site-packages ./app/pilotcli.py -n ${{ github.sha }} - - name: Rename output file - run: mv "./app/bundled_app/linux/${{ github.sha }}" "./app/bundled_app/linux/pilotcli" - - name: Set version in env run: poetry run echo "TAG_VERSION=`poetry version --short`" >> $GITHUB_ENV - name: Create Release id: create_release - uses: softprops/action-gh-release@v1 + uses: actions/create-release@v1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # This token is provided by Actions, you do not need to create your own token with: tag_name: ${{ env.TAG_VERSION }} - name: Release ${{ needs.extract-branch-name.outputs.branch }} ${{ env.TAG_VERSION }} + release_name: Release ${{ needs.extract-branch-name.outputs.branch }} ${{ env.TAG_VERSION }} body: ${{ github.event.head_commit.message }} draft: false prerelease: false - files: ./app/bundled_app/linux/pilotcli + + - name: Upload Release Binary + id: upload-release-binary + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps + asset_path: ./app/bundled_app/linux/${{ github.sha }} + asset_name: pilotcli_linux + asset_content_type: application/octet-stream + + push-binary-macos: + needs: [ push-binary-linux ] + if: ${{ needs.extract-branch-name.outputs.branch == 'main' || needs.extract-branch-name.outputs.branch == 'develop'}} + runs-on: macos-12 + steps: + - name: Checkout repository + uses: actions/checkout@v3 + with: + ref: ${{ needs.extract-branch-name.outputs.branch }} + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: 3.9.16 + + - name: Install Poetry + uses: snok/install-poetry@v1 + with: + version: 1.3.2 + virtualenvs-create: true + virtualenvs-in-project: true + installer-parallel: true + + - name: Set up cache + id: cached-poetry-dependencies + uses: actions/cache@v3 + with: + path: .venv + key: venv-${{ hashFiles('**/poetry.lock') }} + + - name: Ensure cache is healthy + if: steps.cached-poetry-dependencies.outputs.cache-hit == 'true' + run: timeout 10s poetry run pip --version || rm -rf .venv + + - name: Install dependencies + run: poetry install --no-interaction --no-root + + - name: Build binary + run: poetry run pyinstaller -F --distpath ./app/bundled_app/macos --specpath ./app/build/macos --workpath ./app/build/macos --paths=./.venv/lib/python3.9/site-packages ./app/pilotcli.py -n ${{ github.sha }} + + - name: Upload Release Binary + id: upload-release-binary + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ needs.push-binary-linux.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps + asset_path: ./app/bundled_app/macos/${{ github.sha }} + asset_name: pilotcli_macos + asset_content_type: application/octet-stream diff --git a/app/resources/custom_help.py b/app/resources/custom_help.py index f3dd9f2c..08f8e69d 100644 --- a/app/resources/custom_help.py +++ b/app/resources/custom_help.py @@ -6,7 +6,7 @@ class HelpPage: page = { 'update': { - 'version': '2.3.0', + 'version': '2.5.0', '1': 'The logic of normal upload and resumble are splited. ' 'add new command for resumable upload as `pilotcli file resume -r manifest.json`', '2': 'The manifest file will be output for both file/folder upload', diff --git a/app/services/file_manager/file_upload/upload_validator.py b/app/services/file_manager/file_upload/upload_validator.py index 33210e12..c7b754ad 100644 --- a/app/services/file_manager/file_upload/upload_validator.py +++ b/app/services/file_manager/file_upload/upload_validator.py @@ -29,7 +29,7 @@ def validate_zone(self): ECustomizedError.INVALID_UPLOAD_REQUEST, True, value='upload-message is required' ) if self.source: - source_file_info = search_item(self.project_code, AppConfig.Env.green_zone.lower(), self.source, 'file') + source_file_info = search_item(self.project_code, AppConfig.Env.core_zone.lower(), self.source, 'file') source_file_info = source_file_info['result'] if not source_file_info: SrvErrorHandler.customized_handle(ECustomizedError.INVALID_SOURCE_FILE, True, value=self.source) diff --git a/pyproject.toml b/pyproject.toml index 9f1aaf98..59763c07 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "app" -version = "2.4.0" +version = "2.5.0a0" description = "This service is designed to support pilot platform" authors = ["Indoc Research"] From 78024494841e1e13f5a53680e46dec5c01a17d81 Mon Sep 17 00:00:00 2001 From: Color Zhan Date: Thu, 1 Jun 2023 09:24:27 -0400 Subject: [PATCH 05/13] Pilot 2999: fixup the dataset download issue (#61) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * update the lineage to use core zone file instead of greenroom * bumpup version * bumpup version to 2.5.0 * fixes versioning * bumpup version * fixup the dataset download issue. need more test cases * add the test case for dataset download * add back requests for steam usage * fixup the old test * resolve the conflict --------- Co-authored-by: zhiren Co-authored-by: Dušan Andrić --- .../dataset_manager/dataset_download.py | 61 +++---- tests/app/commands/test_dataset.py | 7 +- .../dataset_manager/test_dataset_download.py | 158 ++++++++++++++++++ tests/conftest.py | 2 + 4 files changed, 190 insertions(+), 38 deletions(-) diff --git a/app/services/dataset_manager/dataset_download.py b/app/services/dataset_manager/dataset_download.py index de36cb1d..5a60bdd1 100644 --- a/app/services/dataset_manager/dataset_download.py +++ b/app/services/dataset_manager/dataset_download.py @@ -5,7 +5,8 @@ import datetime import os import time -from urllib.parse import unquote +from typing import Any +from typing import Dict import requests from tqdm import tqdm @@ -18,6 +19,7 @@ from app.services.output_manager.error_handler import ECustomizedError from app.services.output_manager.error_handler import SrvErrorHandler from app.services.output_manager.message_handler import SrvOutPutHandler +from app.utils.aggregated import resilient_session from ..user_authentication.decorator import require_valid_token @@ -43,7 +45,7 @@ def pre_dataset_version_download(self): 'Session-ID': self.session_id, } payload = {'version': self.version} - response = requests.get(url, headers=headers, params=payload) + response = resilient_session().get(url, headers=headers, params=payload) res = response.json() code = response.status_code if code == 200: @@ -54,7 +56,7 @@ def pre_dataset_version_download(self): SrvErrorHandler.default_handle(response.content, True) @require_valid_token() - def pre_dataset_download(self): + def pre_dataset_download(self) -> Dict[str, Any]: url = AppConfig.Connections.url_dataset_v2download + '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/download/pre' headers = { 'Authorization': 'Bearer ' + self.user.access_token, @@ -63,36 +65,18 @@ def pre_dataset_download(self): } payload = {'dataset_code': self.dataset_code, 'session_id': self.session_id, 'operator': self.user.username} try: - response = requests.post(url, headers=headers, json=payload) + response = resilient_session().post(url, headers=headers, json=payload) res = response.json() return res - except Exception: - SrvErrorHandler.default_handle(response.content, True) - - def generate_download_url(self): - if self.version: - download_url = AppConfig.Connections.url_dataset_v2download + f'/download/{self.hash_code}' - else: - download_url = AppConfig.Connections.url_download_core + f'v1/download/{self.hash_code}' - headers = { - 'Authorization': 'Bearer ' + self.user.access_token, - } - res = requests.get(download_url, headers=headers) - res_json = res.json() - if self.version: - self.download_url = self.hash_code - default_filename = self.download_url.split('/')[-1].split('?')[0] - self.default_filename = unquote(default_filename) - else: - self.download_url = download_url - self.default_filename = res_json.get('error_msg').split('/')[-1].rstrip('.') + except Exception as e: + SrvErrorHandler.default_handle(f'error when pre dataset download:{e}', True) @require_valid_token() def download_status(self) -> EFileStatus: url = AppConfig.Connections.url_download_core + f'v1/download/status/{self.hash_code}' - res = requests.get(url) - res_json = res.json() - if res_json.get('code') == 200: + res = resilient_session().get(url) + if res.status_code == 200: + res_json = res.json() status = res_json.get('result').get('status') return EFileStatus(status) else: @@ -107,15 +91,17 @@ def check_download_preparing_status(self) -> EFileStatus: return status @require_valid_token() - def send_download_request(self): + def send_download_request(self) -> str: logger.info('start downloading...') + with requests.get(self.download_url, stream=True, allow_redirects=True) as r: r.raise_for_status() # Since version zip file was created by our system, thus no need to consider filename contain '?' if not self.default_filename: - filename = f'{self.dataset_code}_{self.version}_{str(datetime.datetime.now())}' + filename = f'{self.dataset_code}_{self.version}_{str(datetime.datetime.now())}.zip' else: filename = self.default_filename + output_path = self.avoid_duplicate_file_name(self.output.rstrip('/') + '/' + filename) self.total_size = int(r.headers.get('Content-length')) with open(output_path, 'wb') as file, tqdm( @@ -131,7 +117,7 @@ def send_download_request(self): bar.update(size) return output_path - def avoid_duplicate_file_name(self, filename): + def avoid_duplicate_file_name(self, filename) -> str: suffix = 1 original_filename = filename file, ext = os.path.splitext(original_filename) @@ -148,10 +134,15 @@ def avoid_duplicate_file_name(self, filename): return filename @require_valid_token() - def download_dataset(self): + def download_dataset(self) -> None: pre_result = self.pre_dataset_download() self.hash_code = pre_result.get('result').get('payload').get('hash_code') - self.generate_download_url() + self.download_url = AppConfig.Connections.url_download_core + f'v1/download/{self.hash_code}' + # format the naming for the default filename + self.default_filename = pre_result.get('result').get('target_names')[0] + self.default_filename = self.default_filename.split('/')[-1] + + # wait the download status to be ready status = self.check_download_preparing_status() SrvOutPutHandler.download_status(status) saved_filename = self.send_download_request() @@ -161,11 +152,11 @@ def download_dataset(self): SrvErrorHandler.customized_handle(ECustomizedError.DOWNLOAD_FAIL, True) @require_valid_token() - def download_dataset_version(self, version): + def download_dataset_version(self, version) -> None: self.version = version pre_result = self.pre_dataset_version_download() - self.hash_code = pre_result.get('result').get('source') - self.generate_download_url() + self.download_url = pre_result.get('result').get('source') + saved_filename = self.send_download_request() if os.path.isfile(saved_filename): SrvOutPutHandler.download_success(saved_filename) diff --git a/tests/app/commands/test_dataset.py b/tests/app/commands/test_dataset.py index bf043b9c..76e2ac5e 100644 --- a/tests/app/commands/test_dataset.py +++ b/tests/app/commands/test_dataset.py @@ -6,7 +6,7 @@ from app.configs.app_config import AppConfig -def test_download_not_exited_dataset_version(requests_mock, mocker, cli_runner, capsys): +def test_download_not_exited_dataset_version(requests_mock, httpx_mock, mocker, cli_runner, capsys): mocker.patch('app.services.user_authentication.token_manager.SrvTokenManager.check_valid', return_value=0) requests_mock.get( 'http://bff_cli' + '/v1/dataset/testdataset', @@ -39,8 +39,9 @@ def test_download_not_exited_dataset_version(requests_mock, mocker, cli_runner, }, ) - requests_mock.get( - AppConfig.Connections.url_dataset + '/fake-id/download/pre', + httpx_mock.add_response( + method='GET', + url=AppConfig.Connections.url_dataset + '/fake-id/download/pre?version=1.0', json={'error': 'version does not exist'}, status_code=404, ) diff --git a/tests/app/services/dataset_manager/test_dataset_download.py b/tests/app/services/dataset_manager/test_dataset_download.py index ff372835..0afea52e 100644 --- a/tests/app/services/dataset_manager/test_dataset_download.py +++ b/tests/app/services/dataset_manager/test_dataset_download.py @@ -1,3 +1,161 @@ # Copyright (C) 2022-2023 Indoc Research # # Contact Indoc Research for any questions regarding the use of this source code. + +from app.configs.app_config import AppConfig +from app.services.dataset_manager.dataset_download import SrvDatasetDownloadManager +from app.services.dataset_manager.model import EFileStatus + + +def test_dateset_pre_download_success(httpx_mock, mocker): + mocker.patch('app.services.user_authentication.token_manager.SrvTokenManager.check_valid', return_value=0) + + httpx_mock.add_response( + method='POST', + url=AppConfig.Connections.url_dataset_v2download + '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/download/pre', + json={'result': {'payload': {'hash_code': 'hash_code'}, 'target_names': ['test.txt']}}, + status_code=200, + ) + + dataset_download_cliet = SrvDatasetDownloadManager('output_path', 'dataset_code', 'dataset_geid') + + res = dataset_download_cliet.pre_dataset_download() + assert res.get('result').get('payload').get('hash_code') == 'hash_code' + + +def test_dateset_pre_download_status_waiting(httpx_mock, mocker): + dataset_download_cliet = SrvDatasetDownloadManager('output_path', 'dataset_code', 'dataset_geid') + + mocker.patch('app.services.user_authentication.token_manager.SrvTokenManager.check_valid', return_value=0) + + httpx_mock.add_response( + method='GET', + url=AppConfig.Connections.url_download_core + f'v1/download/status/{dataset_download_cliet.hash_code}', + json={'result': {'status': 'WAITING'}}, + status_code=200, + ) + + status = dataset_download_cliet.download_status() + assert status == EFileStatus.WAITING + + +def test_dateset_pre_download_status_success(httpx_mock, mocker): + dataset_download_cliet = SrvDatasetDownloadManager('output_path', 'dataset_code', 'dataset_geid') + + mocker.patch('app.services.user_authentication.token_manager.SrvTokenManager.check_valid', return_value=0) + + httpx_mock.add_response( + method='GET', + url=AppConfig.Connections.url_download_core + f'v1/download/status/{dataset_download_cliet.hash_code}', + json={'result': {'status': 'SUCCEED'}}, + status_code=200, + ) + + status = dataset_download_cliet.download_status() + assert status == EFileStatus.SUCCEED + + +def test_check_dateset_pre_download_status(httpx_mock, mocker): + dataset_download_cliet = SrvDatasetDownloadManager('output_path', 'dataset_code', 'dataset_geid') + + mocker.patch('app.services.user_authentication.token_manager.SrvTokenManager.check_valid', return_value=0) + + httpx_mock.add_response( + method='GET', + url=AppConfig.Connections.url_download_core + f'v1/download/status/{dataset_download_cliet.hash_code}', + json={'result': {'status': 'SUCCEED'}}, + status_code=200, + ) + + status = dataset_download_cliet.check_download_preparing_status() + assert status == EFileStatus.SUCCEED + + +def test_avoid_duplicate_file_name_no_duplicate(mocker): + dataset_download_cliet = SrvDatasetDownloadManager('output_path', 'dataset_code', 'dataset_geid') + + mocker.patch('os.path.isfile', side_effect=[False]) + file_name = dataset_download_cliet.avoid_duplicate_file_name('test.txt') + + assert file_name == 'test.txt' + + +def test_avoid_duplicate_file_name_duplicate_once(mocker): + dataset_download_cliet = SrvDatasetDownloadManager('output_path', 'dataset_code', 'dataset_geid') + + mocker.patch('os.path.isfile', side_effect=[True, False]) + file_name = dataset_download_cliet.avoid_duplicate_file_name('test.txt') + + assert file_name == 'test (1).txt' + + +def test_download_dataset(httpx_mock, mocker): + dataset_download_cliet = SrvDatasetDownloadManager('output_path', 'dataset_code', 'dataset_geid') + + mocker.patch('app.services.user_authentication.token_manager.SrvTokenManager.check_valid', return_value=0) + mocker.patch( + 'app.services.dataset_manager.dataset_download.SrvDatasetDownloadManager.send_download_request', + return_value='test.zip', + ) + mocker.patch('os.path.isfile', return_value=True) + success_msg = mocker.patch( + 'app.services.output_manager.message_handler.SrvOutPutHandler.download_success', return_value=None + ) + + httpx_mock.add_response( + method='POST', + url=AppConfig.Connections.url_dataset_v2download + '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/download/pre', + json={'result': {'payload': {'hash_code': 'hash_code'}, 'target_names': ['test.txt']}}, + status_code=200, + ) + + httpx_mock.add_response( + method='GET', + url=AppConfig.Connections.url_download_core + 'v1/download/status/hash_code', + json={'result': {'status': 'SUCCEED'}}, + status_code=200, + ) + + dataset_download_cliet.download_dataset() + success_msg.assert_called_once_with('test.zip') + + +def test_pre_dataset_version_download(httpx_mock, mocker): + dataset_download_cliet = SrvDatasetDownloadManager('output_path', 'dataset_code', 'dataset_geid') + + mocker.patch('app.services.user_authentication.token_manager.SrvTokenManager.check_valid', return_value=0) + + httpx_mock.add_response( + method='GET', + url=AppConfig.Connections.url_dataset + f'/{dataset_download_cliet.dataset_geid}/download/pre', + json={'result': {'source': 'test.txt'}}, + status_code=200, + ) + + res = dataset_download_cliet.pre_dataset_version_download() + assert res.get('result').get('source') == 'test.txt' + + +def test_download_dataset_version(httpx_mock, mocker): + dataset_download_cliet = SrvDatasetDownloadManager('output_path', 'dataset_code', 'dataset_geid') + + mocker.patch('app.services.user_authentication.token_manager.SrvTokenManager.check_valid', return_value=0) + mocker.patch( + 'app.services.dataset_manager.dataset_download.SrvDatasetDownloadManager.send_download_request', + return_value='test.zip', + ) + mocker.patch('os.path.isfile', return_value=True) + success_msg = mocker.patch( + 'app.services.output_manager.message_handler.SrvOutPutHandler.download_success', return_value=None + ) + + httpx_mock.add_response( + method='GET', + url=AppConfig.Connections.url_dataset + + f'/{dataset_download_cliet.dataset_geid}/download/pre?version=test_version', + json={'result': {'source': 'test.txt'}}, + status_code=200, + ) + + dataset_download_cliet.download_dataset_version('test_version') + success_msg.assert_called_once_with('test.zip') diff --git a/tests/conftest.py b/tests/conftest.py index aaee5cc6..e0f7bb59 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -21,6 +21,8 @@ def mock_settings(monkeypatch): monkeypatch.setattr(AppConfig.Connections, 'url_authn', 'http://service_auth') monkeypatch.setattr(AppConfig.Connections, 'url_bff', 'http://bff_cli') monkeypatch.setattr(AppConfig.Connections, 'url_dataset', 'http://url_dataset') + monkeypatch.setattr(AppConfig.Connections, 'url_dataset_v2download', 'http://url_dataset_download_v2') + monkeypatch.setattr(AppConfig.Connections, 'url_download_core', 'http://url_dataset_download_core') monkeypatch.setattr(AppConfig.Connections, 'url_upload_greenroom', 'http://upload_gr') monkeypatch.setattr(AppConfig.Connections, 'url_upload_core', 'http://upload_core') monkeypatch.setattr(UserConfig, 'username', 'test-user') From 61e2e79da3e91b3af57704cb8b08205aa11e6637 Mon Sep 17 00:00:00 2001 From: Vadym Moshynskyi Date: Fri, 2 Jun 2023 13:27:32 +0200 Subject: [PATCH 06/13] PILOT-2988: Update welcome message (#63) --- app/services/output_manager/message_handler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/services/output_manager/message_handler.py b/app/services/output_manager/message_handler.py index 2ef1dd89..9be7252c 100644 --- a/app/services/output_manager/message_handler.py +++ b/app/services/output_manager/message_handler.py @@ -39,12 +39,12 @@ def validation_login_input_device_error(): @staticmethod def check_login_device_validation(): """Waiting validation finish.""" - return logger.info('Waiting validation finish...') + return logger.info('Waiting for validation to finish...') @staticmethod def login_success(): """Login succeed!""" - return logger.succeed('Welcome to the Command Line Tool!') + return logger.succeed('Welcome to the Pilot Command Line Interface!') @staticmethod def abort_if_false(ctx, param, value): From 2091e134dfdced14c5f9853f03f879772f599907 Mon Sep 17 00:00:00 2001 From: Vadym Moshynskyi Date: Wed, 7 Jun 2023 16:05:30 +0200 Subject: [PATCH 07/13] PILOT-3002: Remove login using username and password (#65) * PILOT-3002: Bump cryptography * PILOT-3002: Remove login using username and password --- app/commands/user.py | 20 ------- app/configs/user_config.py | 3 + .../user_authentication/token_manager.py | 7 +-- .../user_authentication/user_login_logout.py | 32 +---------- poetry.lock | 57 +++++++++---------- pyproject.toml | 2 +- .../test_user_login_logout.py | 53 +---------------- 7 files changed, 35 insertions(+), 139 deletions(-) diff --git a/app/commands/user.py b/app/commands/user.py index 6cf37c36..7f068383 100644 --- a/app/commands/user.py +++ b/app/commands/user.py @@ -7,8 +7,6 @@ import app.services.output_manager.help_page as user_help import app.services.output_manager.message_handler as mhandler from app.services.user_authentication.decorator import require_login_session - -# from app.services.user_authentication.user_login_logout import user_login from app.services.user_authentication.user_login_logout import user_device_id_login from app.services.user_authentication.user_login_logout import user_logout from app.services.user_authentication.user_login_logout import validate_user_device_login @@ -21,24 +19,6 @@ def cli(): pass -# COMMENT user password login temporally. -# @click.command() -# @click.option( -# '-U', '--username', prompt='Username', help=(user_help.user_help_page(user_help.UserHELP.USER_LOGIN_USERNAME)) -# ) -# @click.option( -# '-P', -# '--password', -# prompt='Password', -# help=(user_help.user_help_page(user_help.UserHELP.USER_LOGIN_PASSWORD)), -# hide_input=True, -# ) -# @doc(user_help.user_help_page(user_help.UserHELP.USER_LOGIN)) -# def login(username, password): -# user_login(username, password) -# mhandler.SrvOutPutHandler.login_success() - - @click.command() @doc(user_help.user_help_page(user_help.UserHELP.USER_LOGIN)) def login(): diff --git a/app/configs/user_config.py b/app/configs/user_config.py index 50080444..718f0704 100644 --- a/app/configs/user_config.py +++ b/app/configs/user_config.py @@ -58,6 +58,9 @@ def clear(self): } self.save() + def is_logged_in(self) -> bool: + return bool(self.access_token and self.refresh_token) + @property def username(self): return decryption(self.config['USER']['username'], self.secret) diff --git a/app/services/user_authentication/token_manager.py b/app/services/user_authentication/token_manager.py index da881065..b0029433 100644 --- a/app/services/user_authentication/token_manager.py +++ b/app/services/user_authentication/token_manager.py @@ -16,13 +16,10 @@ class SrvTokenManager(metaclass=MetaService): def __init__(self): user_config = UserConfig() - has_user = user_config.config.has_section('USER') - has_access_token = user_config.config.has_option('USER', 'access_token') - has_refresh_token = user_config.config.has_option('USER', 'refresh_token') - if has_user and has_access_token and has_refresh_token: + if user_config.is_logged_in(): self.config = user_config else: - raise (Exception('Login session not found, please login first.')) + raise Exception('Login session not found, please login first.') def update_token(self, access_token, refresh_token): self.config.access_token = access_token diff --git a/app/services/user_authentication/user_login_logout.py b/app/services/user_authentication/user_login_logout.py index 891f4ab7..0d28f106 100644 --- a/app/services/user_authentication/user_login_logout.py +++ b/app/services/user_authentication/user_login_logout.py @@ -17,33 +17,6 @@ from app.services.output_manager.message_handler import SrvOutPutHandler -def user_login(username, password): - url = AppConfig.Connections.url_authn - user_config = UserConfig() - request_body = {'username': username, 'password': password} - headers = {'Content-Type': 'application/json'} - response = requests.post(url, json=request_body, headers=headers) - if response.status_code == 200: - res_to_dict = response.json() - user_config.username = username - user_config.password = password - user_config.access_token = res_to_dict['result']['access_token'] - user_config.refresh_token = res_to_dict['result']['refresh_token'] - user_config.last_active = str(int(time.time())) - user_config.hpc_token = '' - user_config.session_id = 'cli-' + str(uuid4()) - user_config.save() - elif response.status_code == 401: - res_to_dict = [] - SrvErrorHandler.customized_handle(ECustomizedError.INVALID_CREDENTIALS, True) - else: - if response.text: - SrvErrorHandler.default_handle(response.text, True) - res_to_dict = response.json() - SrvErrorHandler.default_handle(response.content, True) - return res_to_dict - - def user_device_id_login() -> Dict[str, Any]: """Get device code URL for user login.""" @@ -104,10 +77,7 @@ def validate_user_device_login(device_code: str, expires: int, interval: int) -> def check_is_login(if_print: bool = True) -> bool: user_config = UserConfig() - has_username = user_config.config.has_option('USER', 'username') - has_access_token = user_config.config.has_option('USER', 'access_token') - has_refresh_token = user_config.config.has_option('USER', 'refresh_token') - if has_username and has_access_token and has_refresh_token and user_config.username != '': + if user_config.is_logged_in(): return True else: SrvErrorHandler.customized_handle(ECustomizedError.LOGIN_SESSION_INVALID, if_print) if if_print else None diff --git a/poetry.lock b/poetry.lock index 8fef1458..cd690ee6 100644 --- a/poetry.lock +++ b/poetry.lock @@ -541,46 +541,43 @@ toml = ["tomli"] [[package]] name = "cryptography" -version = "3.1.1" +version = "3.4.8" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." category = "main" optional = false -python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*" -files = [ - {file = "cryptography-3.1.1-cp27-cp27m-macosx_10_10_x86_64.whl", hash = "sha256:65beb15e7f9c16e15934569d29fb4def74ea1469d8781f6b3507ab896d6d8719"}, - {file = "cryptography-3.1.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:983c0c3de4cb9fcba68fd3f45ed846eb86a2a8b8d8bc5bb18364c4d00b3c61fe"}, - {file = "cryptography-3.1.1-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:e97a3b627e3cb63c415a16245d6cef2139cca18bb1183d1b9375a1c14e83f3b3"}, - {file = "cryptography-3.1.1-cp27-cp27m-win32.whl", hash = "sha256:cb179acdd4ae1e4a5a160d80b87841b3d0e0be84af46c7bb2cd7ece57a39c4ba"}, - {file = "cryptography-3.1.1-cp27-cp27m-win_amd64.whl", hash = "sha256:b372026ebf32fe2523159f27d9f0e9f485092e43b00a5adacf732192a70ba118"}, - {file = "cryptography-3.1.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:680da076cad81cdf5ffcac50c477b6790be81768d30f9da9e01960c4b18a66db"}, - {file = "cryptography-3.1.1-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:5d52c72449bb02dd45a773a203196e6d4fae34e158769c896012401f33064396"}, - {file = "cryptography-3.1.1-cp35-abi3-macosx_10_10_x86_64.whl", hash = "sha256:f0e099fc4cc697450c3dd4031791559692dd941a95254cb9aeded66a7aa8b9bc"}, - {file = "cryptography-3.1.1-cp35-abi3-manylinux1_x86_64.whl", hash = "sha256:a7597ffc67987b37b12e09c029bd1dc43965f75d328076ae85721b84046e9ca7"}, - {file = "cryptography-3.1.1-cp35-abi3-manylinux2010_x86_64.whl", hash = "sha256:4549b137d8cbe3c2eadfa56c0c858b78acbeff956bd461e40000b2164d9167c6"}, - {file = "cryptography-3.1.1-cp35-abi3-manylinux2014_aarch64.whl", hash = "sha256:89aceb31cd5f9fc2449fe8cf3810797ca52b65f1489002d58fe190bfb265c536"}, - {file = "cryptography-3.1.1-cp35-cp35m-win32.whl", hash = "sha256:559d622aef2a2dff98a892eef321433ba5bc55b2485220a8ca289c1ecc2bd54f"}, - {file = "cryptography-3.1.1-cp35-cp35m-win_amd64.whl", hash = "sha256:451cdf60be4dafb6a3b78802006a020e6cd709c22d240f94f7a0696240a17154"}, - {file = "cryptography-3.1.1-cp36-abi3-win32.whl", hash = "sha256:762bc5a0df03c51ee3f09c621e1cee64e3a079a2b5020de82f1613873d79ee70"}, - {file = "cryptography-3.1.1-cp36-abi3-win_amd64.whl", hash = "sha256:b12e715c10a13ca1bd27fbceed9adc8c5ff640f8e1f7ea76416352de703523c8"}, - {file = "cryptography-3.1.1-cp36-cp36m-win32.whl", hash = "sha256:21b47c59fcb1c36f1113f3709d37935368e34815ea1d7073862e92f810dc7499"}, - {file = "cryptography-3.1.1-cp36-cp36m-win_amd64.whl", hash = "sha256:48ee615a779ffa749d7d50c291761dc921d93d7cf203dca2db663b4f193f0e49"}, - {file = "cryptography-3.1.1-cp37-cp37m-win32.whl", hash = "sha256:b2bded09c578d19e08bd2c5bb8fed7f103e089752c9cf7ca7ca7de522326e921"}, - {file = "cryptography-3.1.1-cp37-cp37m-win_amd64.whl", hash = "sha256:f99317a0fa2e49917689b8cf977510addcfaaab769b3f899b9c481bbd76730c2"}, - {file = "cryptography-3.1.1-cp38-cp38-win32.whl", hash = "sha256:ab010e461bb6b444eaf7f8c813bb716be2d78ab786103f9608ffd37a4bd7d490"}, - {file = "cryptography-3.1.1-cp38-cp38-win_amd64.whl", hash = "sha256:99d4984aabd4c7182050bca76176ce2dbc9fa9748afe583a7865c12954d714ba"}, - {file = "cryptography-3.1.1.tar.gz", hash = "sha256:9d9fc6a16357965d282dd4ab6531013935425d0dc4950df2e0cf2a1b1ac1017d"}, +python-versions = ">=3.6" +files = [ + {file = "cryptography-3.4.8-cp36-abi3-macosx_10_10_x86_64.whl", hash = "sha256:a00cf305f07b26c351d8d4e1af84ad7501eca8a342dedf24a7acb0e7b7406e14"}, + {file = "cryptography-3.4.8-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:f44d141b8c4ea5eb4dbc9b3ad992d45580c1d22bf5e24363f2fbf50c2d7ae8a7"}, + {file = "cryptography-3.4.8-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:0a7dcbcd3f1913f664aca35d47c1331fce738d44ec34b7be8b9d332151b0b01e"}, + {file = "cryptography-3.4.8-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:34dae04a0dce5730d8eb7894eab617d8a70d0c97da76b905de9efb7128ad7085"}, + {file = "cryptography-3.4.8-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1eb7bb0df6f6f583dd8e054689def236255161ebbcf62b226454ab9ec663746b"}, + {file = "cryptography-3.4.8-cp36-abi3-manylinux_2_24_x86_64.whl", hash = "sha256:9965c46c674ba8cc572bc09a03f4c649292ee73e1b683adb1ce81e82e9a6a0fb"}, + {file = "cryptography-3.4.8-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:3c4129fc3fdc0fa8e40861b5ac0c673315b3c902bbdc05fc176764815b43dd1d"}, + {file = "cryptography-3.4.8-cp36-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:695104a9223a7239d155d7627ad912953b540929ef97ae0c34c7b8bf30857e89"}, + {file = "cryptography-3.4.8-cp36-abi3-win32.whl", hash = "sha256:21ca464b3a4b8d8e86ba0ee5045e103a1fcfac3b39319727bc0fc58c09c6aff7"}, + {file = "cryptography-3.4.8-cp36-abi3-win_amd64.whl", hash = "sha256:3520667fda779eb788ea00080124875be18f2d8f0848ec00733c0ec3bb8219fc"}, + {file = "cryptography-3.4.8-pp36-pypy36_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:d2a6e5ef66503da51d2110edf6c403dc6b494cc0082f85db12f54e9c5d4c3ec5"}, + {file = "cryptography-3.4.8-pp36-pypy36_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a305600e7a6b7b855cd798e00278161b681ad6e9b7eca94c721d5f588ab212af"}, + {file = "cryptography-3.4.8-pp36-pypy36_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:3fa3a7ccf96e826affdf1a0a9432be74dc73423125c8f96a909e3835a5ef194a"}, + {file = "cryptography-3.4.8-pp37-pypy37_pp73-macosx_10_10_x86_64.whl", hash = "sha256:d9ec0e67a14f9d1d48dd87a2531009a9b251c02ea42851c060b25c782516ff06"}, + {file = "cryptography-3.4.8-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:5b0fbfae7ff7febdb74b574055c7466da334a5371f253732d7e2e7525d570498"}, + {file = "cryptography-3.4.8-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:94fff993ee9bc1b2440d3b7243d488c6a3d9724cc2b09cdb297f6a886d040ef7"}, + {file = "cryptography-3.4.8-pp37-pypy37_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:8695456444f277af73a4877db9fc979849cd3ee74c198d04fc0776ebc3db52b9"}, + {file = "cryptography-3.4.8-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:cd65b60cfe004790c795cc35f272e41a3df4631e2fb6b35aa7ac6ef2859d554e"}, + {file = "cryptography-3.4.8.tar.gz", hash = "sha256:94cc5ed4ceaefcbe5bf38c8fba6a21fc1d365bb8fb826ea1688e3370b2e24a1c"}, ] [package.dependencies] -cffi = ">=1.8,<1.11.3 || >1.11.3" -six = ">=1.4.1" +cffi = ">=1.12" [package.extras] -docs = ["sphinx (>=1.6.5,!=1.8.0,!=3.1.0,!=3.1.1)", "sphinx_rtd_theme"] +docs = ["sphinx (>=1.6.5,!=1.8.0,!=3.1.0,!=3.1.1)", "sphinx-rtd-theme"] docstest = ["doc8", "pyenchant (>=1.6.11)", "sphinxcontrib-spelling (>=4.0.1)", "twine (>=1.12.0)"] pep8test = ["black", "flake8", "flake8-import-order", "pep8-naming"] +sdist = ["setuptools-rust (>=0.11.4)"] ssh = ["bcrypt (>=3.1.5)"] -test = ["hypothesis (>=1.11.4,!=3.79.2)", "iso8601", "pretend", "pytest (>=3.6.0,!=3.9.0,!=3.9.1,!=3.9.2)", "pytz"] +test = ["hypothesis (>=1.11.4,!=3.79.2)", "iso8601", "pretend", "pytest (>=6.0)", "pytest-cov", "pytest-subtests", "pytest-xdist", "pytz"] [[package]] name = "distlib" @@ -1897,4 +1894,4 @@ testing = ["flake8 (<5)", "func-timeout", "jaraco.functools", "jaraco.itertools" [metadata] lock-version = "2.0" python-versions = ">=3.7,<3.11" -content-hash = "7b2376e5fb3097faae5241e72a82dbc1c68b6bf2b63da6698b2fe0eb1dde4dc1" +content-hash = "1ac69682539aa9934b4fd61e9f6e0eda8d4ac96b389f51ea3f8f98189f423e76" diff --git a/pyproject.toml b/pyproject.toml index 59763c07..1c2037b0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ click = "7.1.2" PyJWT = "1.7.1" questionary = "^1.10.0" tqdm = "4.56.0" -cryptography = "3.1.1" +cryptography = "3.4.8" urllib3 = "^1.26.14" requests = "^2.28.2" pydantic = "^1.9.2" diff --git a/tests/app/services/user_authentication/test_user_login_logout.py b/tests/app/services/user_authentication/test_user_login_logout.py index bd3d3d35..33fd5cec 100644 --- a/tests/app/services/user_authentication/test_user_login_logout.py +++ b/tests/app/services/user_authentication/test_user_login_logout.py @@ -6,64 +6,13 @@ from app.configs.app_config import AppConfig from app.configs.user_config import UserConfig -from app.resources.custom_error import Error -from app.services.output_manager.error_handler import ECustomizedError from app.services.user_authentication.user_login_logout import check_is_login from app.services.user_authentication.user_login_logout import user_device_id_login -from app.services.user_authentication.user_login_logout import user_login from app.services.user_authentication.user_login_logout import validate_user_device_login -def test_user_login_success(requests_mock): - requests_mock.post( - 'http://service_auth', - json={ - 'code': 200, - 'error_msg': '', - 'page': 0, - 'total': 1, - 'num_of_pages': 1, - 'result': { - 'access_token': 'fake-token', - 'expires_in': 300, - 'refresh_expires_in': 360, - 'refresh_token': 'refresh-token', - 'token_type': 'Bearer', - 'not-before-policy': 0, - 'session_state': 'session-state', - 'scope': 'roles groups profile email', - }, - }, - ) - res = user_login('username', 'password') - assert res.get('code') == 200 - assert res['result'].get('access_token') == 'fake-token' - assert res['result'].get('refresh_token') == 'refresh-token' - assert res.get('error_msg') == '' - - -def test_user_login_wrong_password(requests_mock, capsys): - requests_mock.post( - 'http://service_auth', - json={ - 'code': 401, - 'error_msg': "401: b'{\"error\":\"invalid_grant\",\"error_description\":\"Invalid user credentials\"}'", - 'page': 0, - 'total': 1, - 'num_of_pages': 1, - 'result': [], - }, - status_code=401, - ) - with pytest.raises(SystemExit): - user_login('username', 'password') - out, err = capsys.readouterr() - assert out == Error.error_msg.get(ECustomizedError.INVALID_CREDENTIALS.name, 'Unknown error.') + '\n' - assert err == '' - - def test_check_is_not_login(mocker): - mocker.patch('configparser.ConfigParser.has_option', return_value=False) + mocker.patch('app.configs.user_config.UserConfig.is_logged_in', return_value=False) expected_result = False with pytest.raises(SystemExit): actual = check_is_login() From 561bb839ca00847e0f48482846adced4d5266acb Mon Sep 17 00:00:00 2001 From: Vadym Moshynskyi Date: Wed, 7 Jun 2023 22:08:54 +0200 Subject: [PATCH 08/13] PILOT-3002: Add support for API Key (#66) * PILOT-3002: Add support for API Key * PILOT-3002: Resolve issues with tests * PILOT-3002: Update total coverage threshold --- .editorconfig | 18 ++++++ .github/workflows/run-tests.yml | 2 +- app/commands/user.py | 44 +++++++++---- app/configs/app_config.py | 8 ++- app/configs/user_config.py | 15 ++++- app/models/enums.py | 12 ++++ app/resources/custom_help.py | 1 + app/services/output_manager/help_page.py | 1 + .../output_manager/message_handler.py | 13 ++++ .../user_authentication/token_manager.py | 43 ++++++++++--- .../user_authentication/user_login_logout.py | 39 +++++++++++ env.py | 2 + poetry.lock | 20 +++++- pyproject.toml | 1 + tests/app/commands/test_user.py | 64 +++++++++++++++++++ .../services/hpc_manager/test_hpc_cluster.py | 8 ++- .../user_authentication/test_token_manager.py | 41 ++++++++++++ tests/conftest.py | 10 ++- tests/fixtures/__init__.py | 3 + tests/fixtures/fake.py | 15 +++++ 20 files changed, 325 insertions(+), 35 deletions(-) create mode 100644 .editorconfig create mode 100644 app/models/enums.py create mode 100644 tests/app/commands/test_user.py create mode 100644 tests/app/services/user_authentication/test_token_manager.py create mode 100644 tests/fixtures/__init__.py create mode 100644 tests/fixtures/fake.py diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 00000000..f3515fd3 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,18 @@ +root = true + +[*] +charset = utf-8 +tab_width = 4 +indent_style = space + +[*.py] +end_of_line = lf +indent_size = 4 +trim_trailing_whitespace = true +insert_final_newline = true + +[*.{yml,yaml}] +end_of_line = lf +indent_size = 2 +trim_trailing_whitespace = true +insert_final_newline = true diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 44a0542e..82399a9a 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -47,7 +47,7 @@ jobs: uses: pre-commit/action@v3.0.0 - name: Run tests - run: poetry run pytest -vvv --exitfirst --cov=app --cov-report=term --cov-report=xml --cov-fail-under=23 + run: poetry run pytest -vvv --exitfirst --cov=app --cov-report=term --cov-report=xml --cov-fail-under=54 - name: Coverage report comment uses: mishakav/pytest-coverage-comment@v1.1.42 diff --git a/app/commands/user.py b/app/commands/user.py index 7f068383..d7cfca56 100644 --- a/app/commands/user.py +++ b/app/commands/user.py @@ -2,11 +2,15 @@ # # Contact Indoc Research for any questions regarding the use of this source code. +from typing import Union + import click import app.services.output_manager.help_page as user_help import app.services.output_manager.message_handler as mhandler +from app.models.enums import LoginMethod from app.services.user_authentication.decorator import require_login_session +from app.services.user_authentication.user_login_logout import login_using_api_key from app.services.user_authentication.user_login_logout import user_device_id_login from app.services.user_authentication.user_login_logout import user_logout from app.services.user_authentication.user_login_logout import validate_user_device_login @@ -20,22 +24,36 @@ def cli(): @click.command() +@click.option( + '--api-key', + envvar='PILOT_API_KEY', + help=(user_help.user_help_page(user_help.UserHELP.USER_LOGIN_API_KEY)), +) @doc(user_help.user_help_page(user_help.UserHELP.USER_LOGIN)) -def login(): - device_login = user_device_id_login() - if device_login: - mhandler.SrvOutPutHandler.login_input_device_code(device_login['verification_uri_complete']) - mhandler.SrvOutPutHandler.login_device_code_qrcode(device_login['verification_uri_complete']) +def login(api_key: Union[str, None]): + if api_key: + mhandler.SrvOutPutHandler.login_using_method(LoginMethod.API_KEY) + is_valid = login_using_api_key(api_key) + if is_valid: + mhandler.SrvOutPutHandler.login_success() + else: + mhandler.SrvOutPutHandler.login_using_api_key_failed_error() else: - mhandler.SrvOutPutHandler.login_input_device_error() + mhandler.SrvOutPutHandler.login_using_method(LoginMethod.DEVICE_CODE) + device_login = user_device_id_login() + if device_login: + mhandler.SrvOutPutHandler.login_input_device_code(device_login['verification_uri_complete']) + mhandler.SrvOutPutHandler.login_device_code_qrcode(device_login['verification_uri_complete']) + else: + mhandler.SrvOutPutHandler.login_input_device_error() - is_validated = validate_user_device_login( - device_login['device_code'], device_login['expires'], device_login['interval'] - ) - if is_validated: - mhandler.SrvOutPutHandler.login_success() - else: - mhandler.SrvOutPutHandler.validation_login_input_device_error() + is_validated = validate_user_device_login( + device_login['device_code'], device_login['expires'], device_login['interval'] + ) + if is_validated: + mhandler.SrvOutPutHandler.login_success() + else: + mhandler.SrvOutPutHandler.validation_login_input_device_error() @click.command() diff --git a/app/configs/app_config.py b/app/configs/app_config.py index e2ba4b12..964c7f8c 100644 --- a/app/configs/app_config.py +++ b/app/configs/app_config.py @@ -5,8 +5,8 @@ from env import ConfigClass -class AppConfig(object): - class Env(object): +class AppConfig: + class Env: section = 'environment' project = ConfigClass.project user_config_path = ConfigClass.config_path @@ -34,8 +34,9 @@ class Env(object): greenroom_bucket_prefix = 'gr' keycloak_device_client_id = ConfigClass.keycloak_device_client_id + keycloak_api_key_audience = ConfigClass.keycloak_api_key_audience - class Connections(object): + class Connections: section = 'connections' url_harbor = ConfigClass.url_harbor url_authn = ConfigClass.url_authn @@ -53,6 +54,7 @@ class Connections(object): url_validation = ConfigClass.url_validation url_keycloak = ConfigClass.url_keycloak url_keycloak_token = f'{ConfigClass.url_keycloak}/token' + url_keycloak_realm = ConfigClass.url_keycloak.rstrip('/').replace('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/protocol/openid-connect', '') url_bff = ConfigClass.url_bff # add url_base to check if value exist url_base = ConfigClass.base_url diff --git a/app/configs/user_config.py b/app/configs/user_config.py index 718f0704..6ecd0d6b 100644 --- a/app/configs/user_config.py +++ b/app/configs/user_config.py @@ -6,13 +6,12 @@ import os import time +from app.configs.app_config import AppConfig from app.models.singleton import Singleton from app.services.crypto.crypto import decryption from app.services.crypto.crypto import encryption from app.services.crypto.crypto import generate_secret -from .app_config import AppConfig - class UserConfig(metaclass=Singleton): """The class to maintain the user access/fresh token Note here: the base class is Singleton, meaning no matter how @@ -32,6 +31,7 @@ def __init__(self): self.config['USER'] = { 'username': '', 'password': '', + 'api_key': '', 'access_token': '', 'refresh_token': '', 'secret': generate_secret(), @@ -49,6 +49,7 @@ def clear(self): self.config['USER'] = { 'username': '', 'password': '', + 'api_key': '', 'access_token': '', 'refresh_token': '', 'hpc_token': '', @@ -59,7 +60,7 @@ def clear(self): self.save() def is_logged_in(self) -> bool: - return bool(self.access_token and self.refresh_token) + return bool(self.api_key or (self.access_token and self.refresh_token)) @property def username(self): @@ -77,6 +78,14 @@ def password(self): def password(self, val): self.config['USER']['password'] = encryption(val, self.secret) + @property + def api_key(self): + return decryption(self.config['USER']['api_key'], self.secret) + + @api_key.setter + def api_key(self, val): + self.config['USER']['api_key'] = encryption(val, self.secret) + @property def access_token(self): return decryption(self.config['USER']['access_token'], self.secret) diff --git a/app/models/enums.py b/app/models/enums.py new file mode 100644 index 00000000..ee51e7ba --- /dev/null +++ b/app/models/enums.py @@ -0,0 +1,12 @@ +# Copyright (C) 2023 Indoc Research +# +# Contact Indoc Research for any questions regarding the use of this source code. + +from enum import Enum + + +class LoginMethod(str, Enum): + """Available login methods.""" + + API_KEY = 'api-key' + DEVICE_CODE = 'device-code' diff --git a/app/resources/custom_help.py b/app/resources/custom_help.py index 08f8e69d..5906ad1b 100644 --- a/app/resources/custom_help.py +++ b/app/resources/custom_help.py @@ -27,6 +27,7 @@ class HelpPage: ), 'USER_LOGIN_USERNAME': 'Specify username for login.', 'USER_LOGIN_PASSWORD': 'Specify password for login.', + 'USER_LOGIN_API_KEY': 'Specify API Key for login.', }, 'file': { 'FILE_ATTRIBUTE_LIST': 'List attribute templates of a given Project.', diff --git a/app/services/output_manager/help_page.py b/app/services/output_manager/help_page.py index 177e1753..68102f70 100644 --- a/app/services/output_manager/help_page.py +++ b/app/services/output_manager/help_page.py @@ -53,6 +53,7 @@ class UserHELP(enum.Enum): USER_LOGOUT_CONFIRM = 'USER_LOGOUT_CONFIRM' USER_LOGIN_USERNAME = 'USER_LOGIN_USERNAME' USER_LOGIN_PASSWORD = 'USER_LOGIN_PASSWORD' + USER_LOGIN_API_KEY = 'USER_LOGIN_API_KEY' def user_help_page(UserHELP: UserHELP): diff --git a/app/services/output_manager/message_handler.py b/app/services/output_manager/message_handler.py index 9be7252c..833d8033 100644 --- a/app/services/output_manager/message_handler.py +++ b/app/services/output_manager/message_handler.py @@ -7,10 +7,23 @@ import qrcode import app.services.logger_services.log_functions as logger +from app.models.enums import LoginMethod from app.models.service_meta_class import MetaService class SrvOutPutHandler(metaclass=MetaService): + @staticmethod + def login_using_method(method: LoginMethod): + """Selected login method message.""" + return logger.info(f'Trying to log in using "{method.value}" method.') + + @staticmethod + def login_using_api_key_failed_error(): + """Error when logging in with the API Key!""" + return logger.error( + f'Failed to log in using "{LoginMethod.API_KEY.value}" method, please make sure you are using a valid key!' + ) + @staticmethod def login_device_code_qrcode(url: str): """Print QRCode with login url!""" diff --git a/app/services/user_authentication/token_manager.py b/app/services/user_authentication/token_manager.py index b0029433..90283f9b 100644 --- a/app/services/user_authentication/token_manager.py +++ b/app/services/user_authentication/token_manager.py @@ -9,8 +9,10 @@ from app.configs.app_config import AppConfig from app.configs.user_config import UserConfig +from app.models.enums import LoginMethod from app.models.service_meta_class import MetaService from app.services.output_manager.error_handler import SrvErrorHandler +from app.services.user_authentication.user_login_logout import exchange_api_key class SrvTokenManager(metaclass=MetaService): @@ -37,6 +39,13 @@ def decode_refresh_token(self): tokens = self.get_token() return jwt.decode(tokens[1], verify=False) + def is_api_key(self) -> bool: + token = self.decode_access_token() + audience = token['aud'] + if isinstance(audience, str): + audience = [audience] + return AppConfig.Env.keycloak_api_key_audience.issubset(set(audience)) + def check_valid(self, required_azp): """ check token validation @@ -49,20 +58,26 @@ def check_valid(self, required_azp): now = time.time() diff = expiry_at - now - # TODO: check why here will need enforce the token refresh when - # azp is not `kong`` - # ``kong`` is hardcoded in the decorator definition as default value. - azp_token_condition = decoded_access_token['azp'] not in [required_azp, AppConfig.Env.keycloak_device_client_id] + if not self.is_api_key(): + # TODO: check why here will need enforce the token refresh when + # azp is not `kong`` + # ``kong`` is hardcoded in the decorator definition as default value. + azp_token_condition = decoded_access_token['azp'] not in [ + required_azp, + AppConfig.Env.keycloak_device_client_id, + ] + + if azp_token_condition or expiry_at <= now: + return 2 - if azp_token_condition or expiry_at <= now: - return 2 - # print(expiry_at, now) - # print(diff, AppConfig.Env.token_warn_need_refresh) if diff <= AppConfig.Env.token_warn_need_refresh: return 1 return 0 - def refresh(self, azp: str): + def refresh(self, azp: str) -> None: + if self.is_api_key(): + return self.refresh_api_key() + url = AppConfig.Connections.url_keycloak_token payload = { 'grant_type': 'refresh_token', @@ -79,4 +94,12 @@ def refresh(self, azp: str): self.update_token(response.json()['access_token'], response.json()['refresh_token']) else: SrvErrorHandler.default_handle(response.content) - return response.json() + + def refresh_api_key(self) -> None: + access_token = exchange_api_key(self.config.api_key) + if access_token is None: + return SrvErrorHandler.default_handle( + f'Unable to get access token using "{LoginMethod.API_KEY.value}" method. Unable to proceed.', True + ) + + self.update_token(access_token, '') diff --git a/app/services/user_authentication/user_login_logout.py b/app/services/user_authentication/user_login_logout.py index 0d28f106..b6e8882f 100644 --- a/app/services/user_authentication/user_login_logout.py +++ b/app/services/user_authentication/user_login_logout.py @@ -5,10 +5,12 @@ import time from typing import Any from typing import Dict +from typing import Union from uuid import uuid4 import jwt import requests +from requests import RequestException from app.configs.app_config import AppConfig from app.configs.user_config import UserConfig @@ -17,6 +19,42 @@ from app.services.output_manager.message_handler import SrvOutPutHandler +def exchange_api_key(api_key: str) -> Union[str, None]: + """Exchange API Key with JWT token using Keycloak.""" + + url = f'{AppConfig.Connections.url_keycloak_realm}/api-key/{api_key}' + try: + response = requests.get(url, timeout=5) + response.raise_for_status() + except RequestException: + return None + + return response.json()['access_token'] + + +def login_using_api_key(api_key: str) -> bool: + """Try to log in using API Key and store results in user config.""" + + access_token = exchange_api_key(api_key) + if access_token is None: + return False + + decoded_token = jwt.decode(access_token, verify=False) + username = decoded_token['preferred_username'] + + user_config = UserConfig() + user_config.api_key = api_key + user_config.access_token = access_token + user_config.refresh_token = '' + user_config.username = username + user_config.last_active = str(int(time.time())) + user_config.hpc_token = '' + user_config.session_id = 'cli-' + str(uuid4()) + user_config.save() + + return True + + def user_device_id_login() -> Dict[str, Any]: """Get device code URL for user login.""" @@ -64,6 +102,7 @@ def validate_user_device_login(device_code: str, expires: int, interval: int) -> resp_dict = resp.json() decode_token = jwt.decode(resp_dict['access_token'], verify=False) user_config = UserConfig() + user_config.api_key = '' user_config.access_token = resp_dict['access_token'] user_config.refresh_token = resp_dict['refresh_token'] user_config.username = decode_token['preferred_username'] diff --git a/env.py b/env.py index ae8cd7ae..90f67c74 100644 --- a/env.py +++ b/env.py @@ -3,6 +3,7 @@ # Contact Indoc Research for any questions regarding the use of this source code. import os +from typing import Set from dotenv import load_dotenv from pydantic import BaseSettings @@ -25,6 +26,7 @@ class Settings(BaseSettings): url_keycloak: str = '' keycloak_device_client_id: str = 'cli_test2' + keycloak_api_key_audience: Set[str] = {'api-key'} VM_INFO: str = '' diff --git a/poetry.lock b/poetry.lock index cd690ee6..6c6eb6a7 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry and should not be changed by hand. +# This file is automatically @generated by Poetry 1.4.2 and should not be changed by hand. [[package]] name = "aioboto3" @@ -591,6 +591,22 @@ files = [ {file = "distlib-0.3.6.tar.gz", hash = "sha256:14bad2d9b04d3a36127ac97f30b12a19268f211063d8f8ee4f47108896e11b46"}, ] +[[package]] +name = "faker" +version = "18.9.0" +description = "Faker is a Python package that generates fake data for you." +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "Faker-18.9.0-py3-none-any.whl", hash = "sha256:defe9ed618a67ebf0f3eb1895e198c2355a7128a09087a6dce342ef2253263ea"}, + {file = "Faker-18.9.0.tar.gz", hash = "sha256:80a5ea1464556c06b98bf47ea3adc7f33811a1182518d847860b1874080bd3c9"}, +] + +[package.dependencies] +python-dateutil = ">=2.4" +typing-extensions = {version = ">=3.10.0.1", markers = "python_version < \"3.8\""} + [[package]] name = "filelock" version = "3.9.0" @@ -1894,4 +1910,4 @@ testing = ["flake8 (<5)", "func-timeout", "jaraco.functools", "jaraco.itertools" [metadata] lock-version = "2.0" python-versions = ">=3.7,<3.11" -content-hash = "1ac69682539aa9934b4fd61e9f6e0eda8d4ac96b389f51ea3f8f98189f423e76" +content-hash = "893dd116283c6b9fd4d470d5aea6cb8dbe5c8d0194172f6db67f6639b50d85d9" diff --git a/pyproject.toml b/pyproject.toml index 1c2037b0..8ee25627 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,6 +28,7 @@ pytest-cov = "^3.0.0" pytest-mock = "^3.7.0" pytest-httpx = "^0.21.2" requests-mock = "^1.9.3" +faker = "18.9.0" [tool.poetry.scripts] pilotcli = "app.pilotcli:cli" diff --git a/tests/app/commands/test_user.py b/tests/app/commands/test_user.py new file mode 100644 index 00000000..dfda5a55 --- /dev/null +++ b/tests/app/commands/test_user.py @@ -0,0 +1,64 @@ +# Copyright (C) 2023 Indoc Research +# +# Contact Indoc Research for any questions regarding the use of this source code. + +import jwt + +from app.commands.user import login +from app.configs.app_config import AppConfig +from app.configs.user_config import UserConfig + + +def test_login_command_with_api_key_option_calls_keycloak_and_stores_response_in_user_config( + requests_mock, cli_runner, fake +): + username = fake.user_name() + api_key = fake.pystr(20) + access_token = jwt.encode({'preferred_username': username}, key='').decode() + requests_mock.get( + f'{AppConfig.Connections.url_keycloak_realm}/api-key/{api_key}', json={'access_token': access_token} + ) + + result = cli_runner.invoke(login, ['--api-key', api_key]) + + assert result.exit_code == 0 + assert 'Trying to log in using "api-key" method.' in result.output + + user = UserConfig() + assert user.access_token == access_token + assert user.username == username + + +def test_login_command_without_api_key_option_takes_value_from_environment_variable( + monkeypatch, mocker, cli_runner, fake +): + api_key = fake.pystr(20) + monkeypatch.setenv('PILOT_API_KEY', api_key) + login_using_api_key_mock = mocker.patch('app.commands.user.login_using_api_key', return_value=True) + + result = cli_runner.invoke(login) + + assert result.exit_code == 0 + + login_using_api_key_mock.assert_called_once_with(api_key) + + +def test_login_command_without_api_key_option_falls_back_to_device_code_method(mocker, cli_runner, fake): + device_login = { + 'expires': fake.pyint(), + 'interval': fake.pyint(), + 'device_code': fake.pystr(), + 'verification_uri_complete': fake.url(), + } + user_device_id_login_mock = mocker.patch('app.commands.user.user_device_id_login', return_value=device_login) + validate_user_device_login_mock = mocker.patch('app.commands.user.validate_user_device_login', return_value=True) + + result = cli_runner.invoke(login) + + assert result.exit_code == 0 + assert 'Trying to log in using "device-code" method.' in result.output + + user_device_id_login_mock.assert_called_once() + validate_user_device_login_mock.assert_called_once_with( + device_login['device_code'], device_login['expires'], device_login['interval'] + ) diff --git a/tests/app/services/hpc_manager/test_hpc_cluster.py b/tests/app/services/hpc_manager/test_hpc_cluster.py index fa4935bc..986c7efc 100644 --- a/tests/app/services/hpc_manager/test_hpc_cluster.py +++ b/tests/app/services/hpc_manager/test_hpc_cluster.py @@ -16,6 +16,9 @@ def test_hpc_list_partitions(httpx_mock, mocker): 'app.services.user_authentication.token_manager.SrvTokenManager.decode_access_token', return_value=decoded_token(), ) + user_config = UserConfig() + user_config.username = 'test-user' + user_config.hpc_token = 'test-hpc-token' httpx_mock.add_response( method='GET', url='http://bff_cli/v1/hpc/partitions?host=test_host&username=test-user&token=test-hpc-token', @@ -38,12 +41,13 @@ def test_hpc_list_partitions(httpx_mock, mocker): assert partion == expected_partitions -def test_hpc_list_partitions_no_token(mocker, capsys, monkeypatch): +def test_hpc_list_partitions_no_token(mocker, capsys): mocker.patch( 'app.services.user_authentication.token_manager.SrvTokenManager.decode_access_token', return_value=decoded_token(), ) - monkeypatch.setattr(UserConfig, 'hpc_token', '') + user_config = UserConfig() + user_config.hpc_token = '' with pytest.raises(SystemExit): hpc_mgr = HPCPartitionManager() _ = hpc_mgr.list_partitions('test_host') diff --git a/tests/app/services/user_authentication/test_token_manager.py b/tests/app/services/user_authentication/test_token_manager.py new file mode 100644 index 00000000..1b68e8ac --- /dev/null +++ b/tests/app/services/user_authentication/test_token_manager.py @@ -0,0 +1,41 @@ +# Copyright (C) 2023 Indoc Research +# +# Contact Indoc Research for any questions regarding the use of this source code. + +import jwt + +from app.configs.app_config import AppConfig +from app.configs.user_config import UserConfig +from app.services.user_authentication.token_manager import SrvTokenManager + + +class TestSrvTokenManager: + def test_is_api_key_returns_true_when_audience_has_api_key_entry(self): + user_config = UserConfig() + user_config.access_token = jwt.encode({'aud': 'api-key'}, key='').decode() + manager = SrvTokenManager() + + assert manager.is_api_key() is True + + def test_refresh_calls_refresh_api_key_method_when_is_api_key_method_returns_true(self, mocker, fake): + user_config = UserConfig() + user_config.access_token = jwt.encode({'aud': 'api-key'}, key='').decode() + manager = SrvTokenManager() + refresh_api_key_mock = mocker.patch.object(manager, 'refresh_api_key') + + manager.refresh(fake.pystr()) + + refresh_api_key_mock.assert_called_once() + + def test_refresh_api_key_calls_keycloak_and_stores_access_token_in_config(self, requests_mock): + manager = SrvTokenManager() + access_token = jwt.encode({}, key='').decode() + requests_mock.get( + f'{AppConfig.Connections.url_keycloak_realm}/api-key/{manager.config.api_key}', + json={'access_token': access_token}, + ) + + manager.refresh_api_key() + + assert manager.config.access_token == access_token + assert manager.config.refresh_token == '' diff --git a/tests/conftest.py b/tests/conftest.py index e0f7bb59..ca3f66de 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -17,7 +17,7 @@ def reset_singletons(): @pytest.fixture(autouse=True) -def mock_settings(monkeypatch): +def mock_settings(monkeypatch, mocker): monkeypatch.setattr(AppConfig.Connections, 'url_authn', 'http://service_auth') monkeypatch.setattr(AppConfig.Connections, 'url_bff', 'http://bff_cli') monkeypatch.setattr(AppConfig.Connections, 'url_dataset', 'http://url_dataset') @@ -25,11 +25,14 @@ def mock_settings(monkeypatch): monkeypatch.setattr(AppConfig.Connections, 'url_download_core', 'http://url_dataset_download_core') monkeypatch.setattr(AppConfig.Connections, 'url_upload_greenroom', 'http://upload_gr') monkeypatch.setattr(AppConfig.Connections, 'url_upload_core', 'http://upload_core') + monkeypatch.setattr(AppConfig.Connections, 'url_keycloak_realm', 'http://keycloak_realm') monkeypatch.setattr(UserConfig, 'username', 'test-user') monkeypatch.setattr(UserConfig, 'password', 'test-password') + monkeypatch.setattr(UserConfig, 'api_key', 'test-api-key') monkeypatch.setattr(UserConfig, 'access_token', 'test-access-token') monkeypatch.setattr(UserConfig, 'refresh_token', 'test-refresh-token') monkeypatch.setattr(UserConfig, 'hpc_token', 'test-hpc-token') + mocker.patch('app.configs.user_config.UserConfig.save') # Do not save config when running tests def decoded_token(): @@ -61,3 +64,8 @@ def decoded_token(): 'group': ['sample-group'], 'policy': ['project-admin', 'uma_authorization', 'test'], } + + +pytest_plugins = [ + 'tests.fixtures.fake', +] diff --git a/tests/fixtures/__init__.py b/tests/fixtures/__init__.py new file mode 100644 index 00000000..950d1b40 --- /dev/null +++ b/tests/fixtures/__init__.py @@ -0,0 +1,3 @@ +# Copyright (C) 2023 Indoc Research +# +# Contact Indoc Research for any questions regarding the use of this source code. diff --git a/tests/fixtures/fake.py b/tests/fixtures/fake.py new file mode 100644 index 00000000..d4bc9933 --- /dev/null +++ b/tests/fixtures/fake.py @@ -0,0 +1,15 @@ +# Copyright (C) 2023 Indoc Research +# +# Contact Indoc Research for any questions regarding the use of this source code. + +import faker +import pytest + + +class Faker(faker.Faker): + pass + + +@pytest.fixture +def fake() -> Faker: + yield Faker() From 7f989858f4694012a3fd62603185f2c693b6d234 Mon Sep 17 00:00:00 2001 From: Vadym Moshynskyi Date: Mon, 12 Jun 2023 14:38:01 +0200 Subject: [PATCH 09/13] PILOT-3002: Bump version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 8ee25627..44e8d05c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "app" -version = "2.5.0a0" +version = "2.6.0a0" description = "This service is designed to support pilot platform" authors = ["Indoc Research"] From 85bddf53e3b8e306107dd4e2a1d52872ccd8305d Mon Sep 17 00:00:00 2001 From: zhiren Date: Mon, 12 Jun 2023 15:49:52 -0400 Subject: [PATCH 10/13] update the version to 2.5.0 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 44e8d05c..63754644 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "app" -version = "2.6.0a0" +version = "2.5.0" description = "This service is designed to support pilot platform" authors = ["Indoc Research"] From 0fcf173ef55bd4ff4d1147218d57ce65ba2d3d33 Mon Sep 17 00:00:00 2001 From: zhiren Date: Mon, 12 Jun 2023 15:52:59 -0400 Subject: [PATCH 11/13] bump to next version for staging release --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 63754644..2223036a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "app" -version = "2.5.0" +version = "2.5.1" description = "This service is designed to support pilot platform" authors = ["Indoc Research"] From 8b0fd0c257016e06c360b9aeaac5f8fc785fc176 Mon Sep 17 00:00:00 2001 From: zhiren Date: Mon, 12 Jun 2023 15:53:24 -0400 Subject: [PATCH 12/13] update the version in the help page to 2.5.1 --- app/resources/custom_help.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/resources/custom_help.py b/app/resources/custom_help.py index 5906ad1b..0da765ba 100644 --- a/app/resources/custom_help.py +++ b/app/resources/custom_help.py @@ -6,7 +6,7 @@ class HelpPage: page = { 'update': { - 'version': '2.5.0', + 'version': '2.5.1', '1': 'The logic of normal upload and resumble are splited. ' 'add new command for resumable upload as `pilotcli file resume -r manifest.json`', '2': 'The manifest file will be output for both file/folder upload', From b6344e0de5ef6038617c3f8e48c6e3281b7bf13a Mon Sep 17 00:00:00 2001 From: zhiren Date: Tue, 13 Jun 2023 09:28:15 -0400 Subject: [PATCH 13/13] add back the git action command --- .github/workflows/build-and-publish.yml | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index f3afece7..8ab440fe 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -10,7 +10,6 @@ on: push: branches: - main - - PILOT-2936 jobs: extract-branch-name: @@ -65,31 +64,24 @@ jobs: - name: Build binary run: poetry run pyinstaller -F --distpath ./app/bundled_app/linux --specpath ./app/build/linux --workpath ./app/build/linux --paths=./.venv/lib/python3.9/site-packages ./app/pilotcli.py -n ${{ github.sha }} + - name: Rename output file + run: mv "./app/bundled_app/linux/${{ github.sha }}" "./app/bundled_app/linux/pilotcli_linux" + - name: Set version in env run: poetry run echo "TAG_VERSION=`poetry version --short`" >> $GITHUB_ENV - name: Create Release id: create_release - uses: actions/create-release@v1 + uses: softprops/action-gh-release@v1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # This token is provided by Actions, you do not need to create your own token with: tag_name: ${{ env.TAG_VERSION }} - release_name: Release ${{ needs.extract-branch-name.outputs.branch }} ${{ env.TAG_VERSION }} + name: Release ${{ needs.extract-branch-name.outputs.branch }} ${{ env.TAG_VERSION }} body: ${{ github.event.head_commit.message }} draft: false prerelease: false - - - name: Upload Release Binary - id: upload-release-binary - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ steps.create_release.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps - asset_path: ./app/bundled_app/linux/${{ github.sha }} - asset_name: pilotcli_linux - asset_content_type: application/octet-stream + files: ./app/bundled_app/linux/pilotcli_linux push-binary-macos: needs: [ push-binary-linux ]