From a11a1ce5998e388b50fdad2907127c0f1c97517e Mon Sep 17 00:00:00 2001 From: zhiren Date: Tue, 2 May 2023 16:33:55 -0400 Subject: [PATCH 01/23] update the lineage to use core zone file instead of greenroom --- app/services/file_manager/file_upload/upload_validator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/services/file_manager/file_upload/upload_validator.py b/app/services/file_manager/file_upload/upload_validator.py index 33210e12..c7b754ad 100644 --- a/app/services/file_manager/file_upload/upload_validator.py +++ b/app/services/file_manager/file_upload/upload_validator.py @@ -29,7 +29,7 @@ def validate_zone(self): ECustomizedError.INVALID_UPLOAD_REQUEST, True, value='upload-message is required' ) if self.source: - source_file_info = search_item(self.project_code, AppConfig.Env.green_zone.lower(), self.source, 'file') + source_file_info = search_item(self.project_code, AppConfig.Env.core_zone.lower(), self.source, 'file') source_file_info = source_file_info['result'] if not source_file_info: SrvErrorHandler.customized_handle(ECustomizedError.INVALID_SOURCE_FILE, True, value=self.source) From d801e8d11affd8e090e4ad142b4afbf6168441f8 Mon Sep 17 00:00:00 2001 From: zhiren Date: Mon, 15 May 2023 10:25:25 -0400 Subject: [PATCH 02/23] bumpup version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 9f1aaf98..59763c07 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "app" -version = "2.4.0" +version = "2.5.0a0" description = "This service is designed to support pilot platform" authors = ["Indoc Research"] From 132ec41b06f1736b840cec3d8820d76d8731d269 Mon Sep 17 00:00:00 2001 From: zhiren Date: Mon, 15 May 2023 13:06:22 -0400 Subject: [PATCH 03/23] bumpup version to 2.5.0 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 59763c07..2901987d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "app" -version = "2.5.0a0" +version = "2.5.0" description = "This service is designed to support pilot platform" authors = ["Indoc Research"] From 0d9d769421de246a725908538721f9ccda1e4ee1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Du=C5=A1an=20Andri=C4=87?= Date: Tue, 16 May 2023 10:24:50 -0400 Subject: [PATCH 04/23] fixes versioning --- app/resources/custom_help.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/resources/custom_help.py b/app/resources/custom_help.py index f3dd9f2c..80cd4c4f 100644 --- a/app/resources/custom_help.py +++ b/app/resources/custom_help.py @@ -6,7 +6,7 @@ class HelpPage: page = { 'update': { - 'version': '2.3.0', + 'version': '2.4.0', '1': 'The logic of normal upload and resumble are splited. ' 'add new command for resumable upload as `pilotcli file resume -r manifest.json`', '2': 'The manifest file will be output for both file/folder upload', diff --git a/pyproject.toml b/pyproject.toml index 2901987d..9f1aaf98 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "app" -version = "2.5.0" +version = "2.4.0" description = "This service is designed to support pilot platform" authors = ["Indoc Research"] From 77414e78d71a454c3f1e003910fbf93bdea7ab77 Mon Sep 17 00:00:00 2001 From: zhiren Date: Tue, 16 May 2023 15:07:48 -0400 Subject: [PATCH 05/23] bumpup version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 773ed5a9..da1a2b85 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "app" -version = "2.4.0a0" +version = "2.4.1" description = "This service is designed to support pilot platform" authors = ["Indoc Research"] From 94b803ab18dde0e6f75fd338e50dc0a01ae2d452 Mon Sep 17 00:00:00 2001 From: Color Zhan Date: Thu, 18 May 2023 09:03:02 -0400 Subject: [PATCH 06/23] Pilot 2936: add an action to create the cli with mac version (#59) * add new action for create mac binary * add the tag name * add the python and poetry * output the release url in the linux step not in a dedicate step * output the release url in the linux step not in a dedicate step * output the release url in the linux step not in a dedicate step * bumpup version * bumpup version * bumpup version * fixup the upload_url * fixup the upload_url * fixup the upload_url * use correct naming * add back build step * remove the test branch * bumpup version --------- Co-authored-by: zhiren --- .github/workflows/build-and-publish.yml | 61 +++++++++++++++++++++++-- app/resources/custom_help.py | 2 +- pyproject.toml | 2 +- 3 files changed, 60 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index 7af02d06..f3afece7 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -10,6 +10,7 @@ on: push: branches: - main + - PILOT-2936 jobs: extract-branch-name: @@ -21,10 +22,13 @@ jobs: id: extract_branch shell: bash run: echo "branch=${GITHUB_REF#refs/heads/}" >> $GITHUB_OUTPUT - push-binary: + + push-binary-linux: needs: [ extract-branch-name ] - if: ${{ needs.extract-branch-name.outputs.branch == 'main' || needs.extract-branch-name.outputs.branch == 'develop' }} + if: ${{ needs.extract-branch-name.outputs.branch == 'main' || needs.extract-branch-name.outputs.branch == 'develop'}} runs-on: ubuntu-20.04 + outputs: + upload_url: ${{steps.create_release.outputs.upload_url}} steps: - name: Checkout repository uses: actions/checkout@v3 @@ -84,5 +88,56 @@ jobs: with: upload_url: ${{ steps.create_release.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps asset_path: ./app/bundled_app/linux/${{ github.sha }} - asset_name: pilotcli + asset_name: pilotcli_linux + asset_content_type: application/octet-stream + + push-binary-macos: + needs: [ push-binary-linux ] + if: ${{ needs.extract-branch-name.outputs.branch == 'main' || needs.extract-branch-name.outputs.branch == 'develop'}} + runs-on: macos-12 + steps: + - name: Checkout repository + uses: actions/checkout@v3 + with: + ref: ${{ needs.extract-branch-name.outputs.branch }} + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: 3.9.16 + + - name: Install Poetry + uses: snok/install-poetry@v1 + with: + version: 1.3.2 + virtualenvs-create: true + virtualenvs-in-project: true + installer-parallel: true + + - name: Set up cache + id: cached-poetry-dependencies + uses: actions/cache@v3 + with: + path: .venv + key: venv-${{ hashFiles('**/poetry.lock') }} + + - name: Ensure cache is healthy + if: steps.cached-poetry-dependencies.outputs.cache-hit == 'true' + run: timeout 10s poetry run pip --version || rm -rf .venv + + - name: Install dependencies + run: poetry install --no-interaction --no-root + + - name: Build binary + run: poetry run pyinstaller -F --distpath ./app/bundled_app/macos --specpath ./app/build/macos --workpath ./app/build/macos --paths=./.venv/lib/python3.9/site-packages ./app/pilotcli.py -n ${{ github.sha }} + + - name: Upload Release Binary + id: upload-release-binary + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ needs.push-binary-linux.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps + asset_path: ./app/bundled_app/macos/${{ github.sha }} + asset_name: pilotcli_macos asset_content_type: application/octet-stream diff --git a/app/resources/custom_help.py b/app/resources/custom_help.py index 80cd4c4f..3dd4ec59 100644 --- a/app/resources/custom_help.py +++ b/app/resources/custom_help.py @@ -6,7 +6,7 @@ class HelpPage: page = { 'update': { - 'version': '2.4.0', + 'version': '2.4.3', '1': 'The logic of normal upload and resumble are splited. ' 'add new command for resumable upload as `pilotcli file resume -r manifest.json`', '2': 'The manifest file will be output for both file/folder upload', diff --git a/pyproject.toml b/pyproject.toml index da1a2b85..9dd01c78 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "app" -version = "2.4.1" +version = "2.4.3" description = "This service is designed to support pilot platform" authors = ["Indoc Research"] From b651dd5b6b090ea31599e46d28e73d2e0ac37e33 Mon Sep 17 00:00:00 2001 From: zhiren Date: Mon, 29 May 2023 12:41:32 -0400 Subject: [PATCH 07/23] fixup the dataset download issue. need more test cases --- .../dataset_manager/dataset_download.py | 52 +++++++++++-------- 1 file changed, 30 insertions(+), 22 deletions(-) diff --git a/app/services/dataset_manager/dataset_download.py b/app/services/dataset_manager/dataset_download.py index de36cb1d..979d0c9e 100644 --- a/app/services/dataset_manager/dataset_download.py +++ b/app/services/dataset_manager/dataset_download.py @@ -5,7 +5,6 @@ import datetime import os import time -from urllib.parse import unquote import requests from tqdm import tqdm @@ -69,23 +68,25 @@ def pre_dataset_download(self): except Exception: SrvErrorHandler.default_handle(response.content, True) - def generate_download_url(self): - if self.version: - download_url = AppConfig.Connections.url_dataset_v2download + f'/download/{self.hash_code}' - else: - download_url = AppConfig.Connections.url_download_core + f'v1/download/{self.hash_code}' - headers = { - 'Authorization': 'Bearer ' + self.user.access_token, - } - res = requests.get(download_url, headers=headers) - res_json = res.json() - if self.version: - self.download_url = self.hash_code - default_filename = self.download_url.split('/')[-1].split('?')[0] - self.default_filename = unquote(default_filename) - else: - self.download_url = download_url - self.default_filename = res_json.get('error_msg').split('/')[-1].rstrip('.') + # def generate_download_url(self): + # if self.version: + # download_url = AppConfig.Connections.url_dataset_v2download + f'/download/{self.hash_code}' + # else: + # download_url = AppConfig.Connections.url_download_core + f'v1/download/{self.hash_code}' + # headers = { + # 'Authorization': 'Bearer ' + self.user.access_token, + # } + # print(download_url) + # res = requests.get(download_url, headers=headers) + # res_json = res.json() + # print(res_json) + # if self.version: + # self.download_url = self.hash_code + # default_filename = self.download_url.split('/')[-1].split('?')[0] + # self.default_filename = unquote(default_filename) + # else: + # self.download_url = download_url + # self.default_filename = res_json.get('error_msg').split('/')[-1].rstrip('.') @require_valid_token() def download_status(self) -> EFileStatus: @@ -109,13 +110,15 @@ def check_download_preparing_status(self) -> EFileStatus: @require_valid_token() def send_download_request(self): logger.info('start downloading...') + with requests.get(self.download_url, stream=True, allow_redirects=True) as r: r.raise_for_status() # Since version zip file was created by our system, thus no need to consider filename contain '?' if not self.default_filename: - filename = f'{self.dataset_code}_{self.version}_{str(datetime.datetime.now())}' + filename = f'{self.dataset_code}_{self.version}_{str(datetime.datetime.now())}.zip' else: filename = self.default_filename + output_path = self.avoid_duplicate_file_name(self.output.rstrip('/') + '/' + filename) self.total_size = int(r.headers.get('Content-length')) with open(output_path, 'wb') as file, tqdm( @@ -151,7 +154,12 @@ def avoid_duplicate_file_name(self, filename): def download_dataset(self): pre_result = self.pre_dataset_download() self.hash_code = pre_result.get('result').get('payload').get('hash_code') - self.generate_download_url() + self.download_url = AppConfig.Connections.url_download_core + f'v1/download/{self.hash_code}' + # format the naming for the default filename + self.default_filename = pre_result.get('result').get('target_names')[0] + self.default_filename = self.default_filename.split('/')[-1] + + # wait the download status to be ready status = self.check_download_preparing_status() SrvOutPutHandler.download_status(status) saved_filename = self.send_download_request() @@ -164,8 +172,8 @@ def download_dataset(self): def download_dataset_version(self, version): self.version = version pre_result = self.pre_dataset_version_download() - self.hash_code = pre_result.get('result').get('source') - self.generate_download_url() + self.download_url = pre_result.get('result').get('source') + saved_filename = self.send_download_request() if os.path.isfile(saved_filename): SrvOutPutHandler.download_success(saved_filename) From 85bddf53e3b8e306107dd4e2a1d52872ccd8305d Mon Sep 17 00:00:00 2001 From: zhiren Date: Mon, 12 Jun 2023 15:49:52 -0400 Subject: [PATCH 08/23] update the version to 2.5.0 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 44e8d05c..63754644 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "app" -version = "2.6.0a0" +version = "2.5.0" description = "This service is designed to support pilot platform" authors = ["Indoc Research"] From 0fcf173ef55bd4ff4d1147218d57ce65ba2d3d33 Mon Sep 17 00:00:00 2001 From: zhiren Date: Mon, 12 Jun 2023 15:52:59 -0400 Subject: [PATCH 09/23] bump to next version for staging release --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 63754644..2223036a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "app" -version = "2.5.0" +version = "2.5.1" description = "This service is designed to support pilot platform" authors = ["Indoc Research"] From 8b0fd0c257016e06c360b9aeaac5f8fc785fc176 Mon Sep 17 00:00:00 2001 From: zhiren Date: Mon, 12 Jun 2023 15:53:24 -0400 Subject: [PATCH 10/23] update the version in the help page to 2.5.1 --- app/resources/custom_help.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/resources/custom_help.py b/app/resources/custom_help.py index 5906ad1b..0da765ba 100644 --- a/app/resources/custom_help.py +++ b/app/resources/custom_help.py @@ -6,7 +6,7 @@ class HelpPage: page = { 'update': { - 'version': '2.5.0', + 'version': '2.5.1', '1': 'The logic of normal upload and resumble are splited. ' 'add new command for resumable upload as `pilotcli file resume -r manifest.json`', '2': 'The manifest file will be output for both file/folder upload', From b6344e0de5ef6038617c3f8e48c6e3281b7bf13a Mon Sep 17 00:00:00 2001 From: zhiren Date: Tue, 13 Jun 2023 09:28:15 -0400 Subject: [PATCH 11/23] add back the git action command --- .github/workflows/build-and-publish.yml | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index f3afece7..8ab440fe 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -10,7 +10,6 @@ on: push: branches: - main - - PILOT-2936 jobs: extract-branch-name: @@ -65,31 +64,24 @@ jobs: - name: Build binary run: poetry run pyinstaller -F --distpath ./app/bundled_app/linux --specpath ./app/build/linux --workpath ./app/build/linux --paths=./.venv/lib/python3.9/site-packages ./app/pilotcli.py -n ${{ github.sha }} + - name: Rename output file + run: mv "./app/bundled_app/linux/${{ github.sha }}" "./app/bundled_app/linux/pilotcli_linux" + - name: Set version in env run: poetry run echo "TAG_VERSION=`poetry version --short`" >> $GITHUB_ENV - name: Create Release id: create_release - uses: actions/create-release@v1 + uses: softprops/action-gh-release@v1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # This token is provided by Actions, you do not need to create your own token with: tag_name: ${{ env.TAG_VERSION }} - release_name: Release ${{ needs.extract-branch-name.outputs.branch }} ${{ env.TAG_VERSION }} + name: Release ${{ needs.extract-branch-name.outputs.branch }} ${{ env.TAG_VERSION }} body: ${{ github.event.head_commit.message }} draft: false prerelease: false - - - name: Upload Release Binary - id: upload-release-binary - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ steps.create_release.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps - asset_path: ./app/bundled_app/linux/${{ github.sha }} - asset_name: pilotcli_linux - asset_content_type: application/octet-stream + files: ./app/bundled_app/linux/pilotcli_linux push-binary-macos: needs: [ push-binary-linux ] From f68911c627430982f9f0b5f960326c4b989dd4d7 Mon Sep 17 00:00:00 2001 From: zhiren Date: Wed, 16 Aug 2023 09:07:19 -0400 Subject: [PATCH 12/23] fix the building issue due to the version mismatch between pydantic and pyinnstall --- .github/workflows/build-and-publish.yml | 2 +- poetry.lock | 69 ++++++++++--------------- pyproject.toml | 2 +- 3 files changed, 30 insertions(+), 43 deletions(-) diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index 8ab440fe..07df3c5c 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -24,7 +24,7 @@ jobs: push-binary-linux: needs: [ extract-branch-name ] - if: ${{ needs.extract-branch-name.outputs.branch == 'main' || needs.extract-branch-name.outputs.branch == 'develop'}} + if: ${{ needs.extract-branch-name.outputs.branch == 'main' || needs.extract-branch-name.outputs.branch == 'develop' || needs.extract-branch-name.outputs.branch == 'rc/staging-release-15082023'}} runs-on: ubuntu-20.04 outputs: upload_url: ${{steps.create_release.outputs.upload_url}} diff --git a/poetry.lock b/poetry.lock index 397f8d2a..8a9eb0d9 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.4.2 and should not be changed by hand. +# This file is automatically @generated by Poetry and should not be changed by hand. [[package]] name = "altgraph" @@ -370,17 +370,6 @@ files = [ docs = ["furo (>=2022.12.7)", "sphinx (>=5.3)", "sphinx-autodoc-typehints (>=1.19.5)"] testing = ["covdefaults (>=2.2.2)", "coverage (>=7.0.1)", "pytest (>=7.2)", "pytest-cov (>=4)", "pytest-timeout (>=2.1)"] -[[package]] -name = "future" -version = "0.18.3" -description = "Clean single-source support for Python 3 and 2" -category = "main" -optional = false -python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" -files = [ - {file = "future-0.18.3.tar.gz", hash = "sha256:34a17436ed1e96697a86f9de3d15a3b0be01d8bc8de9c1dffd59fb8234ed5307"}, -] - [[package]] name = "h11" version = "0.12.0" @@ -542,18 +531,16 @@ files = [ [[package]] name = "pefile" -version = "2022.5.30" +version = "2023.2.7" description = "Python PE parsing module" category = "main" optional = false python-versions = ">=3.6.0" files = [ - {file = "pefile-2022.5.30.tar.gz", hash = "sha256:a5488a3dd1fd021ce33f969780b88fe0f7eebb76eb20996d7318f307612a045b"}, + {file = "pefile-2023.2.7-py3-none-any.whl", hash = "sha256:da185cd2af68c08a6cd4481f7325ed600a88f6a813bad9dea07ab3ef73d8d8d6"}, + {file = "pefile-2023.2.7.tar.gz", hash = "sha256:82e6114004b3d6911c77c3953e3838654b04511b8b66e8583db70c65998017dc"}, ] -[package.dependencies] -future = "*" - [[package]] name = "platformdirs" version = "2.6.2" @@ -803,24 +790,24 @@ python-dotenv = ">=0.21.0" [[package]] name = "pyinstaller" -version = "5.7.0" +version = "5.13.0" description = "PyInstaller bundles a Python application and all its dependencies into a single package." category = "main" optional = false -python-versions = "<3.12,>=3.7" +python-versions = "<3.13,>=3.7" files = [ - {file = "pyinstaller-5.7.0-py3-none-macosx_10_13_universal2.whl", hash = "sha256:b967ae71ab7b05e18608dbb4518da5afa54f0835927cb7a5ce52ab8fffed03b6"}, - {file = "pyinstaller-5.7.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:3180b9bf22263380adc5e2ee051b7c21463292877215bbe70c9155dc76f4b966"}, - {file = "pyinstaller-5.7.0-py3-none-manylinux2014_i686.whl", hash = "sha256:0f80e2403e76630ad3392c71f09c1a4284e8d8a8a99fb55ff3a0aba0e06300ed"}, - {file = "pyinstaller-5.7.0-py3-none-manylinux2014_ppc64le.whl", hash = "sha256:2c1dd9d11cfc48bab61eeb06de69a3d1ad742bbb2ef14716965ca0333dd43a5b"}, - {file = "pyinstaller-5.7.0-py3-none-manylinux2014_s390x.whl", hash = "sha256:dfc12e92fe10ae645dd0dd1fcfa4cd7677b2e96119e3cd4980d742e09bb78925"}, - {file = "pyinstaller-5.7.0-py3-none-manylinux2014_x86_64.whl", hash = "sha256:f35f06d48faea0ad738429c009941059beebaa306e9d9ead95f1df4b441de2aa"}, - {file = "pyinstaller-5.7.0-py3-none-musllinux_1_1_aarch64.whl", hash = "sha256:28a8a0da656493aa32d9665e2f6f84775da0f23174859ed8facaa4226fe77a17"}, - {file = "pyinstaller-5.7.0-py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:1ac3f09b838710c43e34b0a7ad003bd168a754b0b786c561b47baf1af9104354"}, - {file = "pyinstaller-5.7.0-py3-none-win32.whl", hash = "sha256:9cdb8ee8622ee8d2c6cd67f001b610019d4371a8bf3f7850562640ce786894d7"}, - {file = "pyinstaller-5.7.0-py3-none-win_amd64.whl", hash = "sha256:9b47c10fbefac6f6493266f8b1689109b2b14efa9142dbd2cd7549226a4568b7"}, - {file = "pyinstaller-5.7.0-py3-none-win_arm64.whl", hash = "sha256:3e51e18a16dec0414079762843cf892a5d70749ad56ca7b3c7b5f8367dc50b1e"}, - {file = "pyinstaller-5.7.0.tar.gz", hash = "sha256:0e5953937d35f0b37543cc6915dacaf3239bcbdf3fd3ecbb7866645468a16775"}, + {file = "pyinstaller-5.13.0-py3-none-macosx_10_13_universal2.whl", hash = "sha256:7fdd319828de679f9c5e381eff998ee9b4164bf4457e7fca56946701cf002c3f"}, + {file = "pyinstaller-5.13.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:0df43697c4914285ecd333be968d2cd042ab9b2670124879ee87931d2344eaf5"}, + {file = "pyinstaller-5.13.0-py3-none-manylinux2014_i686.whl", hash = "sha256:28d9742c37e9fb518444b12f8c8ab3cb4ba212d752693c34475c08009aa21ccf"}, + {file = "pyinstaller-5.13.0-py3-none-manylinux2014_ppc64le.whl", hash = "sha256:e5fb17de6c325d3b2b4ceaeb55130ad7100a79096490e4c5b890224406fa42f4"}, + {file = "pyinstaller-5.13.0-py3-none-manylinux2014_s390x.whl", hash = "sha256:78975043edeb628e23a73fb3ef0a273cda50e765f1716f75212ea3e91b09dede"}, + {file = "pyinstaller-5.13.0-py3-none-manylinux2014_x86_64.whl", hash = "sha256:cd7d5c06f2847195a23d72ede17c60857d6f495d6f0727dc6c9bc1235f2eb79c"}, + {file = "pyinstaller-5.13.0-py3-none-musllinux_1_1_aarch64.whl", hash = "sha256:24009eba63cfdbcde6d2634e9c87f545eb67249ddf3b514e0cd3b2cdaa595828"}, + {file = "pyinstaller-5.13.0-py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:1fde4381155f21d6354dc450dcaa338cd8a40aaacf6bd22b987b0f3e1f96f3ee"}, + {file = "pyinstaller-5.13.0-py3-none-win32.whl", hash = "sha256:2d03419904d1c25c8968b0ad21da0e0f33d8d65716e29481b5bd83f7f342b0c5"}, + {file = "pyinstaller-5.13.0-py3-none-win_amd64.whl", hash = "sha256:9fc27c5a853b14a90d39c252707673c7a0efec921cd817169aff3af0fca8c127"}, + {file = "pyinstaller-5.13.0-py3-none-win_arm64.whl", hash = "sha256:3a331951f9744bc2379ea5d65d36f3c828eaefe2785f15039592cdc08560b262"}, + {file = "pyinstaller-5.13.0.tar.gz", hash = "sha256:5e446df41255e815017d96318e39f65a3eb807e74a796c7e7ff7f13b6366a2e9"}, ] [package.dependencies] @@ -829,7 +816,7 @@ importlib-metadata = {version = ">=1.4", markers = "python_version < \"3.8\""} macholib = {version = ">=1.8", markers = "sys_platform == \"darwin\""} pefile = {version = ">=2022.5.30", markers = "sys_platform == \"win32\""} pyinstaller-hooks-contrib = ">=2021.4" -pywin32-ctypes = {version = ">=0.2.0", markers = "sys_platform == \"win32\""} +pywin32-ctypes = {version = ">=0.2.1", markers = "sys_platform == \"win32\""} setuptools = ">=42.0.0" [package.extras] @@ -838,14 +825,14 @@ hook-testing = ["execnet (>=1.5.0)", "psutil", "pytest (>=2.7.3)"] [[package]] name = "pyinstaller-hooks-contrib" -version = "2022.15" +version = "2023.6" description = "Community maintained hooks for PyInstaller" category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "pyinstaller-hooks-contrib-2022.15.tar.gz", hash = "sha256:73fd4051dc1620f3ae9643291cd9e2f47bfed582ade2eb05e3247ecab4a4f5f3"}, - {file = "pyinstaller_hooks_contrib-2022.15-py2.py3-none-any.whl", hash = "sha256:55c1def8066d0279d06cd67eea30c12ffcdb961a5edeeaf361adac0164baef30"}, + {file = "pyinstaller-hooks-contrib-2023.6.tar.gz", hash = "sha256:596a72009d8692b043e0acbf5e1b476d93149900142ba01845dded91a0770cb5"}, + {file = "pyinstaller_hooks_contrib-2023.6-py2.py3-none-any.whl", hash = "sha256:aa6d7d038814df6aa7bec7bdbebc7cb4c693d3398df858f6062957f0797d397b"}, ] [[package]] @@ -1022,14 +1009,14 @@ cli = ["click (>=5.0)"] [[package]] name = "pywin32-ctypes" -version = "0.2.0" -description = "" +version = "0.2.2" +description = "A (partial) reimplementation of pywin32 using ctypes/cffi" category = "main" optional = false -python-versions = "*" +python-versions = ">=3.6" files = [ - {file = "pywin32-ctypes-0.2.0.tar.gz", hash = "sha256:24ffc3b341d457d48e8922352130cf2644024a4ff09762a2261fd34c36ee5942"}, - {file = "pywin32_ctypes-0.2.0-py2.py3-none-any.whl", hash = "sha256:9dc2d991b3479cc2df15930958b674a48a227d5361d413827a4cfd0b5876fc98"}, + {file = "pywin32-ctypes-0.2.2.tar.gz", hash = "sha256:3426e063bdd5fd4df74a14fa3cf80a0b42845a87e1d1e81f6549f9daec593a60"}, + {file = "pywin32_ctypes-0.2.2-py3-none-any.whl", hash = "sha256:bf490a1a709baf35d688fe0ecf980ed4de11d2b3e37b51e5442587a75d9957e7"}, ] [[package]] @@ -1347,4 +1334,4 @@ testing = ["flake8 (<5)", "func-timeout", "jaraco.functools", "jaraco.itertools" [metadata] lock-version = "2.0" python-versions = ">=3.7,<3.11" -content-hash = "8924baccf5bed75b69d29e6157b43f5e90dbe084b73c9fdb695ca0d30d52863b" +content-hash = "bbfe2b73cfbb058677d72db2f3bf1b936ef2904c13a489c2fd323f11a633d603" diff --git a/pyproject.toml b/pyproject.toml index 957a5c54..c40e98df 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,10 +16,10 @@ requests = "^2.28.2" pydantic-settings = "2.0.1" cffi = "^1.15.1" pre-commit = "^2.19.0" -pyinstaller = "^5.4.1" httpx = "^0.23.0" qrcode = "^7.4.2" pytest-click = "^1.1.0" +pyinstaller = "^5.13.0" [tool.poetry.dev-dependencies] pytest = "6.2.5" From 0908567fc63b4f79e7b19d01f9dd5024a82a237e Mon Sep 17 00:00:00 2001 From: zhiren Date: Wed, 16 Aug 2023 09:10:14 -0400 Subject: [PATCH 13/23] test workflow --- .github/workflows/build-and-publish.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index 07df3c5c..f4f98bd9 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -5,6 +5,7 @@ on: workflows: [ "Run Tests" ] branches: - develop + - rc/staging-release-15082023 types: - completed push: From 64b0051e0ad869f2582f7d039a1dfb8aa22b76a6 Mon Sep 17 00:00:00 2001 From: zhiren Date: Wed, 16 Aug 2023 09:15:39 -0400 Subject: [PATCH 14/23] test workflow --- .github/workflows/build-and-publish.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index f4f98bd9..b4588836 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -11,6 +11,7 @@ on: push: branches: - main + - rc/staging-release-15082023 jobs: extract-branch-name: From 30640da338a11b414d65c30981980103f24d01c0 Mon Sep 17 00:00:00 2001 From: zhiren Date: Wed, 16 Aug 2023 09:24:38 -0400 Subject: [PATCH 15/23] remove the test workflow --- .github/workflows/build-and-publish.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index b4588836..f2423380 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -11,7 +11,6 @@ on: push: branches: - main - - rc/staging-release-15082023 jobs: extract-branch-name: @@ -26,7 +25,7 @@ jobs: push-binary-linux: needs: [ extract-branch-name ] - if: ${{ needs.extract-branch-name.outputs.branch == 'main' || needs.extract-branch-name.outputs.branch == 'develop' || needs.extract-branch-name.outputs.branch == 'rc/staging-release-15082023'}} + if: ${{ needs.extract-branch-name.outputs.branch == 'main' || needs.extract-branch-name.outputs.branch == 'develop'}} runs-on: ubuntu-20.04 outputs: upload_url: ${{steps.create_release.outputs.upload_url}} From 5784ebf676f198fa8d697c634621bb5a2506cdd8 Mon Sep 17 00:00:00 2001 From: zhiren Date: Wed, 16 Aug 2023 09:25:50 -0400 Subject: [PATCH 16/23] remove the test workflow --- .github/workflows/build-and-publish.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index f2423380..8ab440fe 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -5,7 +5,6 @@ on: workflows: [ "Run Tests" ] branches: - develop - - rc/staging-release-15082023 types: - completed push: From 000c8335c794a53660cceb9180bfc942e16df9e7 Mon Sep 17 00:00:00 2001 From: zhiren Date: Thu, 17 Aug 2023 10:10:51 -0400 Subject: [PATCH 17/23] bumpup to 2.6.0 to align with portal version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index c40e98df..18e505cd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "app" -version = "2.5.2" +version = "2.6.0" description = "This service is designed to support pilot platform" authors = ["Indoc Systems"] From ac97a77b3f70a22e403d7e0059720224371925eb Mon Sep 17 00:00:00 2001 From: zhiren Date: Thu, 17 Aug 2023 10:14:02 -0400 Subject: [PATCH 18/23] update the version in help page --- app/resources/custom_help.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/resources/custom_help.py b/app/resources/custom_help.py index 1b57fe49..87e0eee7 100644 --- a/app/resources/custom_help.py +++ b/app/resources/custom_help.py @@ -6,7 +6,7 @@ class HelpPage: page = { 'update': { - 'version': '2.5.1', + 'version': '2.6.0', '1': 'The logic of normal upload and resumble are splited. ' 'add new command for resumable upload as `pilotcli file resume -r manifest.json`', '2': 'The manifest file will be output for both file/folder upload', From f6d56be9ebdb97f3124a40860d7da885092d82ea Mon Sep 17 00:00:00 2001 From: Daniel Date: Thu, 17 Aug 2023 14:07:53 -0400 Subject: [PATCH 19/23] PILOT-3493: Attribute automated release git tags to main branch --- .github/workflows/build-and-publish.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index 8ab440fe..862e8221 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -81,6 +81,7 @@ jobs: body: ${{ github.event.head_commit.message }} draft: false prerelease: false + target_commitish: main files: ./app/bundled_app/linux/pilotcli_linux push-binary-macos: From 2cf74fb58348c6fbced4a9331446003cba25b11b Mon Sep 17 00:00:00 2001 From: Daniel Date: Tue, 22 Aug 2023 14:52:03 -0400 Subject: [PATCH 20/23] PILOT-3493: Align release commit according to the branch being built --- .github/workflows/build-and-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index 862e8221..b7582de0 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -81,7 +81,7 @@ jobs: body: ${{ github.event.head_commit.message }} draft: false prerelease: false - target_commitish: main + target_commitish: ${{ needs.extract-branch-name.outputs.branch }} files: ./app/bundled_app/linux/pilotcli_linux push-binary-macos: From 595da2c90f69872b12301c667ab95d8497bf0f50 Mon Sep 17 00:00:00 2001 From: Color Zhan Date: Wed, 23 Aug 2023 09:11:51 -0400 Subject: [PATCH 21/23] PILOT-3486: fixup the upload and resumable upload issues (#88) * fixup the upload and resumable upload issues * fixup the test case --------- Co-authored-by: zhiren --- .../file_manager/file_upload/upload_client.py | 15 ++++++++++----- .../file_upload/test_upload_client.py | 2 +- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/app/services/file_manager/file_upload/upload_client.py b/app/services/file_manager/file_upload/upload_client.py index 0427c796..202986f5 100644 --- a/app/services/file_manager/file_upload/upload_client.py +++ b/app/services/file_manager/file_upload/upload_client.py @@ -262,13 +262,13 @@ def stream_upload(self, file_object: FileObject, pool: ThreadPool) -> List[Apply while True: chunk = f.read(self.chunk_size) chunk_etag = file_object.uploaded_chunks.get(str(count + 1)) + local_chunk_etag = hashlib.md5(chunk).hexdigest() if not chunk: break # if current chunk has been uploaded to object storage # only check the md5 if the file is same. If ture, # skip current chunk, if not, raise the error. elif chunk_etag: - local_chunk_etag = hashlib.md5(chunk).hexdigest() if chunk_etag != local_chunk_etag: SrvErrorHandler.customized_handle(ECustomizedError.INVALID_CHUNK_UPLOAD, value=count + 1) raise INVALID_CHUNK_ETAG(count + 1) @@ -276,7 +276,7 @@ def stream_upload(self, file_object: FileObject, pool: ThreadPool) -> List[Apply else: res = pool.apply_async( self.upload_chunk, - args=(file_object, count + 1, chunk), + args=(file_object, count + 1, chunk, local_chunk_etag), ) chunk_result.append(res) @@ -286,7 +286,7 @@ def stream_upload(self, file_object: FileObject, pool: ThreadPool) -> List[Apply return chunk_result - def upload_chunk(self, file_object: FileObject, chunk_number: int, chunk: str) -> None: + def upload_chunk(self, file_object: FileObject, chunk_number: int, chunk: str, etag: str) -> None: """ Summary: The function is to upload a chunk directly into minio storage. @@ -295,6 +295,7 @@ def upload_chunk(self, file_object: FileObject, chunk_number: int, chunk: str) - information for chunk uploading. - chunk_number(int): the number of current chunk. - chunk(str): the chunk data. + - etag(str): the md5 of chunk data. return: - None """ @@ -313,7 +314,11 @@ def upload_chunk(self, file_object: FileObject, chunk_number: int, chunk: str) - 'upload_id': file_object.resumable_id, 'chunk_number': chunk_number, } - headers = {'Authorization': 'Bearer ' + self.user.access_token, 'Session-ID': self.user.session_id} + headers = { + 'Authorization': 'Bearer ' + self.user.access_token, + 'Session-ID': self.user.session_id, + 'Content-MD5': etag, + } response = httpx.get( self.base_url + '/v1/files/chunks/presigned', params=params, @@ -326,7 +331,7 @@ def upload_chunk(self, file_object: FileObject, chunk_number: int, chunk: str) - presigned_chunk_url = response.json().get('result') res = httpx.put(presigned_chunk_url, data=chunk, timeout=None) - if res.status_code != 200: + if res.status_code not in [200, 201]: error_msg = 'Fail to upload the chunck %s: %s' % (chunk_number, str(res.text)) raise Exception(error_msg) diff --git a/tests/app/services/file_manager/file_upload/test_upload_client.py b/tests/app/services/file_manager/file_upload/test_upload_client.py index 4f7b3bf3..93f2b036 100644 --- a/tests/app/services/file_manager/file_upload/test_upload_client.py +++ b/tests/app/services/file_manager/file_upload/test_upload_client.py @@ -73,7 +73,7 @@ def test_chunk_upload(httpx_mock, mocker): mocker.patch('app.services.file_manager.file_upload.models.FileObject.generate_meta', return_value=(1, 1)) test_obj = FileObject('test', 'test', 'test', 'test', 'test') - res = upload_client.upload_chunk(test_obj, 0, b'1') + res = upload_client.upload_chunk(test_obj, 0, b'1', 'test_etag') assert test_obj.progress_bar.n == 1 assert res.status_code == 200 From 31c2ceb3b0eac78872e1295d9e58cac3212fb0ad Mon Sep 17 00:00:00 2001 From: zhiren Date: Fri, 1 Sep 2023 15:20:13 -0400 Subject: [PATCH 22/23] merge back with next preminor version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 18e505cd..40a0f3e6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "app" -version = "2.6.0" +version = "2.7.0a0" description = "This service is designed to support pilot platform" authors = ["Indoc Systems"] From 655462bdd0d0853825a9e637ba4f998adae2fc11 Mon Sep 17 00:00:00 2001 From: zhiren Date: Fri, 1 Sep 2023 15:33:47 -0400 Subject: [PATCH 23/23] update version number in user help page --- app/resources/custom_help.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/resources/custom_help.py b/app/resources/custom_help.py index 87e0eee7..1d7628cd 100644 --- a/app/resources/custom_help.py +++ b/app/resources/custom_help.py @@ -6,7 +6,7 @@ class HelpPage: page = { 'update': { - 'version': '2.6.0', + 'version': '2.7.0a0', '1': 'The logic of normal upload and resumble are splited. ' 'add new command for resumable upload as `pilotcli file resume -r manifest.json`', '2': 'The manifest file will be output for both file/folder upload',