diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index 8ab440fe..b7582de0 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -81,6 +81,7 @@ jobs: body: ${{ github.event.head_commit.message }} draft: false prerelease: false + target_commitish: ${{ needs.extract-branch-name.outputs.branch }} files: ./app/bundled_app/linux/pilotcli_linux push-binary-macos: diff --git a/app/resources/custom_help.py b/app/resources/custom_help.py index 1b57fe49..1d7628cd 100644 --- a/app/resources/custom_help.py +++ b/app/resources/custom_help.py @@ -6,7 +6,7 @@ class HelpPage: page = { 'update': { - 'version': '2.5.1', + 'version': '2.7.0a0', '1': 'The logic of normal upload and resumble are splited. ' 'add new command for resumable upload as `pilotcli file resume -r manifest.json`', '2': 'The manifest file will be output for both file/folder upload', diff --git a/app/services/file_manager/file_upload/upload_client.py b/app/services/file_manager/file_upload/upload_client.py index 0427c796..202986f5 100644 --- a/app/services/file_manager/file_upload/upload_client.py +++ b/app/services/file_manager/file_upload/upload_client.py @@ -262,13 +262,13 @@ def stream_upload(self, file_object: FileObject, pool: ThreadPool) -> List[Apply while True: chunk = f.read(self.chunk_size) chunk_etag = file_object.uploaded_chunks.get(str(count + 1)) + local_chunk_etag = hashlib.md5(chunk).hexdigest() if not chunk: break # if current chunk has been uploaded to object storage # only check the md5 if the file is same. If ture, # skip current chunk, if not, raise the error. elif chunk_etag: - local_chunk_etag = hashlib.md5(chunk).hexdigest() if chunk_etag != local_chunk_etag: SrvErrorHandler.customized_handle(ECustomizedError.INVALID_CHUNK_UPLOAD, value=count + 1) raise INVALID_CHUNK_ETAG(count + 1) @@ -276,7 +276,7 @@ def stream_upload(self, file_object: FileObject, pool: ThreadPool) -> List[Apply else: res = pool.apply_async( self.upload_chunk, - args=(file_object, count + 1, chunk), + args=(file_object, count + 1, chunk, local_chunk_etag), ) chunk_result.append(res) @@ -286,7 +286,7 @@ def stream_upload(self, file_object: FileObject, pool: ThreadPool) -> List[Apply return chunk_result - def upload_chunk(self, file_object: FileObject, chunk_number: int, chunk: str) -> None: + def upload_chunk(self, file_object: FileObject, chunk_number: int, chunk: str, etag: str) -> None: """ Summary: The function is to upload a chunk directly into minio storage. @@ -295,6 +295,7 @@ def upload_chunk(self, file_object: FileObject, chunk_number: int, chunk: str) - information for chunk uploading. - chunk_number(int): the number of current chunk. - chunk(str): the chunk data. + - etag(str): the md5 of chunk data. return: - None """ @@ -313,7 +314,11 @@ def upload_chunk(self, file_object: FileObject, chunk_number: int, chunk: str) - 'upload_id': file_object.resumable_id, 'chunk_number': chunk_number, } - headers = {'Authorization': 'Bearer ' + self.user.access_token, 'Session-ID': self.user.session_id} + headers = { + 'Authorization': 'Bearer ' + self.user.access_token, + 'Session-ID': self.user.session_id, + 'Content-MD5': etag, + } response = httpx.get( self.base_url + '/v1/files/chunks/presigned', params=params, @@ -326,7 +331,7 @@ def upload_chunk(self, file_object: FileObject, chunk_number: int, chunk: str) - presigned_chunk_url = response.json().get('result') res = httpx.put(presigned_chunk_url, data=chunk, timeout=None) - if res.status_code != 200: + if res.status_code not in [200, 201]: error_msg = 'Fail to upload the chunck %s: %s' % (chunk_number, str(res.text)) raise Exception(error_msg) diff --git a/poetry.lock b/poetry.lock index 397f8d2a..8a9eb0d9 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.4.2 and should not be changed by hand. +# This file is automatically @generated by Poetry and should not be changed by hand. [[package]] name = "altgraph" @@ -370,17 +370,6 @@ files = [ docs = ["furo (>=2022.12.7)", "sphinx (>=5.3)", "sphinx-autodoc-typehints (>=1.19.5)"] testing = ["covdefaults (>=2.2.2)", "coverage (>=7.0.1)", "pytest (>=7.2)", "pytest-cov (>=4)", "pytest-timeout (>=2.1)"] -[[package]] -name = "future" -version = "0.18.3" -description = "Clean single-source support for Python 3 and 2" -category = "main" -optional = false -python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" -files = [ - {file = "future-0.18.3.tar.gz", hash = "sha256:34a17436ed1e96697a86f9de3d15a3b0be01d8bc8de9c1dffd59fb8234ed5307"}, -] - [[package]] name = "h11" version = "0.12.0" @@ -542,18 +531,16 @@ files = [ [[package]] name = "pefile" -version = "2022.5.30" +version = "2023.2.7" description = "Python PE parsing module" category = "main" optional = false python-versions = ">=3.6.0" files = [ - {file = "pefile-2022.5.30.tar.gz", hash = "sha256:a5488a3dd1fd021ce33f969780b88fe0f7eebb76eb20996d7318f307612a045b"}, + {file = "pefile-2023.2.7-py3-none-any.whl", hash = "sha256:da185cd2af68c08a6cd4481f7325ed600a88f6a813bad9dea07ab3ef73d8d8d6"}, + {file = "pefile-2023.2.7.tar.gz", hash = "sha256:82e6114004b3d6911c77c3953e3838654b04511b8b66e8583db70c65998017dc"}, ] -[package.dependencies] -future = "*" - [[package]] name = "platformdirs" version = "2.6.2" @@ -803,24 +790,24 @@ python-dotenv = ">=0.21.0" [[package]] name = "pyinstaller" -version = "5.7.0" +version = "5.13.0" description = "PyInstaller bundles a Python application and all its dependencies into a single package." category = "main" optional = false -python-versions = "<3.12,>=3.7" +python-versions = "<3.13,>=3.7" files = [ - {file = "pyinstaller-5.7.0-py3-none-macosx_10_13_universal2.whl", hash = "sha256:b967ae71ab7b05e18608dbb4518da5afa54f0835927cb7a5ce52ab8fffed03b6"}, - {file = "pyinstaller-5.7.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:3180b9bf22263380adc5e2ee051b7c21463292877215bbe70c9155dc76f4b966"}, - {file = "pyinstaller-5.7.0-py3-none-manylinux2014_i686.whl", hash = "sha256:0f80e2403e76630ad3392c71f09c1a4284e8d8a8a99fb55ff3a0aba0e06300ed"}, - {file = "pyinstaller-5.7.0-py3-none-manylinux2014_ppc64le.whl", hash = "sha256:2c1dd9d11cfc48bab61eeb06de69a3d1ad742bbb2ef14716965ca0333dd43a5b"}, - {file = "pyinstaller-5.7.0-py3-none-manylinux2014_s390x.whl", hash = "sha256:dfc12e92fe10ae645dd0dd1fcfa4cd7677b2e96119e3cd4980d742e09bb78925"}, - {file = "pyinstaller-5.7.0-py3-none-manylinux2014_x86_64.whl", hash = "sha256:f35f06d48faea0ad738429c009941059beebaa306e9d9ead95f1df4b441de2aa"}, - {file = "pyinstaller-5.7.0-py3-none-musllinux_1_1_aarch64.whl", hash = "sha256:28a8a0da656493aa32d9665e2f6f84775da0f23174859ed8facaa4226fe77a17"}, - {file = "pyinstaller-5.7.0-py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:1ac3f09b838710c43e34b0a7ad003bd168a754b0b786c561b47baf1af9104354"}, - {file = "pyinstaller-5.7.0-py3-none-win32.whl", hash = "sha256:9cdb8ee8622ee8d2c6cd67f001b610019d4371a8bf3f7850562640ce786894d7"}, - {file = "pyinstaller-5.7.0-py3-none-win_amd64.whl", hash = "sha256:9b47c10fbefac6f6493266f8b1689109b2b14efa9142dbd2cd7549226a4568b7"}, - {file = "pyinstaller-5.7.0-py3-none-win_arm64.whl", hash = "sha256:3e51e18a16dec0414079762843cf892a5d70749ad56ca7b3c7b5f8367dc50b1e"}, - {file = "pyinstaller-5.7.0.tar.gz", hash = "sha256:0e5953937d35f0b37543cc6915dacaf3239bcbdf3fd3ecbb7866645468a16775"}, + {file = "pyinstaller-5.13.0-py3-none-macosx_10_13_universal2.whl", hash = "sha256:7fdd319828de679f9c5e381eff998ee9b4164bf4457e7fca56946701cf002c3f"}, + {file = "pyinstaller-5.13.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:0df43697c4914285ecd333be968d2cd042ab9b2670124879ee87931d2344eaf5"}, + {file = "pyinstaller-5.13.0-py3-none-manylinux2014_i686.whl", hash = "sha256:28d9742c37e9fb518444b12f8c8ab3cb4ba212d752693c34475c08009aa21ccf"}, + {file = "pyinstaller-5.13.0-py3-none-manylinux2014_ppc64le.whl", hash = "sha256:e5fb17de6c325d3b2b4ceaeb55130ad7100a79096490e4c5b890224406fa42f4"}, + {file = "pyinstaller-5.13.0-py3-none-manylinux2014_s390x.whl", hash = "sha256:78975043edeb628e23a73fb3ef0a273cda50e765f1716f75212ea3e91b09dede"}, + {file = "pyinstaller-5.13.0-py3-none-manylinux2014_x86_64.whl", hash = "sha256:cd7d5c06f2847195a23d72ede17c60857d6f495d6f0727dc6c9bc1235f2eb79c"}, + {file = "pyinstaller-5.13.0-py3-none-musllinux_1_1_aarch64.whl", hash = "sha256:24009eba63cfdbcde6d2634e9c87f545eb67249ddf3b514e0cd3b2cdaa595828"}, + {file = "pyinstaller-5.13.0-py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:1fde4381155f21d6354dc450dcaa338cd8a40aaacf6bd22b987b0f3e1f96f3ee"}, + {file = "pyinstaller-5.13.0-py3-none-win32.whl", hash = "sha256:2d03419904d1c25c8968b0ad21da0e0f33d8d65716e29481b5bd83f7f342b0c5"}, + {file = "pyinstaller-5.13.0-py3-none-win_amd64.whl", hash = "sha256:9fc27c5a853b14a90d39c252707673c7a0efec921cd817169aff3af0fca8c127"}, + {file = "pyinstaller-5.13.0-py3-none-win_arm64.whl", hash = "sha256:3a331951f9744bc2379ea5d65d36f3c828eaefe2785f15039592cdc08560b262"}, + {file = "pyinstaller-5.13.0.tar.gz", hash = "sha256:5e446df41255e815017d96318e39f65a3eb807e74a796c7e7ff7f13b6366a2e9"}, ] [package.dependencies] @@ -829,7 +816,7 @@ importlib-metadata = {version = ">=1.4", markers = "python_version < \"3.8\""} macholib = {version = ">=1.8", markers = "sys_platform == \"darwin\""} pefile = {version = ">=2022.5.30", markers = "sys_platform == \"win32\""} pyinstaller-hooks-contrib = ">=2021.4" -pywin32-ctypes = {version = ">=0.2.0", markers = "sys_platform == \"win32\""} +pywin32-ctypes = {version = ">=0.2.1", markers = "sys_platform == \"win32\""} setuptools = ">=42.0.0" [package.extras] @@ -838,14 +825,14 @@ hook-testing = ["execnet (>=1.5.0)", "psutil", "pytest (>=2.7.3)"] [[package]] name = "pyinstaller-hooks-contrib" -version = "2022.15" +version = "2023.6" description = "Community maintained hooks for PyInstaller" category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "pyinstaller-hooks-contrib-2022.15.tar.gz", hash = "sha256:73fd4051dc1620f3ae9643291cd9e2f47bfed582ade2eb05e3247ecab4a4f5f3"}, - {file = "pyinstaller_hooks_contrib-2022.15-py2.py3-none-any.whl", hash = "sha256:55c1def8066d0279d06cd67eea30c12ffcdb961a5edeeaf361adac0164baef30"}, + {file = "pyinstaller-hooks-contrib-2023.6.tar.gz", hash = "sha256:596a72009d8692b043e0acbf5e1b476d93149900142ba01845dded91a0770cb5"}, + {file = "pyinstaller_hooks_contrib-2023.6-py2.py3-none-any.whl", hash = "sha256:aa6d7d038814df6aa7bec7bdbebc7cb4c693d3398df858f6062957f0797d397b"}, ] [[package]] @@ -1022,14 +1009,14 @@ cli = ["click (>=5.0)"] [[package]] name = "pywin32-ctypes" -version = "0.2.0" -description = "" +version = "0.2.2" +description = "A (partial) reimplementation of pywin32 using ctypes/cffi" category = "main" optional = false -python-versions = "*" +python-versions = ">=3.6" files = [ - {file = "pywin32-ctypes-0.2.0.tar.gz", hash = "sha256:24ffc3b341d457d48e8922352130cf2644024a4ff09762a2261fd34c36ee5942"}, - {file = "pywin32_ctypes-0.2.0-py2.py3-none-any.whl", hash = "sha256:9dc2d991b3479cc2df15930958b674a48a227d5361d413827a4cfd0b5876fc98"}, + {file = "pywin32-ctypes-0.2.2.tar.gz", hash = "sha256:3426e063bdd5fd4df74a14fa3cf80a0b42845a87e1d1e81f6549f9daec593a60"}, + {file = "pywin32_ctypes-0.2.2-py3-none-any.whl", hash = "sha256:bf490a1a709baf35d688fe0ecf980ed4de11d2b3e37b51e5442587a75d9957e7"}, ] [[package]] @@ -1347,4 +1334,4 @@ testing = ["flake8 (<5)", "func-timeout", "jaraco.functools", "jaraco.itertools" [metadata] lock-version = "2.0" python-versions = ">=3.7,<3.11" -content-hash = "8924baccf5bed75b69d29e6157b43f5e90dbe084b73c9fdb695ca0d30d52863b" +content-hash = "bbfe2b73cfbb058677d72db2f3bf1b936ef2904c13a489c2fd323f11a633d603" diff --git a/pyproject.toml b/pyproject.toml index 957a5c54..40a0f3e6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "app" -version = "2.5.2" +version = "2.7.0a0" description = "This service is designed to support pilot platform" authors = ["Indoc Systems"] @@ -16,10 +16,10 @@ requests = "^2.28.2" pydantic-settings = "2.0.1" cffi = "^1.15.1" pre-commit = "^2.19.0" -pyinstaller = "^5.4.1" httpx = "^0.23.0" qrcode = "^7.4.2" pytest-click = "^1.1.0" +pyinstaller = "^5.13.0" [tool.poetry.dev-dependencies] pytest = "6.2.5" diff --git a/tests/app/services/file_manager/file_upload/test_upload_client.py b/tests/app/services/file_manager/file_upload/test_upload_client.py index 4f7b3bf3..93f2b036 100644 --- a/tests/app/services/file_manager/file_upload/test_upload_client.py +++ b/tests/app/services/file_manager/file_upload/test_upload_client.py @@ -73,7 +73,7 @@ def test_chunk_upload(httpx_mock, mocker): mocker.patch('app.services.file_manager.file_upload.models.FileObject.generate_meta', return_value=(1, 1)) test_obj = FileObject('test', 'test', 'test', 'test', 'test') - res = upload_client.upload_chunk(test_obj, 0, b'1') + res = upload_client.upload_chunk(test_obj, 0, b'1', 'test_etag') assert test_obj.progress_bar.n == 1 assert res.status_code == 200