Rc/staging release 15052023

app/services/dataset_manager/dataset_download.py

@@ -43,15 +43,14 @@ def pre_dataset_version_download(self):
             'Session-ID': self.session_id,
         }
         payload = {'version': self.version}
-        try:
-            response = requests.get(url, headers=headers, params=payload)
-            res = response.json()
-            code = res.get('code')
-            if code == 404:
-                SrvErrorHandler.customized_handle(ECustomizedError.VERSION_NOT_EXIST, True, self.version)
-            else:
-                return res
-        except Exception:
+        response = requests.get(url, headers=headers, params=payload)
+        res = response.json()
+        code = response.status_code
+        if code == 200:
+            return res
+        elif code == 404:
+            SrvErrorHandler.customized_handle(ECustomizedError.VERSION_NOT_EXIST, True, self.version)
+        else:
             SrvErrorHandler.default_handle(response.content, True)
 
     @require_valid_token()

app/services/file_manager/file_list.py

@@ -67,9 +67,10 @@ def list_files_without_pagination(self, paths, zone, page, page_size):
     def list_files_with_pagination(self, paths, zone, page, page_size):
         while True:
             files = self.list_files(paths, zone, page, page_size)
-            if len(files) < page_size and page == 0:
-                break
-            elif len(files) < page_size and page != 0:
+            file_list = files.split('...')[:-1] if files != '' else []
+            if len(file_list) < page_size and page == 0:
+                choice = ['exit']
+            elif len(file_list) < page_size and page != 0:
                 choice = ['previous page', 'exit']
             elif page == 0:
                 choice = ['next page', 'exit']

app/services/file_manager/file_upload/file_upload.py

@@ -117,7 +117,7 @@ def simple_upload(  # noqa: C901
     tags = upload_event.get('tags')
     zone = upload_event.get('zone')
     # process_pipeline = upload_event.get('process_pipeline', None)
-    # upload_message = upload_event.get('upload_message')
+    upload_message = upload_event.get('upload_message')
     current_folder_node = upload_event.get('current_folder_node', '')
     parent_folder_id = upload_event.get('parent_folder_id', '')
     create_folder_flag = upload_event.get('create_folder_flag', False)
@@ -155,6 +155,7 @@ def simple_upload(  # noqa: C901
         regular_file=regular_file,
         tags=tags,
         source_id=source_id,
+        upload_message=upload_message,
     )
 
     # format the local path into object storage path for preupload

app/services/file_manager/file_upload/upload_client.py

@@ -224,6 +224,7 @@ def output_manifest(self, file_objects: List[FileObject], output_path: str) -> D
             'parent_folder_id': self.parent_folder_id,
             'current_folder_node': self.current_folder_node,
             'tags': self.tags,
+            'upload_message': self.upload_message,
             'file_objects': {file_object.item_id: file_object.to_dict() for file_object in file_objects},
         }
 

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "app"
-version = "2.3.2"
+version = "2.5.0"
 description = "This service is designed to support pilot platform"
 authors = ["Indoc Research"]
 

tests/app/commands/test_dataset.py

@@ -0,0 +1,51 @@
+# Copyright (C) 2022-2023 Indoc Research
+#
+# Contact Indoc Research for any questions regarding the use of this source code.
+
+from app.commands.dataset import dataset_download
+from app.configs.app_config import AppConfig
+
+
+def test_download_not_exited_dataset_version(requests_mock, mocker, cli_runner, capsys):
+    mocker.patch('app.services.user_authentication.token_manager.SrvTokenManager.check_valid', return_value=0)
+    requests_mock.get(
+        'http://bff_cli' + '/v1/dataset/testdataset',
+        json={
+            'code': 200,
+            'error_msg': '',
+            'result': {
+                'general_info': {
+                    'id': 'fake-id',
+                    'source': '',
+                    'authors': ['test-admin', 'test-user'],
+                    'code': 'testdataset',
+                    'type': 'GENERAL',
+                    'modality': [],
+                    'collection_method': [],
+                    'license': '',
+                    'tags': ['cdsa'],
+                    'description': 'Description example.',
+                    'size': 25,
+                    'total_files': 1,
+                    'title': 'test dataset',
+                    'creator': 'test-admin',
+                    'project_id': 'project-id',
+                    'created_at': '2022-02-03T19:49:35',
+                    'updated_at': '2022-03-18T18:08:33',
+                },
+                'version_detail': [],
+                'version_no': 0,
+            },
+        },
+    )
+
+    requests_mock.get(
+        AppConfig.Connections.url_dataset + '/fake-id/download/pre',
+        json={'error': 'version does not exist'},
+        status_code=404,
+    )
+
+    result = cli_runner.invoke(dataset_download, ['testdataset', '.', '-v', '1.0'])
+    outputs = result.output.split('\n')
+    assert outputs[0] == 'Current dataset version: 1.0'
+    assert outputs[1] == 'Version not available: 1.0'

tests/app/commands/test_file.py

@@ -3,12 +3,15 @@
 # Contact Indoc Research for any questions regarding the use of this source code.
 
 import click
+import questionary
 
+from app.commands.file import file_list
 from app.commands.file import file_put
 from app.commands.file import file_resume
 from app.services.file_manager.file_upload.models import FileObject
 from app.services.output_manager.error_handler import ECustomizedError
 from app.services.output_manager.error_handler import customized_error_msg
+from tests.conftest import decoded_token
 
 
 def test_file_upload_command_success_with_attribute(mocker, cli_runner):
@@ -59,3 +62,43 @@ def test_resumable_upload_command_failed_with_file_not_exists(mocker, cli_runner
     result = cli_runner.invoke(file_resume, ['--resumable-manifest', 'test.json', '--thread', 1])
     assert result.exit_code == 0
     assert result.output == customized_error_msg(ECustomizedError.INVALID_RESUMABLE) + '\n'
+
+
+def test_file_list_with_pagination(requests_mock, mocker, cli_runner):
+    mocker.patch(
+        'app.services.user_authentication.token_manager.SrvTokenManager.decode_access_token',
+        return_value=decoded_token(),
+    )
+
+    mocker.patch('app.services.file_manager.file_list.search_item', return_value=None)
+    requests_mock.get(
+        'http://bff_cli' + '/v1/testproject/files/query',
+        json={
+            'code': 200,
+            'error_msg': '',
+            'result': [{'type': 'file', 'name': 'file1'}, {'type': 'file', 'name': 'file2'}],
+        },
+    )
+    mocker.patch.object(questionary, 'select')
+    questionary.select.return_value.ask.return_value = 'exit'
+    result = cli_runner.invoke(file_list, ['testproject/admin', '-z', 'greenroom'])
+    outputs = result.output.split('\n')
+    assert outputs[0] == 'file1  file2   '
+
+
+def test_empty_file_list_with_pagination(requests_mock, mocker, cli_runner):
+    mocker.patch(
+        'app.services.user_authentication.token_manager.SrvTokenManager.decode_access_token',
+        return_value=decoded_token(),
+    )
+
+    mocker.patch('app.services.file_manager.file_list.search_item', return_value=None)
+    requests_mock.get(
+        'http://bff_cli' + '/v1/testproject/files/query',
+        json={'code': 200, 'error_msg': '', 'result': []},
+    )
+    mocker.patch.object(questionary, 'select')
+    questionary.select.return_value.ask.return_value = 'exit'
+    result = cli_runner.invoke(file_list, ['testproject/admin', '-z', 'greenroom'])
+    outputs = result.output.split('\n')
+    assert outputs[0] == ' '

tests/conftest.py

@@ -20,6 +20,7 @@ def reset_singletons():
 def mock_settings(monkeypatch):
     monkeypatch.setattr(AppConfig.Connections, 'url_authn', 'http://service_auth')
     monkeypatch.setattr(AppConfig.Connections, 'url_bff', 'http://bff_cli')
+    monkeypatch.setattr(AppConfig.Connections, 'url_dataset', 'http://url_dataset')
     monkeypatch.setattr(AppConfig.Connections, 'url_upload_greenroom', 'http://upload_gr')
     monkeypatch.setattr(AppConfig.Connections, 'url_upload_core', 'http://upload_core')
     monkeypatch.setattr(UserConfig, 'username', 'test-user')