From 2c2adc627fae0c35dff2b974fc06e8402ce06582 Mon Sep 17 00:00:00 2001 From: "steve@roderick.com" Date: Mon, 20 Jan 2020 15:13:23 +0000 Subject: [PATCH 01/30] Swap call to repo to use enhanced repocache --- weblab/experiments/tests/test_views.py | 2 ++ weblab/experiments/views.py | 4 ++-- weblab/repocache/models.py | 12 ++++++++++++ 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/weblab/experiments/tests/test_views.py b/weblab/experiments/tests/test_views.py index 04da9d536..0e8eec778 100644 --- a/weblab/experiments/tests/test_views.py +++ b/weblab/experiments/tests/test_views.py @@ -21,6 +21,7 @@ PlannedExperiment, RunningExperiment, ) +from repocache.populate import populate_entity_cache def generate_response(template='%s succ celery-task-id'): @@ -1168,6 +1169,7 @@ def test_compare_experiments(self, client, experiment_version, helpers): protocol = recipes.protocol.make() protocol_commit = helpers.add_version(protocol, visibility='public') exp.protocol.repo.tag('v1') + populate_entity_cache(exp.protocol) version2 = recipes.experiment_version.make( status='SUCCESS', diff --git a/weblab/experiments/views.py b/weblab/experiments/views.py index cb024adb0..f263e5a38 100644 --- a/weblab/experiments/views.py +++ b/weblab/experiments/views.py @@ -450,8 +450,8 @@ def _version_json(self, version, model_version_in_name, protocol_version_in_name 'versionId': version.id, 'modelName': exp.model.name, 'protoName': exp.protocol.name, - 'modelVersion': exp.model.repo.get_name_for_commit(exp.model_version), # TODO #191: Use repocache instead - 'protoVersion': exp.protocol.repo.get_name_for_commit(exp.protocol_version), + 'modelVersion': exp.model.repocache.get_version_name(exp.model_version), + 'protoVersion': exp.protocol.repocache.get_version_name(exp.protocol_version), 'runNumber': version.run_number, }) return details diff --git a/weblab/repocache/models.py b/weblab/repocache/models.py index 31cc30500..3b5bd7f93 100644 --- a/weblab/repocache/models.py +++ b/weblab/repocache/models.py @@ -67,6 +67,18 @@ def get_version(self, sha): except ObjectDoesNotExist: raise RepoCacheMiss("Entity version not found") + def get_version_name(self, sha): + """Get a human-friendly display name for the given version + + :param sha: version sha + :return: first cached tag for this commit, if any, or sha if not + """ + version = self.get_version(sha) + first_tag = version.tags.first() + if first_tag is not None: + return first_tag.tag + return sha + def add_version(self, sha): """ Add an entity version to the cache From bb60c870978b1588e47908a92c5a7f435a2a6d2a Mon Sep 17 00:00:00 2001 From: "steve@roderick.com" Date: Mon, 20 Jan 2020 19:05:43 +0000 Subject: [PATCH 02/30] Removed get_name_for_commit method --- weblab/entities/models.py | 4 ++-- weblab/entities/repository.py | 13 ------------- weblab/entities/templatetags/entities.py | 4 ++-- weblab/entities/tests/test_models.py | 1 + weblab/entities/tests/test_repository.py | 13 ------------- weblab/entities/tests/test_templatetags.py | 3 +++ weblab/entities/tests/test_views.py | 2 ++ weblab/experiments/tests/test_models.py | 4 ++++ weblab/experiments/views.py | 4 ++-- weblab/repocache/models.py | 8 ++++++-- weblab/repocache/tests/test_models.py | 20 ++++++++++++++++++++ 11 files changed, 42 insertions(+), 34 deletions(-) diff --git a/weblab/entities/models.py b/weblab/entities/models.py index e375b3c9c..aeb0e2318 100644 --- a/weblab/entities/models.py +++ b/weblab/entities/models.py @@ -93,7 +93,7 @@ def repo_abs_path(self): ) def nice_version(self, commit): - version = self.repo.get_name_for_commit(commit) + version = self.repocache.get_name_for_version(commit) if len(version) > 20: version = version[:8] + '...' return version @@ -296,7 +296,7 @@ def get_version_json(self, commit, ns): 'visibility': self.get_version_visibility(commit.sha, default=self.DEFAULT_VISIBILITY), 'created': commit.timestamp, 'name': self.name, - 'version': self.repo.get_name_for_commit(commit.sha), # TODO #191 use repocache instead + 'version': self.repocache.get_name_for_version(commit.sha), 'files': files, 'commitMessage': commit.message, 'numFiles': len(files), diff --git a/weblab/entities/repository.py b/weblab/entities/repository.py index 1e8fed3a5..35b8fc3da 100644 --- a/weblab/entities/repository.py +++ b/weblab/entities/repository.py @@ -138,19 +138,6 @@ def tag_dict(self): tags.setdefault(tag.commit.hexsha, []).append(tag) return tags - def get_name_for_commit(self, version): - """Get a human-friendly display name for the given version - - :param version: Revision specification (sha, branch name, tag etc.) - or 'latest' to get latest revision - :return: tag for this commit, if any, or version if not - """ - commit = self.get_commit(version) - for tag in self._repo.tags: - if tag.commit == commit._commit: - return tag.name - return version - def hard_reset(self): """ Reset the working tree diff --git a/weblab/entities/templatetags/entities.py b/weblab/entities/templatetags/entities.py index 6e2e13598..2ceceb368 100644 --- a/weblab/entities/templatetags/entities.py +++ b/weblab/entities/templatetags/entities.py @@ -116,14 +116,14 @@ def url_entity_diff_base(context, entity_type): @register.filter def name_of_model(experiment): model = experiment.model - model_version = model.repo.get_name_for_commit(experiment.model_version) + model_version = model.repocache.get_name_for_version(experiment.model_version) return '%s @ %s' % (model.name, model_version) @register.filter def name_of_protocol(experiment): protocol = experiment.protocol - protocol_version = protocol.repo.get_name_for_commit(experiment.protocol_version) + protocol_version = protocol.repocache.get_name_for_version(experiment.protocol_version) return '%s @ %s' % (protocol.name, protocol_version) diff --git a/weblab/entities/tests/test_models.py b/weblab/entities/tests/test_models.py index ebfe1c9ad..58d57a929 100644 --- a/weblab/entities/tests/test_models.py +++ b/weblab/entities/tests/test_models.py @@ -97,6 +97,7 @@ def test_nice_version(self, model_with_version): assert model_with_version.nice_version(commit) == '%s...' % commit[:8] model_with_version.repo.tag('v1') + populate_entity_cache(model_with_version) assert model_with_version.nice_version(commit) == 'v1' def test_set_and_get_version_visibility(self, model_with_version): diff --git a/weblab/entities/tests/test_repository.py b/weblab/entities/tests/test_repository.py index df22dbb73..959bca818 100644 --- a/weblab/entities/tests/test_repository.py +++ b/weblab/entities/tests/test_repository.py @@ -77,19 +77,6 @@ def test_tag(self, repo, repo_file, author): with pytest.raises(GitCommandError): repo.tag('v1') - def test_name_for_commit(self, repo, repo_file, author): - repo.add_file(repo_file) - commit = repo.commit('commit_message', author) - - assert repo.get_name_for_commit(commit.sha) == commit.sha - assert repo.get_name_for_commit('latest') == 'latest' - - repo.tag('v1') - - assert repo.get_name_for_commit(commit.sha) == 'v1' - assert repo.get_name_for_commit('v1') == 'v1' - assert repo.get_name_for_commit('latest') == 'v1' - def test_has_changes(self, repo, repo_file, author): assert not repo.has_changes repo.add_file(repo_file) diff --git a/weblab/entities/tests/test_templatetags.py b/weblab/entities/tests/test_templatetags.py index ae3c9191f..79e812547 100644 --- a/weblab/entities/tests/test_templatetags.py +++ b/weblab/entities/tests/test_templatetags.py @@ -2,6 +2,7 @@ import entities.templatetags.entities as entity_tags from core import recipes +from repocache.populate import populate_entity_cache def test_human_readable_bytes(): @@ -98,6 +99,8 @@ def test_protocol_urls(protocol_with_version): def test_name_of_entity_linked_to_experiment(model_with_version, protocol_with_version): model_with_version.repo.tag('v1') protocol_with_version.repo.tag('v2') + populate_entity_cache(model_with_version) + populate_entity_cache(protocol_with_version) exp = recipes.experiment_version.make( status='SUCCESS', diff --git a/weblab/entities/tests/test_views.py b/weblab/entities/tests/test_views.py index b1fc4ed6f..c81c17ce7 100644 --- a/weblab/entities/tests/test_views.py +++ b/weblab/entities/tests/test_views.py @@ -19,6 +19,7 @@ from entities.models import AnalysisTask, ModelEntity, ProtocolEntity from experiments.models import Experiment, PlannedExperiment from repocache.models import ProtocolInterface +from repocache.populate import populate_entity_cache @pytest.fixture @@ -305,6 +306,7 @@ def test_version_json(self, client, logged_in_user, helpers, can_create_expt, is version = helpers.add_version(model) model.set_version_visibility(version.sha, 'public') model.repo.tag('v1') + populate_entity_cache(model) planned_expt = PlannedExperiment( model=model, model_version=version.sha, protocol=recipes.protocol.make(), protocol_version=uuid.uuid4(), diff --git a/weblab/experiments/tests/test_models.py b/weblab/experiments/tests/test_models.py index 11209f417..4cf2734bb 100644 --- a/weblab/experiments/tests/test_models.py +++ b/weblab/experiments/tests/test_models.py @@ -7,6 +7,7 @@ from core import recipes from experiments.models import Experiment, ExperimentVersion +from repocache.populate import populate_entity_cache @pytest.fixture @@ -56,9 +57,12 @@ def test_nice_versions(self, experiment_version): assert exp.nice_protocol_version == exp.protocol.repo.latest_commit.sha[:8] + '...' exp.model.repo.tag('v1') + populate_entity_cache(exp.model) assert exp.nice_model_version == 'v1' exp.protocol.repo.tag('v2') + populate_entity_cache(exp.protocol) + assert exp.nice_protocol_version == 'v2' def test_visibility(self, helpers): diff --git a/weblab/experiments/views.py b/weblab/experiments/views.py index f263e5a38..2c9f7cab4 100644 --- a/weblab/experiments/views.py +++ b/weblab/experiments/views.py @@ -450,8 +450,8 @@ def _version_json(self, version, model_version_in_name, protocol_version_in_name 'versionId': version.id, 'modelName': exp.model.name, 'protoName': exp.protocol.name, - 'modelVersion': exp.model.repocache.get_version_name(exp.model_version), - 'protoVersion': exp.protocol.repocache.get_version_name(exp.protocol_version), + 'modelVersion': exp.model.repocache.get_name_for_version(exp.model_version), + 'protoVersion': exp.protocol.repocache.get_name_for_version(exp.protocol_version), 'runNumber': version.run_number, }) return details diff --git a/weblab/repocache/models.py b/weblab/repocache/models.py index 3b5bd7f93..c790904cc 100644 --- a/weblab/repocache/models.py +++ b/weblab/repocache/models.py @@ -67,13 +67,17 @@ def get_version(self, sha): except ObjectDoesNotExist: raise RepoCacheMiss("Entity version not found") - def get_version_name(self, sha): + def get_name_for_version(self, sha): """Get a human-friendly display name for the given version :param sha: version sha :return: first cached tag for this commit, if any, or sha if not """ - version = self.get_version(sha) + try: + version = self.get_version(sha) + except ObjectDoesNotExist: + raise RepoCacheMiss("Entity version not found") + first_tag = version.tags.first() if first_tag is not None: return first_tag.tag diff --git a/weblab/repocache/tests/test_models.py b/weblab/repocache/tests/test_models.py index 9dd8f6f04..972d2a073 100644 --- a/weblab/repocache/tests/test_models.py +++ b/weblab/repocache/tests/test_models.py @@ -1,8 +1,11 @@ import pytest +from django.core.exceptions import ObjectDoesNotExist from django.db.utils import IntegrityError from core import recipes +from repocache.exceptions import RepoCacheMiss from repocache.models import CachedModel, CachedProtocol +from repocache.populate import populate_entity_cache @pytest.mark.django_db @@ -111,3 +114,20 @@ def test_add_version(self, helpers): assert version.author == commit.author.name assert version.numfiles == len(commit.filenames) assert version.visibility == 'public' + + def test_get_name_for_version(self, helpers): + model = recipes.model.make() + commit = helpers.add_version(model, visibility='public', cache=False) + populate_entity_cache(model) + assert model.repocache.get_name_for_version(commit.sha) == commit.sha + assert model.repocache.get_name_for_version('latest') == 'latest' + + model.repo.tag('v1') + assert model.repocache.get_name_for_version(commit.sha) == commit.sha + populate_entity_cache(model) + assert model.repocache.get_name_for_version(commit.sha) == 'v1' + assert model.repocache.get_name_for_version('latest') == 'v1' + + # get_name_for_version must be sha or latest + with pytest.raises(RepoCacheMiss): + model.repocache.get_name_for_version('v1') From ded2be4da1f98b6f6a4eb0bc3e94d4046c81e4d9 Mon Sep 17 00:00:00 2001 From: "steve@roderick.com" Date: Mon, 20 Jan 2020 19:12:04 +0000 Subject: [PATCH 03/30] Change comment reference to commit --- weblab/repocache/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weblab/repocache/models.py b/weblab/repocache/models.py index c790904cc..51291e562 100644 --- a/weblab/repocache/models.py +++ b/weblab/repocache/models.py @@ -71,7 +71,7 @@ def get_name_for_version(self, sha): """Get a human-friendly display name for the given version :param sha: version sha - :return: first cached tag for this commit, if any, or sha if not + :return: first cached tag for this version, if any, or sha if not """ try: version = self.get_version(sha) From bad008daeac62a89cd16ac0df1fc952e4ebd3e08 Mon Sep 17 00:00:00 2001 From: "steve@roderick.com" Date: Mon, 20 Jan 2020 19:16:23 +0000 Subject: [PATCH 04/30] Removed import --- weblab/repocache/tests/test_models.py | 1 - 1 file changed, 1 deletion(-) diff --git a/weblab/repocache/tests/test_models.py b/weblab/repocache/tests/test_models.py index 972d2a073..f9ea415c4 100644 --- a/weblab/repocache/tests/test_models.py +++ b/weblab/repocache/tests/test_models.py @@ -1,5 +1,4 @@ import pytest -from django.core.exceptions import ObjectDoesNotExist from django.db.utils import IntegrityError from core import recipes From e7adfe0c68209bf8794b8f7155750bd9bf0a4c32 Mon Sep 17 00:00:00 2001 From: "steve@roderick.com" Date: Tue, 21 Jan 2020 10:02:25 +0000 Subject: [PATCH 05/30] Altered error handling in test --- weblab/repocache/models.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/weblab/repocache/models.py b/weblab/repocache/models.py index 51291e562..c2e1e20ca 100644 --- a/weblab/repocache/models.py +++ b/weblab/repocache/models.py @@ -73,10 +73,7 @@ def get_name_for_version(self, sha): :param sha: version sha :return: first cached tag for this version, if any, or sha if not """ - try: - version = self.get_version(sha) - except ObjectDoesNotExist: - raise RepoCacheMiss("Entity version not found") + version = self.get_version(sha) first_tag = version.tags.first() if first_tag is not None: From 4b4115e59082ddfe054044201c309a3b79d440c7 Mon Sep 17 00:00:00 2001 From: Steve Roderick Date: Wed, 22 Jan 2020 13:08:32 +0000 Subject: [PATCH 06/30] Change to make test cross-platform --- weblab/entities/tests/test_views.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/weblab/entities/tests/test_views.py b/weblab/entities/tests/test_views.py index c81c17ce7..6c33cb8fb 100644 --- a/weblab/entities/tests/test_views.py +++ b/weblab/entities/tests/test_views.py @@ -1,5 +1,6 @@ import io import json +import os import uuid import zipfile from datetime import timedelta @@ -1969,8 +1970,8 @@ def test_bives_diff(self, mock_post, client, helpers): 'https://bives.bio.informatik.uni-rostock.de/', json={ 'files': [ - 'v1 contents\n', - 'v2 contents\n' + 'v1 contents' + os.linesep, + 'v2 contents' + os.linesep ], 'commands': [ 'compHierarchyJson', From 23948e9446d12b8eb64a2774d23dc7c4cf8511b5 Mon Sep 17 00:00:00 2001 From: Steve Roderick Date: Fri, 24 Jan 2020 19:32:02 +0000 Subject: [PATCH 07/30] Change to entity cache where possible and change tests --- weblab/entities/tests/test_views.py | 174 ++++++++++++++++++---------- weblab/entities/views.py | 48 ++++++-- 2 files changed, 151 insertions(+), 71 deletions(-) diff --git a/weblab/entities/tests/test_views.py b/weblab/entities/tests/test_views.py index 6c33cb8fb..6454e0907 100644 --- a/weblab/entities/tests/test_views.py +++ b/weblab/entities/tests/test_views.py @@ -154,43 +154,43 @@ def check(self, client, url, version, tags): def test_view_entity_version(self, client, logged_in_user, helpers): model = recipes.model.make() helpers.add_version(model, visibility='public') - commit = model.repo.latest_commit - self.check(client, '/entities/models/%d/versions/%s' % (model.pk, commit.sha), - commit, []) + version = model.repocache.latest_version + self.check(client, '/entities/models/%d/versions/%s' % (model.pk, version.sha), + version, []) self.check(client, '/entities/models/%d/versions/latest' % model.pk, - commit, []) + version, []) # Now add a second version with tag - assert len(list(model.repo.commits)) == 1 - commit2 = helpers.add_version(model, visibility='public') - model.add_tag('my_tag', commit2.sha) + assert model.repocache.versions.count() == 1 + version2 = helpers.add_version(model, visibility='public') + model.add_tag('my_tag', version2.sha) # Commits are yielded newest first - assert len(list(model.repo.commits)) == 2 - assert commit == list(model.repo.commits)[-1] - commit = model.repo.latest_commit + assert model.repocache.versions.count() == 2 + assert version == model.repocache.versions.last() + version = model.repocache.latest_version - self.check(client, '/entities/models/%d/versions/%s' % (model.pk, commit.sha), - commit, ['my_tag']) + self.check(client, '/entities/models/%d/versions/%s' % (model.pk, version.sha), + version, ['my_tag']) self.check(client, '/entities/models/%d/versions/%s' % (model.pk, 'my_tag'), - commit, ['my_tag']) + version, ['my_tag']) self.check(client, '/entities/models/%d/versions/latest' % model.pk, - commit, ['my_tag']) + version, ['my_tag']) def test_version_with_two_tags(self, client, helpers): model = recipes.model.make() helpers.add_version(model, visibility='public') - commit = model.repo.latest_commit - model.add_tag('tag1', commit.sha) - model.add_tag('tag2', commit.sha) - self.check(client, '/entities/models/%d/versions/%s' % (model.pk, commit.sha), - commit, ['tag1', 'tag2']) + version = model.repocache.latest_version + model.add_tag('tag1', version.sha) + model.add_tag('tag2', version.sha) + self.check(client, '/entities/models/%d/versions/%s' % (model.pk, version.sha), + version, ['tag1', 'tag2']) self.check(client, '/entities/models/%d/versions/%s' % (model.pk, 'tag1'), - commit, ['tag1', 'tag2']) + version, ['tag1', 'tag2']) self.check(client, '/entities/models/%d/versions/%s' % (model.pk, 'tag2'), - commit, ['tag1', 'tag2']) + version, ['tag1', 'tag2']) self.check(client, '/entities/models/%d/versions/latest' % model.pk, - commit, ['tag1', 'tag2']) + version, ['tag1', 'tag2']) def test_shows_correct_visibility(self, client, logged_in_user, model_with_version): model = model_with_version @@ -808,19 +808,18 @@ def test_view_entity_version_list(self, client, helpers): commit1 = helpers.add_version(model, visibility='public') commit2 = helpers.add_version(model, visibility='moderated') model.add_tag('v1', commit2.sha) - response = client.get('/entities/models/%d/versions/' % model.pk) assert response.status_code == 200 assert response.context['versions'] == [ - (['v1'], commit2), - ([], commit1), + (['v1'], model.repocache.get_version(commit2.sha)), + ([], model.repocache.get_version(commit1.sha)), ] def test_only_shows_visible_versions(self, client, helpers): model = recipes.model.make() - helpers.add_version(model, visibility='private') - commit2 = helpers.add_version(model, visibility='public') - helpers.add_version(model, visibility='private') + helpers.add_fake_version(model, visibility='private') + commit2 = helpers.add_fake_version(model, visibility='public') + helpers.add_fake_version(model, visibility='private') response = client.get('/entities/models/%d/versions/' % model.pk) assert response.status_code == 200 @@ -2107,13 +2106,16 @@ def test_view_run_experiment_model(self, client, helpers, logged_in_user): commit2 = helpers.add_version(protocol, visibility='public') protocol.add_tag('v1', commit2.sha) + version1 = protocol.repocache.get_version(commit1.sha) + version2 = protocol.repocache.get_version(commit2.sha) + response = client.get( '/entities/models/%d/versions/%s/runexperiments' % (model.pk, 'latest')) assert response.status_code == 200 assert response.context['object_list'] == [{'id': protocol.pk, 'name': 'myprotocol1', - 'versions': [{'commit': commit2, 'tags': ['v1'], 'latest': True}, - {'commit': commit1, 'tags': [], 'latest': False}]}, + 'versions': [{'commit': version2, 'tags': ['v1'], 'latest': True}, + {'commit': version1, 'tags': [], 'latest': False}]}, ] assert response.context['preposition'] == 'under' @@ -2132,19 +2134,25 @@ def test_view_run_experiment_model_multiple_users(self, client, helpers, logged_ other_commit2 = helpers.add_version(other_protocol, visibility='public') other_protocol.add_tag('v1', other_commit2.sha) + version1 = protocol.repocache.get_version(commit1.sha) + version2 = protocol.repocache.get_version(commit2.sha) + + other_version1 = other_protocol.repocache.get_version(other_commit1.sha) + other_version2 = other_protocol.repocache.get_version(other_commit2.sha) + response = client.get( '/entities/models/%d/versions/%s/runexperiments' % (model.pk, 'latest')) assert response.status_code == 200 assert response.context['object_list'] == [{'id': protocol.pk, 'name': 'myprotocol1', - 'versions': [{'commit': commit2, 'tags': ['v1'], 'latest': True}, - {'commit': commit1, 'tags': [], 'latest': False}]}, + 'versions': [{'commit': version2, 'tags': ['v1'], 'latest': True}, + {'commit': version1, 'tags': [], 'latest': False}]}, ] assert response.context['other_object_list'] == [ {'id': other_protocol.pk, 'name': 'myprotocol2', - 'versions': [{'commit': other_commit2, 'tags': ['v1'], 'latest': True}, - {'commit': other_commit1, 'tags': [], 'latest': False}]}, + 'versions': [{'commit': other_version2, 'tags': ['v1'], 'latest': True}, + {'commit': other_version1, 'tags': [], 'latest': False}]}, ] assert response.context['preposition'] == 'under' @@ -2158,14 +2166,17 @@ def test_view_run_experiment_model_post(self, client, helpers, logged_in_user): commit2 = helpers.add_version(protocol, visibility='public') protocol.add_tag('v1', commit2.sha) + version1 = protocol.repocache.get_version(commit1.sha) + version2 = protocol.repocache.get_version(commit2.sha) + # Test context has correct information response = client.get( '/entities/models/%d/versions/%s/runexperiments' % (model.pk, commit_model.sha)) assert response.status_code == 200 assert response.context['object_list'] == [{'id': protocol.pk, 'name': 'myprotocol1', - 'versions': [{'commit': commit2, 'tags': ['v1'], 'latest': True}, - {'commit': commit1, 'tags': [], 'latest': False}]}, + 'versions': [{'commit': version2, 'tags': ['v1'], 'latest': True}, + {'commit': version1, 'tags': [], 'latest': False}]}, ] # Test post returns correct response data = {'model_protocol_list[]': ['%d:%s' % (protocol.pk, commit2.sha)], @@ -2196,6 +2207,9 @@ def test_view_run_experiment_model_post_exclude_existing(self, client, helpers, commit2 = helpers.add_version(protocol, visibility='public') protocol.add_tag('v1', commit2.sha) + version1 = protocol.repocache.get_version(commit1.sha) + version2 = protocol.repocache.get_version(commit2.sha) + recipes.experiment_version.make( status='SUCCESS', experiment__model=model, @@ -2209,8 +2223,8 @@ def test_view_run_experiment_model_post_exclude_existing(self, client, helpers, assert response.status_code == 200 assert response.context['object_list'] == [{'id': protocol.pk, 'name': 'myprotocol1', - 'versions': [{'commit': commit2, 'tags': ['v1'], 'latest': True}, - {'commit': commit1, 'tags': [], 'latest': False}]}, + 'versions': [{'commit': version2, 'tags': ['v1'], 'latest': True}, + {'commit': version1, 'tags': [], 'latest': False}]}, ] # Test post returns correct response data = {'model_protocol_list[]': ['%d:%s' % (protocol.pk, commit1.sha), @@ -2247,20 +2261,26 @@ def test_view_run_experiment_post_model_multiple_users(self, client, helpers, lo other_commit2 = helpers.add_version(other_protocol, visibility='public') other_protocol.add_tag('v1', other_commit2.sha) + version1 = protocol.repocache.get_version(commit1.sha) + version2 = protocol.repocache.get_version(commit2.sha) + + other_version1 = other_protocol.repocache.get_version(other_commit1.sha) + other_version2 = other_protocol.repocache.get_version(other_commit2.sha) + # check context response = client.get( '/entities/models/%d/versions/%s/runexperiments' % (model.pk, commit_model.sha)) assert response.status_code == 200 assert response.context['object_list'] == [{'id': protocol.pk, 'name': 'myprotocol1', - 'versions': [{'commit': commit2, 'tags': ['v1'], 'latest': True}, - {'commit': commit1, 'tags': [], 'latest': False}]}, + 'versions': [{'commit': version2, 'tags': ['v1'], 'latest': True}, + {'commit': version1, 'tags': [], 'latest': False}]}, ] assert response.context['other_object_list'] == [ {'id': other_protocol.pk, 'name': 'myprotocol2', - 'versions': [{'commit': other_commit2, 'tags': ['v1'], 'latest': True}, - {'commit': other_commit1, 'tags': [], 'latest': False}]}, + 'versions': [{'commit': other_version2, 'tags': ['v1'], 'latest': True}, + {'commit': other_version1, 'tags': [], 'latest': False}]}, ] # Test post returns correct response data = {'model_protocol_list[]': ['%d:%s' % (protocol.pk, commit2.sha), @@ -2298,14 +2318,17 @@ def test_view_run_experiment_model_not_latest(self, client, helpers, logged_in_u commit2 = helpers.add_version(protocol, visibility='public') protocol.add_tag('v1', commit2.sha) + version1 = protocol.repocache.get_version(commit1.sha) + version2 = protocol.repocache.get_version(commit2.sha) + # display page using tag response = client.get( '/entities/models/%d/versions/%s/runexperiments' % (model.pk, 'model_v1')) assert response.status_code == 200 assert response.context['object_list'] == [{'id': protocol.pk, 'name': 'myprotocol1', - 'versions': [{'commit': commit2, 'tags': ['v1'], 'latest': True}, - {'commit': commit1, 'tags': [], 'latest': False}]}, + 'versions': [{'commit': version2, 'tags': ['v1'], 'latest': True}, + {'commit': version1, 'tags': [], 'latest': False}]}, ] assert response.context['preposition'] == 'under' @@ -2340,13 +2363,16 @@ def test_view_run_experiment_protocol(self, client, helpers, logged_in_user): protocol = recipes.protocol.make(author=logged_in_user) helpers.add_version(protocol, visibility='private') + version1 = model.repocache.get_version(commit1.sha) + version2 = model.repocache.get_version(commit2.sha) + response = client.get( '/entities/protocols/%d/versions/%s/runexperiments' % (protocol.pk, 'latest')) assert response.status_code == 200 assert response.context['object_list'] == [{'id': model.pk, 'name': 'mymodel1', - 'versions': [{'commit': commit2, 'tags': ['v1'], 'latest': True}, - {'commit': commit1, 'tags': [], 'latest': False}]}, + 'versions': [{'commit': version2, 'tags': ['v1'], 'latest': True}, + {'commit': version1, 'tags': [], 'latest': False}]}, ] assert response.context['preposition'] == 'on' @@ -2365,19 +2391,25 @@ def test_view_run_experiment_protocol_multiple_users(self, client, helpers, logg protocol = recipes.protocol.make(author=logged_in_user) helpers.add_version(protocol, visibility='private') + version1 = model.repocache.get_version(commit1.sha) + version2 = model.repocache.get_version(commit2.sha) + + other_version1 = other_model.repocache.get_version(other_commit1.sha) + other_version2 = other_model.repocache.get_version(other_commit2.sha) + response = client.get( '/entities/protocols/%d/versions/%s/runexperiments' % (protocol.pk, 'latest')) assert response.status_code == 200 assert response.context['object_list'] == [{'id': model.pk, 'name': 'mymodel1', - 'versions': [{'commit': commit2, 'tags': ['v1'], 'latest': True}, - {'commit': commit1, 'tags': [], 'latest': False}]}, + 'versions': [{'commit': version2, 'tags': ['v1'], 'latest': True}, + {'commit': version1, 'tags': [], 'latest': False}]}, ] assert response.context['other_object_list'] == [ {'id': other_model.pk, 'name': 'mymodel2', - 'versions': [{'commit': other_commit2, 'tags': ['v1'], 'latest': True}, - {'commit': other_commit1, 'tags': [], 'latest': False}]}, + 'versions': [{'commit': other_version2, 'tags': ['v1'], 'latest': True}, + {'commit': other_version1, 'tags': [], 'latest': False}]}, ] assert response.context['preposition'] == 'on' @@ -2390,13 +2422,16 @@ def test_view_run_experiment_protocol_post(self, client, helpers, logged_in_user protocol = recipes.protocol.make(author=logged_in_user) commit_protocol = helpers.add_version(protocol, visibility='public') + version1 = model.repocache.get_version(commit1.sha) + version2 = model.repocache.get_version(commit2.sha) + response = client.get( '/entities/protocols/%d/versions/%s/runexperiments' % (protocol.pk, commit_protocol.sha)) assert response.status_code == 200 assert response.context['object_list'] == [{'id': model.pk, 'name': 'mymodel1', - 'versions': [{'commit': commit2, 'tags': ['v1'], 'latest': True}, - {'commit': commit1, 'tags': [], 'latest': False}]}, + 'versions': [{'commit': version2, 'tags': ['v1'], 'latest': True}, + {'commit': version1, 'tags': [], 'latest': False}]}, ] # Test post returns correct response data = {'model_protocol_list[]': ['%d:%s' % (model.pk, commit1.sha), '%d:%s' % (model.pk, commit2.sha)], @@ -2440,14 +2475,17 @@ def test_view_run_experiment_protocol_post_exclude_existing(self, client, helper protocol=protocol, protocol_version=commit_protocol.sha) + version1 = model.repocache.get_version(commit1.sha) + version2 = model.repocache.get_version(commit2.sha) + # Test context has correct information response = client.get( '/entities/protocols/%d/versions/%s/runexperiments' % (protocol.pk, commit_protocol.sha)) assert response.status_code == 200 assert response.context['object_list'] == [{'id': model.pk, 'name': 'mymodel1', - 'versions': [{'commit': commit2, 'tags': ['v1'], 'latest': True}, - {'commit': commit1, 'tags': [], 'latest': False}]}, + 'versions': [{'commit': version2, 'tags': ['v1'], 'latest': True}, + {'commit': version1, 'tags': [], 'latest': False}]}, ] # Test post returns correct response data = {'model_protocol_list[]': ['%d:%s' % (model.pk, commit1.sha), '%d:%s' % (model.pk, commit2.sha)], @@ -2474,6 +2512,7 @@ def test_view_run_experiment_post_protocol_multiple_users(self, client, helpers, commit1 = helpers.add_version(model, visibility='public') commit2 = helpers.add_version(model, visibility='public') model.add_tag('v1', commit2.sha) + protocol = recipes.protocol.make(author=logged_in_user) commit_protocol = helpers.add_version(protocol, visibility='public') @@ -2482,19 +2521,24 @@ def test_view_run_experiment_post_protocol_multiple_users(self, client, helpers, other_commit2 = helpers.add_version(other_model, visibility='public') other_model.add_tag('v1', other_commit2.sha) + version1 = model.repocache.get_version(commit1.sha) + version2 = model.repocache.get_version(commit2.sha) + other_version1 = other_model.repocache.get_version(other_commit1.sha) + other_version2 = other_model.repocache.get_version(other_commit2.sha) + response = client.get( '/entities/protocols/%d/versions/%s/runexperiments' % (protocol.pk, commit_protocol.sha)) assert response.status_code == 200 assert response.context['object_list'] == [{'id': model.pk, 'name': 'mymodel1', - 'versions': [{'commit': commit2, 'tags': ['v1'], 'latest': True}, - {'commit': commit1, 'tags': [], 'latest': False}]}, + 'versions': [{'commit': version2, 'tags': ['v1'], 'latest': True}, + {'commit': version1, 'tags': [], 'latest': False}]}, ] assert response.context['other_object_list'] == [ {'id': other_model.pk, 'name': 'mymodel2', - 'versions': [{'commit': other_commit2, 'tags': ['v1'], 'latest': True}, - {'commit': other_commit1, 'tags': [], 'latest': False}]}, + 'versions': [{'commit': other_version2, 'tags': ['v1'], 'latest': True}, + {'commit': other_version1, 'tags': [], 'latest': False}]}, ] # Test post returns correct response data = {'model_protocol_list[]': ['%d:%s' % (model.pk, commit1.sha), @@ -2529,14 +2573,17 @@ def test_view_run_experiment_none_checked(self, client, helpers, logged_in_user) commit2 = helpers.add_version(protocol, visibility='public') protocol.add_tag('v1', commit2.sha) + version1 = protocol.repocache.get_version(commit1.sha) + version2 = protocol.repocache.get_version(commit2.sha) + # Test context has correct information response = client.get( '/entities/models/%d/versions/%s/runexperiments' % (model.pk, commit_model.sha)) assert response.status_code == 200 assert response.context['object_list'] == [{'id': protocol.pk, 'name': 'myprotocol1', - 'versions': [{'commit': commit2, 'tags': ['v1'], 'latest': True}, - {'commit': commit1, 'tags': [], 'latest': False}]}, + 'versions': [{'commit': version2, 'tags': ['v1'], 'latest': True}, + {'commit': version1, 'tags': [], 'latest': False}]}, ] # Test post returns correct response data = {'model_protocol_list[]': [], @@ -2573,14 +2620,17 @@ def test_view_run_experiment_protocol_not_latest(self, client, helpers, logged_i protocol.add_tag('p1', proto_commit1.sha) protocol.add_tag('p2', proto_commit2.sha) + version1 = model.repocache.get_version(commit1.sha) + version2 = model.repocache.get_version(commit2.sha) + # display using sha response = client.get( '/entities/protocols/%d/versions/%s/runexperiments' % (protocol.pk, proto_commit1.sha)) assert response.status_code == 200 assert response.context['object_list'] == [{'id': model.pk, 'name': 'mymodel1', - 'versions': [{'commit': commit2, 'tags': ['v1'], 'latest': True}, - {'commit': commit1, 'tags': [], 'latest': False}]}, + 'versions': [{'commit': version2, 'tags': ['v1'], 'latest': True}, + {'commit': version1, 'tags': [], 'latest': False}]}, ] assert response.context['preposition'] == 'on' diff --git a/weblab/entities/views.py b/weblab/entities/views.py index 63670de61..801b04f8e 100644 --- a/weblab/entities/views.py +++ b/weblab/entities/views.py @@ -123,6 +123,36 @@ def get_commit(self): return self._commit + def get_version(self): + """ + Get the cached entity for this version + + :return: CachedEntityVersion object + :raise: Http404 if version not found + """ + sha_or_tag = self.kwargs['sha'] + cached_entity = self.object.cachedentity + try: + if sha_or_tag == 'latest': + latest_version = cached_entity.latest_version + if latest_version: + self._commit = latest_version + return latest_version + + for version in cached_entity.versions.all(): + if version.sha == sha_or_tag: + self._commit = version + return version + + for tag in version.tags.all(): + if tag.tag == sha_or_tag: + self._commit = version + return version + except BadName: + raise Http404 + + raise Http404 + def _get_object(self): if not hasattr(self, 'object'): self.object = self.get_object() @@ -192,7 +222,7 @@ class EntityVersionView(EntityTypeMixin, EntityVersionMixin, DetailView): def get_context_data(self, **kwargs): entity = self._get_object() - visibility = entity.get_version_visibility(self.get_commit().sha) + visibility = entity.get_version_visibility(self.get_version().sha) kwargs['form'] = EntityChangeVisibilityForm( user=self.request.user, initial={ @@ -204,7 +234,7 @@ def get_context_data(self, **kwargs): class EntityVersionJsonView(EntityTypeMixin, EntityVersionMixin, SingleObjectMixin, View): def _planned_experiments(self): obj = self._get_object() - commit = self.get_commit() + commit = self.get_version() kwargs = { obj.entity_type: obj, obj.entity_type + '_version': commit.sha @@ -249,7 +279,7 @@ class EntityCompareExperimentsView(EntityTypeMixin, EntityVersionMixin, DetailVi def get_context_data(self, **kwargs): entity = self._get_object() - commit = self.get_commit() + commit = self.get_version() entity_type = entity.entity_type other_type = entity.other_type @@ -369,7 +399,7 @@ def post(self, request, *args, **kwargs): form = self.get_form() if form.is_valid(): entity = self._get_object() - commit = self.get_commit() + commit = self.get_version() tag = form.cleaned_data['tag'] try: entity.add_tag(tag, commit.sha) @@ -637,7 +667,7 @@ def get_context_data(self, **kwargs): kwargs.update(**{ 'versions': list( (list(version.tags.values_list('tag', flat=True)), - entity.repo.get_commit(version.sha)) + entity.repocache.get_version(version.sha)) for version in versions.prefetch_related('tags') ) }) @@ -1007,9 +1037,9 @@ def get_context_data(self, **kwargs): version_info = [] for version in versions.prefetch_related('tags'): tag_list = list(version.tags.values_list('tag', flat=True)) - commit = item.repo.get_commit(version.sha) - latest = item.repo.latest_commit - version_info.append({'commit': commit, 'tags': tag_list, 'latest': latest == commit}) + ver = item.repocache.get_version(version.sha) + latest_version = item.repocache.latest_version + version_info.append({'commit': ver, 'tags': tag_list, 'latest': latest_version == ver}) if item.author == self.request.user: context['object_list'].append({'id': item.id, 'name': item.name, 'versions': version_info}) else: @@ -1021,7 +1051,7 @@ def post(self, request, *args, **kwargs): # in get context self.object was the entity being worked with # here we have to retrieve it this_entity = self.get_object() - this_version = self.get_commit().sha + this_version = self.get_version().sha is_latest = (this_version == this_entity.repocache.latest_version.sha) exclude_existing = 'rerun_expts' not in request.POST experiments_to_run = request.POST.getlist('model_protocol_list[]') From 385de35dee45021c82ac5fecf7668ef2f31a4df5 Mon Sep 17 00:00:00 2001 From: Steve Roderick Date: Mon, 27 Jan 2020 09:07:13 +0000 Subject: [PATCH 08/30] Indentation change --- weblab/entities/tests/test_views.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/weblab/entities/tests/test_views.py b/weblab/entities/tests/test_views.py index 6454e0907..1ca9ee3df 100644 --- a/weblab/entities/tests/test_views.py +++ b/weblab/entities/tests/test_views.py @@ -183,10 +183,8 @@ def test_version_with_two_tags(self, client, helpers): version = model.repocache.latest_version model.add_tag('tag1', version.sha) model.add_tag('tag2', version.sha) - self.check(client, '/entities/models/%d/versions/%s' % (model.pk, version.sha), - version, ['tag1', 'tag2']) - self.check(client, '/entities/models/%d/versions/%s' % (model.pk, 'tag1'), - version, ['tag1', 'tag2']) + self.check(client, '/entities/models/%d/versions/%s' % (model.pk, version.sha), version, ['tag1', 'tag2']) + self.check(client, '/entities/models/%d/versions/%s' % (model.pk, 'tag1'), version, ['tag1', 'tag2']) self.check(client, '/entities/models/%d/versions/%s' % (model.pk, 'tag2'), version, ['tag1', 'tag2']) self.check(client, '/entities/models/%d/versions/latest' % model.pk, From 20753cf6cb2584fafa45120ddab470fa3f8916a4 Mon Sep 17 00:00:00 2001 From: "steve@roderick.com" Date: Mon, 27 Jan 2020 11:42:45 +0000 Subject: [PATCH 09/30] Added template option to alert for invalid strings --- weblab/config/settings/dev.py | 31 +++++++++++++++++++++++++++++++ weblab/config/settings/test.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+) diff --git a/weblab/config/settings/dev.py b/weblab/config/settings/dev.py index 58f31e423..671135d9e 100644 --- a/weblab/config/settings/dev.py +++ b/weblab/config/settings/dev.py @@ -80,3 +80,34 @@ app.split('.')[0]: copy.deepcopy(local_logger_conf) for app in LOCAL_APPS }) + + +class InvalidStringShowWarning(str): + def __mod__(self, other): + import logging + logger = logging.getLogger(__name__) + logger.warning("In template, undefined variable or unknown value for: '%s'" % (other,)) + return "" + + +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [ + BASE_DIR / 'templates', + ], + 'APP_DIRS': True, + 'OPTIONS': { + 'string_if_invalid': InvalidStringShowWarning("%s"), + 'context_processors': [ + 'django.template.context_processors.debug', + 'django.template.context_processors.request', + 'django.contrib.auth.context_processors.auth', + 'django.contrib.messages.context_processors.messages', + 'social_django.context_processors.backends', + 'social_django.context_processors.login_redirect', + 'core.context_processors.common', + ], + }, + }, +] \ No newline at end of file diff --git a/weblab/config/settings/test.py b/weblab/config/settings/test.py index 0b310c036..0d273f0cb 100644 --- a/weblab/config/settings/test.py +++ b/weblab/config/settings/test.py @@ -2,3 +2,32 @@ # Settings file used by pytest, whether locally or on Travis +class InvalidStringShowWarning(str): + def __mod__(self, other): + import logging + logger = logging.getLogger(__name__) + logger.warning("In template, undefined variable or unknown value for: '%s'" % (other,)) + return "" + + +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [ + BASE_DIR / 'templates', + ], + 'APP_DIRS': True, + 'OPTIONS': { + 'string_if_invalid': InvalidStringShowWarning("%s"), + 'context_processors': [ + 'django.template.context_processors.debug', + 'django.template.context_processors.request', + 'django.contrib.auth.context_processors.auth', + 'django.contrib.messages.context_processors.messages', + 'social_django.context_processors.backends', + 'social_django.context_processors.login_redirect', + 'core.context_processors.common', + ], + }, + }, +] \ No newline at end of file From 0e5c4b6db87696bbbe83831bf665879db7061229 Mon Sep 17 00:00:00 2001 From: "steve@roderick.com" Date: Mon, 27 Jan 2020 12:04:18 +0000 Subject: [PATCH 10/30] New line at EOF --- weblab/config/settings/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weblab/config/settings/test.py b/weblab/config/settings/test.py index 0d273f0cb..efde37b35 100644 --- a/weblab/config/settings/test.py +++ b/weblab/config/settings/test.py @@ -30,4 +30,4 @@ def __mod__(self, other): ], }, }, -] \ No newline at end of file +] From fad3ff5b51cf94b0b723259da916e53264ff3a14 Mon Sep 17 00:00:00 2001 From: "steve@roderick.com" Date: Mon, 27 Jan 2020 12:13:10 +0000 Subject: [PATCH 11/30] Flake8 error - make import selective --- weblab/config/settings/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weblab/config/settings/test.py b/weblab/config/settings/test.py index efde37b35..f07822425 100644 --- a/weblab/config/settings/test.py +++ b/weblab/config/settings/test.py @@ -1,4 +1,4 @@ -from .base import * # noqa +from .base import BASE_DIR # noqa # Settings file used by pytest, whether locally or on Travis From a3e841eda5baa6d334f06b8f1acbc179501bfb41 Mon Sep 17 00:00:00 2001 From: "steve@roderick.com" Date: Mon, 27 Jan 2020 14:43:06 +0000 Subject: [PATCH 12/30] Flake8 error - change import syntax --- weblab/config/settings/test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/weblab/config/settings/test.py b/weblab/config/settings/test.py index f07822425..e258384ca 100644 --- a/weblab/config/settings/test.py +++ b/weblab/config/settings/test.py @@ -1,4 +1,4 @@ -from .base import BASE_DIR # noqa +from . import base # Settings file used by pytest, whether locally or on Travis @@ -14,7 +14,7 @@ def __mod__(self, other): { 'BACKEND': 'django.template.backends.django.DjangoTemplates', 'DIRS': [ - BASE_DIR / 'templates', + base.BASE_DIR / 'templates', ], 'APP_DIRS': True, 'OPTIONS': { From a0e924c12239ec34452dbef08f0cfeae55fde341 Mon Sep 17 00:00:00 2001 From: Steve Roderick Date: Mon, 27 Jan 2020 14:49:28 +0000 Subject: [PATCH 13/30] Indentation change --- weblab/config/settings/test.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/weblab/config/settings/test.py b/weblab/config/settings/test.py index e258384ca..c46cf819a 100644 --- a/weblab/config/settings/test.py +++ b/weblab/config/settings/test.py @@ -1,7 +1,11 @@ from . import base - # Settings file used by pytest, whether locally or on Travis + +# Test settings +SECRET_KEY = 'test-secret-key' + + class InvalidStringShowWarning(str): def __mod__(self, other): import logging From 8f86c588e6062523eb058b0b256087d86a743811 Mon Sep 17 00:00:00 2001 From: "steve@roderick.com" Date: Mon, 27 Jan 2020 14:53:37 +0000 Subject: [PATCH 14/30] Reverted changes to test.py --- weblab/config/settings/test.py | 37 ++-------------------------------- 1 file changed, 2 insertions(+), 35 deletions(-) diff --git a/weblab/config/settings/test.py b/weblab/config/settings/test.py index c46cf819a..0b310c036 100644 --- a/weblab/config/settings/test.py +++ b/weblab/config/settings/test.py @@ -1,37 +1,4 @@ -from . import base - -# Settings file used by pytest, whether locally or on Travis - -# Test settings -SECRET_KEY = 'test-secret-key' +from .base import * # noqa -class InvalidStringShowWarning(str): - def __mod__(self, other): - import logging - logger = logging.getLogger(__name__) - logger.warning("In template, undefined variable or unknown value for: '%s'" % (other,)) - return "" - - -TEMPLATES = [ - { - 'BACKEND': 'django.template.backends.django.DjangoTemplates', - 'DIRS': [ - base.BASE_DIR / 'templates', - ], - 'APP_DIRS': True, - 'OPTIONS': { - 'string_if_invalid': InvalidStringShowWarning("%s"), - 'context_processors': [ - 'django.template.context_processors.debug', - 'django.template.context_processors.request', - 'django.contrib.auth.context_processors.auth', - 'django.contrib.messages.context_processors.messages', - 'social_django.context_processors.backends', - 'social_django.context_processors.login_redirect', - 'core.context_processors.common', - ], - }, - }, -] +# Settings file used by pytest, whether locally or on Travis From 3d8e6a4a1af2f891acaa28ec31606947f3bd4b70 Mon Sep 17 00:00:00 2001 From: "steve@roderick.com" Date: Mon, 27 Jan 2020 15:36:37 +0000 Subject: [PATCH 15/30] Changed the templates dict is set --- weblab/config/settings/dev.py | 22 +--------------------- weblab/config/settings/test.py | 9 +++++++++ 2 files changed, 10 insertions(+), 21 deletions(-) diff --git a/weblab/config/settings/dev.py b/weblab/config/settings/dev.py index 671135d9e..057aa2496 100644 --- a/weblab/config/settings/dev.py +++ b/weblab/config/settings/dev.py @@ -90,24 +90,4 @@ def __mod__(self, other): return "" -TEMPLATES = [ - { - 'BACKEND': 'django.template.backends.django.DjangoTemplates', - 'DIRS': [ - BASE_DIR / 'templates', - ], - 'APP_DIRS': True, - 'OPTIONS': { - 'string_if_invalid': InvalidStringShowWarning("%s"), - 'context_processors': [ - 'django.template.context_processors.debug', - 'django.template.context_processors.request', - 'django.contrib.auth.context_processors.auth', - 'django.contrib.messages.context_processors.messages', - 'social_django.context_processors.backends', - 'social_django.context_processors.login_redirect', - 'core.context_processors.common', - ], - }, - }, -] \ No newline at end of file +TEMPLATES[0]['OPTIONS']['string_if_invalid'] = InvalidStringShowWarning("%s") diff --git a/weblab/config/settings/test.py b/weblab/config/settings/test.py index 0b310c036..6a15f7140 100644 --- a/weblab/config/settings/test.py +++ b/weblab/config/settings/test.py @@ -2,3 +2,12 @@ # Settings file used by pytest, whether locally or on Travis +class InvalidStringShowWarning(str): + def __mod__(self, other): + import logging + logger = logging.getLogger(__name__) + logger.warning("In template, undefined variable or unknown value for: '%s'" % (other,)) + return "" + + +TEMPLATES[0]['OPTIONS']['string_if_invalid'] = InvalidStringShowWarning("%s") \ No newline at end of file From 4c65d2f87b637dc055184178d301fa46d5bbe188 Mon Sep 17 00:00:00 2001 From: "steve@roderick.com" Date: Mon, 27 Jan 2020 15:51:10 +0000 Subject: [PATCH 16/30] Flake8 error fix --- weblab/config/settings/test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/weblab/config/settings/test.py b/weblab/config/settings/test.py index 6a15f7140..abba9637f 100644 --- a/weblab/config/settings/test.py +++ b/weblab/config/settings/test.py @@ -1,4 +1,4 @@ -from .base import * # noqa +from .base import TEMPLATES # noqa # Settings file used by pytest, whether locally or on Travis @@ -10,4 +10,4 @@ def __mod__(self, other): return "" -TEMPLATES[0]['OPTIONS']['string_if_invalid'] = InvalidStringShowWarning("%s") \ No newline at end of file +TEMPLATES[0]['OPTIONS']['string_if_invalid'] = InvalidStringShowWarning("%s") From 4e12549cc0840dbe4047871c381a88ee23f0bc2d Mon Sep 17 00:00:00 2001 From: Steve Roderick Date: Mon, 27 Jan 2020 15:59:48 +0000 Subject: [PATCH 17/30] Re-added secret_key for Flake failure --- weblab/config/settings/test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/weblab/config/settings/test.py b/weblab/config/settings/test.py index abba9637f..2a53053de 100644 --- a/weblab/config/settings/test.py +++ b/weblab/config/settings/test.py @@ -2,6 +2,8 @@ # Settings file used by pytest, whether locally or on Travis +SECRET_KEY = 'test-secret-key' + class InvalidStringShowWarning(str): def __mod__(self, other): import logging From 77c461a8dd47fbf84a3186821a74f99788cde1e7 Mon Sep 17 00:00:00 2001 From: Steve Roderick Date: Mon, 27 Jan 2020 16:03:48 +0000 Subject: [PATCH 18/30] White space issue --- weblab/config/settings/test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/weblab/config/settings/test.py b/weblab/config/settings/test.py index 2a53053de..259ba0cc9 100644 --- a/weblab/config/settings/test.py +++ b/weblab/config/settings/test.py @@ -4,6 +4,7 @@ # Settings file used by pytest, whether locally or on Travis SECRET_KEY = 'test-secret-key' + class InvalidStringShowWarning(str): def __mod__(self, other): import logging From ac1887b0e2c9d33c8a33740a376eba31c92e2be3 Mon Sep 17 00:00:00 2001 From: Steve Roderick Date: Tue, 28 Jan 2020 11:38:05 +0000 Subject: [PATCH 19/30] Minor refactoring --- weblab/entities/models.py | 16 +++++++++----- weblab/entities/views.py | 45 ++++++++++++++++----------------------- 2 files changed, 29 insertions(+), 32 deletions(-) diff --git a/weblab/entities/models.py b/weblab/entities/models.py index aeb0e2318..bbea62de3 100644 --- a/weblab/entities/models.py +++ b/weblab/entities/models.py @@ -92,11 +92,17 @@ def repo_abs_path(self): self.author.get_storage_dir('repo'), '%ss' % self.entity_type, str(self.id) ) - def nice_version(self, commit): - version = self.repocache.get_name_for_version(commit) - if len(version) > 20: - version = version[:8] + '...' - return version + def nice_version(self, sha_or_tag): + """ + Returns tag/sha with ellipses + + :param sha_or_tag: version sha or tag string + :return version_name: string with the sha_or_tag formatted + """ + version_name = self.repocache.get_name_for_version(sha_or_tag) + if len(version_name) > 20: + version_name = version_name[:8] + '...' + return version_name def get_visibility_from_repo(self, commit): """ diff --git a/weblab/entities/views.py b/weblab/entities/views.py index 801b04f8e..70d7c713b 100644 --- a/weblab/entities/views.py +++ b/weblab/entities/views.py @@ -15,7 +15,7 @@ PermissionRequiredMixin, UserPassesTestMixin, ) -from django.core.exceptions import PermissionDenied +from django.core.exceptions import PermissionDenied, ObjectDoesNotExist from django.core.urlresolvers import reverse from django.db.models import ( Count, @@ -125,33 +125,24 @@ def get_commit(self): def get_version(self): """ - Get the cached entity for this version + Get the cached entity for this version :return: CachedEntityVersion object :raise: Http404 if version not found """ + if hasattr(self, '_version'): + return self._version + sha_or_tag = self.kwargs['sha'] cached_entity = self.object.cachedentity try: - if sha_or_tag == 'latest': - latest_version = cached_entity.latest_version - if latest_version: - self._commit = latest_version - return latest_version - - for version in cached_entity.versions.all(): - if version.sha == sha_or_tag: - self._commit = version - return version - - for tag in version.tags.all(): - if tag.tag == sha_or_tag: - self._commit = version - return version - except BadName: - raise Http404 - - raise Http404 + self._version = cached_entity.get_version(sha_or_tag) + except RepoCacheMiss: + try: + self._version = cached_entity.tags.get(tag=sha_or_tag).version + except ObjectDoesNotExist: + raise Http404 + return self._version def _get_object(self): if not hasattr(self, 'object'): @@ -160,12 +151,12 @@ def _get_object(self): def get_context_data(self, **kwargs): entity = self._get_object() - commit = self.get_commit() + version = self.get_version() kwargs.update(**{ - 'version': commit, + 'version': version, 'visibility': self.get_visibility(), - 'tags': entity.get_tags(commit.sha), - 'master_filename': commit.master_filename, + 'tags': entity.get_tags(version.sha), + 'master_filename': version.master_filename, }) return super().get_context_data(**kwargs) @@ -234,7 +225,7 @@ def get_context_data(self, **kwargs): class EntityVersionJsonView(EntityTypeMixin, EntityVersionMixin, SingleObjectMixin, View): def _planned_experiments(self): obj = self._get_object() - commit = self.get_version() + commit = self.get_commit() kwargs = { obj.entity_type: obj, obj.entity_type + '_version': commit.sha @@ -399,7 +390,7 @@ def post(self, request, *args, **kwargs): form = self.get_form() if form.is_valid(): entity = self._get_object() - commit = self.get_version() + commit = self.get_commit() tag = form.cleaned_data['tag'] try: entity.add_tag(tag, commit.sha) From 66018613ebb5881b4960eb182901e1c42d7b9811 Mon Sep 17 00:00:00 2001 From: Steve Roderick Date: Tue, 28 Jan 2020 11:39:01 +0000 Subject: [PATCH 20/30] Change to call of pytest on travis to trap template errors --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index cee0575d4..e083b6ed7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,7 +15,7 @@ before_script: install: pip install -r requirements/test.txt script: - - pytest --cov=weblab --cov-config=weblab/.coveragerc weblab + - pytest --fail-on-template-vars --cov=weblab --cov-config=weblab/.coveragerc weblab - flake8 weblab - isort --verbose --check-only --diff --recursive weblab after_success: From 26a0fdb34972789bc32b929d740d6fbd760291a8 Mon Sep 17 00:00:00 2001 From: Steve Roderick Date: Tue, 28 Jan 2020 11:39:45 +0000 Subject: [PATCH 21/30] Reverted changes to test.py --- weblab/config/settings/test.py | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/weblab/config/settings/test.py b/weblab/config/settings/test.py index 259ba0cc9..0b310c036 100644 --- a/weblab/config/settings/test.py +++ b/weblab/config/settings/test.py @@ -1,16 +1,4 @@ -from .base import TEMPLATES # noqa +from .base import * # noqa # Settings file used by pytest, whether locally or on Travis -SECRET_KEY = 'test-secret-key' - - -class InvalidStringShowWarning(str): - def __mod__(self, other): - import logging - logger = logging.getLogger(__name__) - logger.warning("In template, undefined variable or unknown value for: '%s'" % (other,)) - return "" - - -TEMPLATES[0]['OPTIONS']['string_if_invalid'] = InvalidStringShowWarning("%s") From ea368150b919f175bb7f9f97726b161e7a9fc9b9 Mon Sep 17 00:00:00 2001 From: Steve Roderick Date: Tue, 28 Jan 2020 11:53:34 +0000 Subject: [PATCH 22/30] isort fix --- weblab/entities/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weblab/entities/views.py b/weblab/entities/views.py index 70d7c713b..daab60dd7 100644 --- a/weblab/entities/views.py +++ b/weblab/entities/views.py @@ -15,7 +15,7 @@ PermissionRequiredMixin, UserPassesTestMixin, ) -from django.core.exceptions import PermissionDenied, ObjectDoesNotExist +from django.core.exceptions import ObjectDoesNotExist, PermissionDenied from django.core.urlresolvers import reverse from django.db.models import ( Count, From f5c08ebea1e81ac0b4c71e69cc41fd658b4c8637 Mon Sep 17 00:00:00 2001 From: Steve Roderick Date: Tue, 28 Jan 2020 13:30:19 +0000 Subject: [PATCH 23/30] Removed use of cache in EntityRunExperimentView --- weblab/entities/tests/test_views.py | 129 +++++++++------------------- weblab/entities/views.py | 8 +- 2 files changed, 43 insertions(+), 94 deletions(-) diff --git a/weblab/entities/tests/test_views.py b/weblab/entities/tests/test_views.py index 1ca9ee3df..946bf0892 100644 --- a/weblab/entities/tests/test_views.py +++ b/weblab/entities/tests/test_views.py @@ -809,15 +809,15 @@ def test_view_entity_version_list(self, client, helpers): response = client.get('/entities/models/%d/versions/' % model.pk) assert response.status_code == 200 assert response.context['versions'] == [ - (['v1'], model.repocache.get_version(commit2.sha)), - ([], model.repocache.get_version(commit1.sha)), + (['v1'], commit2), + ([], commit1), ] def test_only_shows_visible_versions(self, client, helpers): model = recipes.model.make() - helpers.add_fake_version(model, visibility='private') - commit2 = helpers.add_fake_version(model, visibility='public') - helpers.add_fake_version(model, visibility='private') + helpers.add_version(model, visibility='private') + commit2 = helpers.add_version(model, visibility='public') + helpers.add_version(model, visibility='private') response = client.get('/entities/models/%d/versions/' % model.pk) assert response.status_code == 200 @@ -2104,16 +2104,13 @@ def test_view_run_experiment_model(self, client, helpers, logged_in_user): commit2 = helpers.add_version(protocol, visibility='public') protocol.add_tag('v1', commit2.sha) - version1 = protocol.repocache.get_version(commit1.sha) - version2 = protocol.repocache.get_version(commit2.sha) - response = client.get( '/entities/models/%d/versions/%s/runexperiments' % (model.pk, 'latest')) assert response.status_code == 200 assert response.context['object_list'] == [{'id': protocol.pk, 'name': 'myprotocol1', - 'versions': [{'commit': version2, 'tags': ['v1'], 'latest': True}, - {'commit': version1, 'tags': [], 'latest': False}]}, + 'versions': [{'commit': commit2, 'tags': ['v1'], 'latest': True}, + {'commit': commit1, 'tags': [], 'latest': False}]}, ] assert response.context['preposition'] == 'under' @@ -2132,25 +2129,19 @@ def test_view_run_experiment_model_multiple_users(self, client, helpers, logged_ other_commit2 = helpers.add_version(other_protocol, visibility='public') other_protocol.add_tag('v1', other_commit2.sha) - version1 = protocol.repocache.get_version(commit1.sha) - version2 = protocol.repocache.get_version(commit2.sha) - - other_version1 = other_protocol.repocache.get_version(other_commit1.sha) - other_version2 = other_protocol.repocache.get_version(other_commit2.sha) - response = client.get( '/entities/models/%d/versions/%s/runexperiments' % (model.pk, 'latest')) assert response.status_code == 200 assert response.context['object_list'] == [{'id': protocol.pk, 'name': 'myprotocol1', - 'versions': [{'commit': version2, 'tags': ['v1'], 'latest': True}, - {'commit': version1, 'tags': [], 'latest': False}]}, + 'versions': [{'commit': commit2, 'tags': ['v1'], 'latest': True}, + {'commit': commit1, 'tags': [], 'latest': False}]}, ] assert response.context['other_object_list'] == [ {'id': other_protocol.pk, 'name': 'myprotocol2', - 'versions': [{'commit': other_version2, 'tags': ['v1'], 'latest': True}, - {'commit': other_version1, 'tags': [], 'latest': False}]}, + 'versions': [{'commit': other_commit2, 'tags': ['v1'], 'latest': True}, + {'commit': other_commit1, 'tags': [], 'latest': False}]}, ] assert response.context['preposition'] == 'under' @@ -2164,17 +2155,14 @@ def test_view_run_experiment_model_post(self, client, helpers, logged_in_user): commit2 = helpers.add_version(protocol, visibility='public') protocol.add_tag('v1', commit2.sha) - version1 = protocol.repocache.get_version(commit1.sha) - version2 = protocol.repocache.get_version(commit2.sha) - # Test context has correct information response = client.get( '/entities/models/%d/versions/%s/runexperiments' % (model.pk, commit_model.sha)) assert response.status_code == 200 assert response.context['object_list'] == [{'id': protocol.pk, 'name': 'myprotocol1', - 'versions': [{'commit': version2, 'tags': ['v1'], 'latest': True}, - {'commit': version1, 'tags': [], 'latest': False}]}, + 'versions': [{'commit': commit2, 'tags': ['v1'], 'latest': True}, + {'commit': commit1, 'tags': [], 'latest': False}]}, ] # Test post returns correct response data = {'model_protocol_list[]': ['%d:%s' % (protocol.pk, commit2.sha)], @@ -2205,9 +2193,6 @@ def test_view_run_experiment_model_post_exclude_existing(self, client, helpers, commit2 = helpers.add_version(protocol, visibility='public') protocol.add_tag('v1', commit2.sha) - version1 = protocol.repocache.get_version(commit1.sha) - version2 = protocol.repocache.get_version(commit2.sha) - recipes.experiment_version.make( status='SUCCESS', experiment__model=model, @@ -2221,8 +2206,8 @@ def test_view_run_experiment_model_post_exclude_existing(self, client, helpers, assert response.status_code == 200 assert response.context['object_list'] == [{'id': protocol.pk, 'name': 'myprotocol1', - 'versions': [{'commit': version2, 'tags': ['v1'], 'latest': True}, - {'commit': version1, 'tags': [], 'latest': False}]}, + 'versions': [{'commit': commit2, 'tags': ['v1'], 'latest': True}, + {'commit': commit1, 'tags': [], 'latest': False}]}, ] # Test post returns correct response data = {'model_protocol_list[]': ['%d:%s' % (protocol.pk, commit1.sha), @@ -2259,26 +2244,20 @@ def test_view_run_experiment_post_model_multiple_users(self, client, helpers, lo other_commit2 = helpers.add_version(other_protocol, visibility='public') other_protocol.add_tag('v1', other_commit2.sha) - version1 = protocol.repocache.get_version(commit1.sha) - version2 = protocol.repocache.get_version(commit2.sha) - - other_version1 = other_protocol.repocache.get_version(other_commit1.sha) - other_version2 = other_protocol.repocache.get_version(other_commit2.sha) - # check context response = client.get( '/entities/models/%d/versions/%s/runexperiments' % (model.pk, commit_model.sha)) assert response.status_code == 200 assert response.context['object_list'] == [{'id': protocol.pk, 'name': 'myprotocol1', - 'versions': [{'commit': version2, 'tags': ['v1'], 'latest': True}, - {'commit': version1, 'tags': [], 'latest': False}]}, + 'versions': [{'commit': commit2, 'tags': ['v1'], 'latest': True}, + {'commit': commit1, 'tags': [], 'latest': False}]}, ] assert response.context['other_object_list'] == [ {'id': other_protocol.pk, 'name': 'myprotocol2', - 'versions': [{'commit': other_version2, 'tags': ['v1'], 'latest': True}, - {'commit': other_version1, 'tags': [], 'latest': False}]}, + 'versions': [{'commit': other_commit2, 'tags': ['v1'], 'latest': True}, + {'commit': other_commit1, 'tags': [], 'latest': False}]}, ] # Test post returns correct response data = {'model_protocol_list[]': ['%d:%s' % (protocol.pk, commit2.sha), @@ -2316,17 +2295,14 @@ def test_view_run_experiment_model_not_latest(self, client, helpers, logged_in_u commit2 = helpers.add_version(protocol, visibility='public') protocol.add_tag('v1', commit2.sha) - version1 = protocol.repocache.get_version(commit1.sha) - version2 = protocol.repocache.get_version(commit2.sha) - # display page using tag response = client.get( '/entities/models/%d/versions/%s/runexperiments' % (model.pk, 'model_v1')) assert response.status_code == 200 assert response.context['object_list'] == [{'id': protocol.pk, 'name': 'myprotocol1', - 'versions': [{'commit': version2, 'tags': ['v1'], 'latest': True}, - {'commit': version1, 'tags': [], 'latest': False}]}, + 'versions': [{'commit': commit2, 'tags': ['v1'], 'latest': True}, + {'commit': commit1, 'tags': [], 'latest': False}]}, ] assert response.context['preposition'] == 'under' @@ -2361,16 +2337,13 @@ def test_view_run_experiment_protocol(self, client, helpers, logged_in_user): protocol = recipes.protocol.make(author=logged_in_user) helpers.add_version(protocol, visibility='private') - version1 = model.repocache.get_version(commit1.sha) - version2 = model.repocache.get_version(commit2.sha) - response = client.get( '/entities/protocols/%d/versions/%s/runexperiments' % (protocol.pk, 'latest')) assert response.status_code == 200 assert response.context['object_list'] == [{'id': model.pk, 'name': 'mymodel1', - 'versions': [{'commit': version2, 'tags': ['v1'], 'latest': True}, - {'commit': version1, 'tags': [], 'latest': False}]}, + 'versions': [{'commit': commit2, 'tags': ['v1'], 'latest': True}, + {'commit': commit1, 'tags': [], 'latest': False}]}, ] assert response.context['preposition'] == 'on' @@ -2389,25 +2362,19 @@ def test_view_run_experiment_protocol_multiple_users(self, client, helpers, logg protocol = recipes.protocol.make(author=logged_in_user) helpers.add_version(protocol, visibility='private') - version1 = model.repocache.get_version(commit1.sha) - version2 = model.repocache.get_version(commit2.sha) - - other_version1 = other_model.repocache.get_version(other_commit1.sha) - other_version2 = other_model.repocache.get_version(other_commit2.sha) - response = client.get( '/entities/protocols/%d/versions/%s/runexperiments' % (protocol.pk, 'latest')) assert response.status_code == 200 assert response.context['object_list'] == [{'id': model.pk, 'name': 'mymodel1', - 'versions': [{'commit': version2, 'tags': ['v1'], 'latest': True}, - {'commit': version1, 'tags': [], 'latest': False}]}, + 'versions': [{'commit': commit2, 'tags': ['v1'], 'latest': True}, + {'commit': commit1, 'tags': [], 'latest': False}]}, ] assert response.context['other_object_list'] == [ {'id': other_model.pk, 'name': 'mymodel2', - 'versions': [{'commit': other_version2, 'tags': ['v1'], 'latest': True}, - {'commit': other_version1, 'tags': [], 'latest': False}]}, + 'versions': [{'commit': other_commit2, 'tags': ['v1'], 'latest': True}, + {'commit': other_commit1, 'tags': [], 'latest': False}]}, ] assert response.context['preposition'] == 'on' @@ -2420,16 +2387,13 @@ def test_view_run_experiment_protocol_post(self, client, helpers, logged_in_user protocol = recipes.protocol.make(author=logged_in_user) commit_protocol = helpers.add_version(protocol, visibility='public') - version1 = model.repocache.get_version(commit1.sha) - version2 = model.repocache.get_version(commit2.sha) - response = client.get( '/entities/protocols/%d/versions/%s/runexperiments' % (protocol.pk, commit_protocol.sha)) assert response.status_code == 200 assert response.context['object_list'] == [{'id': model.pk, 'name': 'mymodel1', - 'versions': [{'commit': version2, 'tags': ['v1'], 'latest': True}, - {'commit': version1, 'tags': [], 'latest': False}]}, + 'versions': [{'commit': commit2, 'tags': ['v1'], 'latest': True}, + {'commit': commit1, 'tags': [], 'latest': False}]}, ] # Test post returns correct response data = {'model_protocol_list[]': ['%d:%s' % (model.pk, commit1.sha), '%d:%s' % (model.pk, commit2.sha)], @@ -2473,17 +2437,14 @@ def test_view_run_experiment_protocol_post_exclude_existing(self, client, helper protocol=protocol, protocol_version=commit_protocol.sha) - version1 = model.repocache.get_version(commit1.sha) - version2 = model.repocache.get_version(commit2.sha) - # Test context has correct information response = client.get( '/entities/protocols/%d/versions/%s/runexperiments' % (protocol.pk, commit_protocol.sha)) assert response.status_code == 200 assert response.context['object_list'] == [{'id': model.pk, 'name': 'mymodel1', - 'versions': [{'commit': version2, 'tags': ['v1'], 'latest': True}, - {'commit': version1, 'tags': [], 'latest': False}]}, + 'versions': [{'commit': commit2, 'tags': ['v1'], 'latest': True}, + {'commit': commit1, 'tags': [], 'latest': False}]}, ] # Test post returns correct response data = {'model_protocol_list[]': ['%d:%s' % (model.pk, commit1.sha), '%d:%s' % (model.pk, commit2.sha)], @@ -2510,7 +2471,6 @@ def test_view_run_experiment_post_protocol_multiple_users(self, client, helpers, commit1 = helpers.add_version(model, visibility='public') commit2 = helpers.add_version(model, visibility='public') model.add_tag('v1', commit2.sha) - protocol = recipes.protocol.make(author=logged_in_user) commit_protocol = helpers.add_version(protocol, visibility='public') @@ -2519,24 +2479,19 @@ def test_view_run_experiment_post_protocol_multiple_users(self, client, helpers, other_commit2 = helpers.add_version(other_model, visibility='public') other_model.add_tag('v1', other_commit2.sha) - version1 = model.repocache.get_version(commit1.sha) - version2 = model.repocache.get_version(commit2.sha) - other_version1 = other_model.repocache.get_version(other_commit1.sha) - other_version2 = other_model.repocache.get_version(other_commit2.sha) - response = client.get( '/entities/protocols/%d/versions/%s/runexperiments' % (protocol.pk, commit_protocol.sha)) assert response.status_code == 200 assert response.context['object_list'] == [{'id': model.pk, 'name': 'mymodel1', - 'versions': [{'commit': version2, 'tags': ['v1'], 'latest': True}, - {'commit': version1, 'tags': [], 'latest': False}]}, + 'versions': [{'commit': commit2, 'tags': ['v1'], 'latest': True}, + {'commit': commit1, 'tags': [], 'latest': False}]}, ] assert response.context['other_object_list'] == [ {'id': other_model.pk, 'name': 'mymodel2', - 'versions': [{'commit': other_version2, 'tags': ['v1'], 'latest': True}, - {'commit': other_version1, 'tags': [], 'latest': False}]}, + 'versions': [{'commit': other_commit2, 'tags': ['v1'], 'latest': True}, + {'commit': other_commit1, 'tags': [], 'latest': False}]}, ] # Test post returns correct response data = {'model_protocol_list[]': ['%d:%s' % (model.pk, commit1.sha), @@ -2571,17 +2526,14 @@ def test_view_run_experiment_none_checked(self, client, helpers, logged_in_user) commit2 = helpers.add_version(protocol, visibility='public') protocol.add_tag('v1', commit2.sha) - version1 = protocol.repocache.get_version(commit1.sha) - version2 = protocol.repocache.get_version(commit2.sha) - # Test context has correct information response = client.get( '/entities/models/%d/versions/%s/runexperiments' % (model.pk, commit_model.sha)) assert response.status_code == 200 assert response.context['object_list'] == [{'id': protocol.pk, 'name': 'myprotocol1', - 'versions': [{'commit': version2, 'tags': ['v1'], 'latest': True}, - {'commit': version1, 'tags': [], 'latest': False}]}, + 'versions': [{'commit': commit2, 'tags': ['v1'], 'latest': True}, + {'commit': commit1, 'tags': [], 'latest': False}]}, ] # Test post returns correct response data = {'model_protocol_list[]': [], @@ -2618,17 +2570,14 @@ def test_view_run_experiment_protocol_not_latest(self, client, helpers, logged_i protocol.add_tag('p1', proto_commit1.sha) protocol.add_tag('p2', proto_commit2.sha) - version1 = model.repocache.get_version(commit1.sha) - version2 = model.repocache.get_version(commit2.sha) - # display using sha response = client.get( '/entities/protocols/%d/versions/%s/runexperiments' % (protocol.pk, proto_commit1.sha)) assert response.status_code == 200 assert response.context['object_list'] == [{'id': model.pk, 'name': 'mymodel1', - 'versions': [{'commit': version2, 'tags': ['v1'], 'latest': True}, - {'commit': version1, 'tags': [], 'latest': False}]}, + 'versions': [{'commit': commit2, 'tags': ['v1'], 'latest': True}, + {'commit': commit1, 'tags': [], 'latest': False}]}, ] assert response.context['preposition'] == 'on' diff --git a/weblab/entities/views.py b/weblab/entities/views.py index daab60dd7..5d7753855 100644 --- a/weblab/entities/views.py +++ b/weblab/entities/views.py @@ -658,7 +658,7 @@ def get_context_data(self, **kwargs): kwargs.update(**{ 'versions': list( (list(version.tags.values_list('tag', flat=True)), - entity.repocache.get_version(version.sha)) + entity.repo.get_commit(version.sha)) for version in versions.prefetch_related('tags') ) }) @@ -1028,9 +1028,9 @@ def get_context_data(self, **kwargs): version_info = [] for version in versions.prefetch_related('tags'): tag_list = list(version.tags.values_list('tag', flat=True)) - ver = item.repocache.get_version(version.sha) - latest_version = item.repocache.latest_version - version_info.append({'commit': ver, 'tags': tag_list, 'latest': latest_version == ver}) + commit = item.repo.get_commit(version.sha) + latest = item.repo.latest_commit + version_info.append({'commit': commit, 'tags': tag_list, 'latest': latest == commit}) if item.author == self.request.user: context['object_list'].append({'id': item.id, 'name': item.name, 'versions': version_info}) else: From b574349b27d35a2e5820fd88cd5c2275fd9a756f Mon Sep 17 00:00:00 2001 From: Steve Roderick Date: Tue, 28 Jan 2020 14:45:04 +0000 Subject: [PATCH 24/30] Reinstate use of cache - remove ref to commit.filenames in templates --- weblab/entities/tests/test_views.py | 129 ++++++++++++------ weblab/entities/views.py | 8 +- .../entities/entity_runexperiments.html | 4 +- .../templates/entities/entity_versions.html | 4 +- 4 files changed, 96 insertions(+), 49 deletions(-) diff --git a/weblab/entities/tests/test_views.py b/weblab/entities/tests/test_views.py index 946bf0892..1ca9ee3df 100644 --- a/weblab/entities/tests/test_views.py +++ b/weblab/entities/tests/test_views.py @@ -809,15 +809,15 @@ def test_view_entity_version_list(self, client, helpers): response = client.get('/entities/models/%d/versions/' % model.pk) assert response.status_code == 200 assert response.context['versions'] == [ - (['v1'], commit2), - ([], commit1), + (['v1'], model.repocache.get_version(commit2.sha)), + ([], model.repocache.get_version(commit1.sha)), ] def test_only_shows_visible_versions(self, client, helpers): model = recipes.model.make() - helpers.add_version(model, visibility='private') - commit2 = helpers.add_version(model, visibility='public') - helpers.add_version(model, visibility='private') + helpers.add_fake_version(model, visibility='private') + commit2 = helpers.add_fake_version(model, visibility='public') + helpers.add_fake_version(model, visibility='private') response = client.get('/entities/models/%d/versions/' % model.pk) assert response.status_code == 200 @@ -2104,13 +2104,16 @@ def test_view_run_experiment_model(self, client, helpers, logged_in_user): commit2 = helpers.add_version(protocol, visibility='public') protocol.add_tag('v1', commit2.sha) + version1 = protocol.repocache.get_version(commit1.sha) + version2 = protocol.repocache.get_version(commit2.sha) + response = client.get( '/entities/models/%d/versions/%s/runexperiments' % (model.pk, 'latest')) assert response.status_code == 200 assert response.context['object_list'] == [{'id': protocol.pk, 'name': 'myprotocol1', - 'versions': [{'commit': commit2, 'tags': ['v1'], 'latest': True}, - {'commit': commit1, 'tags': [], 'latest': False}]}, + 'versions': [{'commit': version2, 'tags': ['v1'], 'latest': True}, + {'commit': version1, 'tags': [], 'latest': False}]}, ] assert response.context['preposition'] == 'under' @@ -2129,19 +2132,25 @@ def test_view_run_experiment_model_multiple_users(self, client, helpers, logged_ other_commit2 = helpers.add_version(other_protocol, visibility='public') other_protocol.add_tag('v1', other_commit2.sha) + version1 = protocol.repocache.get_version(commit1.sha) + version2 = protocol.repocache.get_version(commit2.sha) + + other_version1 = other_protocol.repocache.get_version(other_commit1.sha) + other_version2 = other_protocol.repocache.get_version(other_commit2.sha) + response = client.get( '/entities/models/%d/versions/%s/runexperiments' % (model.pk, 'latest')) assert response.status_code == 200 assert response.context['object_list'] == [{'id': protocol.pk, 'name': 'myprotocol1', - 'versions': [{'commit': commit2, 'tags': ['v1'], 'latest': True}, - {'commit': commit1, 'tags': [], 'latest': False}]}, + 'versions': [{'commit': version2, 'tags': ['v1'], 'latest': True}, + {'commit': version1, 'tags': [], 'latest': False}]}, ] assert response.context['other_object_list'] == [ {'id': other_protocol.pk, 'name': 'myprotocol2', - 'versions': [{'commit': other_commit2, 'tags': ['v1'], 'latest': True}, - {'commit': other_commit1, 'tags': [], 'latest': False}]}, + 'versions': [{'commit': other_version2, 'tags': ['v1'], 'latest': True}, + {'commit': other_version1, 'tags': [], 'latest': False}]}, ] assert response.context['preposition'] == 'under' @@ -2155,14 +2164,17 @@ def test_view_run_experiment_model_post(self, client, helpers, logged_in_user): commit2 = helpers.add_version(protocol, visibility='public') protocol.add_tag('v1', commit2.sha) + version1 = protocol.repocache.get_version(commit1.sha) + version2 = protocol.repocache.get_version(commit2.sha) + # Test context has correct information response = client.get( '/entities/models/%d/versions/%s/runexperiments' % (model.pk, commit_model.sha)) assert response.status_code == 200 assert response.context['object_list'] == [{'id': protocol.pk, 'name': 'myprotocol1', - 'versions': [{'commit': commit2, 'tags': ['v1'], 'latest': True}, - {'commit': commit1, 'tags': [], 'latest': False}]}, + 'versions': [{'commit': version2, 'tags': ['v1'], 'latest': True}, + {'commit': version1, 'tags': [], 'latest': False}]}, ] # Test post returns correct response data = {'model_protocol_list[]': ['%d:%s' % (protocol.pk, commit2.sha)], @@ -2193,6 +2205,9 @@ def test_view_run_experiment_model_post_exclude_existing(self, client, helpers, commit2 = helpers.add_version(protocol, visibility='public') protocol.add_tag('v1', commit2.sha) + version1 = protocol.repocache.get_version(commit1.sha) + version2 = protocol.repocache.get_version(commit2.sha) + recipes.experiment_version.make( status='SUCCESS', experiment__model=model, @@ -2206,8 +2221,8 @@ def test_view_run_experiment_model_post_exclude_existing(self, client, helpers, assert response.status_code == 200 assert response.context['object_list'] == [{'id': protocol.pk, 'name': 'myprotocol1', - 'versions': [{'commit': commit2, 'tags': ['v1'], 'latest': True}, - {'commit': commit1, 'tags': [], 'latest': False}]}, + 'versions': [{'commit': version2, 'tags': ['v1'], 'latest': True}, + {'commit': version1, 'tags': [], 'latest': False}]}, ] # Test post returns correct response data = {'model_protocol_list[]': ['%d:%s' % (protocol.pk, commit1.sha), @@ -2244,20 +2259,26 @@ def test_view_run_experiment_post_model_multiple_users(self, client, helpers, lo other_commit2 = helpers.add_version(other_protocol, visibility='public') other_protocol.add_tag('v1', other_commit2.sha) + version1 = protocol.repocache.get_version(commit1.sha) + version2 = protocol.repocache.get_version(commit2.sha) + + other_version1 = other_protocol.repocache.get_version(other_commit1.sha) + other_version2 = other_protocol.repocache.get_version(other_commit2.sha) + # check context response = client.get( '/entities/models/%d/versions/%s/runexperiments' % (model.pk, commit_model.sha)) assert response.status_code == 200 assert response.context['object_list'] == [{'id': protocol.pk, 'name': 'myprotocol1', - 'versions': [{'commit': commit2, 'tags': ['v1'], 'latest': True}, - {'commit': commit1, 'tags': [], 'latest': False}]}, + 'versions': [{'commit': version2, 'tags': ['v1'], 'latest': True}, + {'commit': version1, 'tags': [], 'latest': False}]}, ] assert response.context['other_object_list'] == [ {'id': other_protocol.pk, 'name': 'myprotocol2', - 'versions': [{'commit': other_commit2, 'tags': ['v1'], 'latest': True}, - {'commit': other_commit1, 'tags': [], 'latest': False}]}, + 'versions': [{'commit': other_version2, 'tags': ['v1'], 'latest': True}, + {'commit': other_version1, 'tags': [], 'latest': False}]}, ] # Test post returns correct response data = {'model_protocol_list[]': ['%d:%s' % (protocol.pk, commit2.sha), @@ -2295,14 +2316,17 @@ def test_view_run_experiment_model_not_latest(self, client, helpers, logged_in_u commit2 = helpers.add_version(protocol, visibility='public') protocol.add_tag('v1', commit2.sha) + version1 = protocol.repocache.get_version(commit1.sha) + version2 = protocol.repocache.get_version(commit2.sha) + # display page using tag response = client.get( '/entities/models/%d/versions/%s/runexperiments' % (model.pk, 'model_v1')) assert response.status_code == 200 assert response.context['object_list'] == [{'id': protocol.pk, 'name': 'myprotocol1', - 'versions': [{'commit': commit2, 'tags': ['v1'], 'latest': True}, - {'commit': commit1, 'tags': [], 'latest': False}]}, + 'versions': [{'commit': version2, 'tags': ['v1'], 'latest': True}, + {'commit': version1, 'tags': [], 'latest': False}]}, ] assert response.context['preposition'] == 'under' @@ -2337,13 +2361,16 @@ def test_view_run_experiment_protocol(self, client, helpers, logged_in_user): protocol = recipes.protocol.make(author=logged_in_user) helpers.add_version(protocol, visibility='private') + version1 = model.repocache.get_version(commit1.sha) + version2 = model.repocache.get_version(commit2.sha) + response = client.get( '/entities/protocols/%d/versions/%s/runexperiments' % (protocol.pk, 'latest')) assert response.status_code == 200 assert response.context['object_list'] == [{'id': model.pk, 'name': 'mymodel1', - 'versions': [{'commit': commit2, 'tags': ['v1'], 'latest': True}, - {'commit': commit1, 'tags': [], 'latest': False}]}, + 'versions': [{'commit': version2, 'tags': ['v1'], 'latest': True}, + {'commit': version1, 'tags': [], 'latest': False}]}, ] assert response.context['preposition'] == 'on' @@ -2362,19 +2389,25 @@ def test_view_run_experiment_protocol_multiple_users(self, client, helpers, logg protocol = recipes.protocol.make(author=logged_in_user) helpers.add_version(protocol, visibility='private') + version1 = model.repocache.get_version(commit1.sha) + version2 = model.repocache.get_version(commit2.sha) + + other_version1 = other_model.repocache.get_version(other_commit1.sha) + other_version2 = other_model.repocache.get_version(other_commit2.sha) + response = client.get( '/entities/protocols/%d/versions/%s/runexperiments' % (protocol.pk, 'latest')) assert response.status_code == 200 assert response.context['object_list'] == [{'id': model.pk, 'name': 'mymodel1', - 'versions': [{'commit': commit2, 'tags': ['v1'], 'latest': True}, - {'commit': commit1, 'tags': [], 'latest': False}]}, + 'versions': [{'commit': version2, 'tags': ['v1'], 'latest': True}, + {'commit': version1, 'tags': [], 'latest': False}]}, ] assert response.context['other_object_list'] == [ {'id': other_model.pk, 'name': 'mymodel2', - 'versions': [{'commit': other_commit2, 'tags': ['v1'], 'latest': True}, - {'commit': other_commit1, 'tags': [], 'latest': False}]}, + 'versions': [{'commit': other_version2, 'tags': ['v1'], 'latest': True}, + {'commit': other_version1, 'tags': [], 'latest': False}]}, ] assert response.context['preposition'] == 'on' @@ -2387,13 +2420,16 @@ def test_view_run_experiment_protocol_post(self, client, helpers, logged_in_user protocol = recipes.protocol.make(author=logged_in_user) commit_protocol = helpers.add_version(protocol, visibility='public') + version1 = model.repocache.get_version(commit1.sha) + version2 = model.repocache.get_version(commit2.sha) + response = client.get( '/entities/protocols/%d/versions/%s/runexperiments' % (protocol.pk, commit_protocol.sha)) assert response.status_code == 200 assert response.context['object_list'] == [{'id': model.pk, 'name': 'mymodel1', - 'versions': [{'commit': commit2, 'tags': ['v1'], 'latest': True}, - {'commit': commit1, 'tags': [], 'latest': False}]}, + 'versions': [{'commit': version2, 'tags': ['v1'], 'latest': True}, + {'commit': version1, 'tags': [], 'latest': False}]}, ] # Test post returns correct response data = {'model_protocol_list[]': ['%d:%s' % (model.pk, commit1.sha), '%d:%s' % (model.pk, commit2.sha)], @@ -2437,14 +2473,17 @@ def test_view_run_experiment_protocol_post_exclude_existing(self, client, helper protocol=protocol, protocol_version=commit_protocol.sha) + version1 = model.repocache.get_version(commit1.sha) + version2 = model.repocache.get_version(commit2.sha) + # Test context has correct information response = client.get( '/entities/protocols/%d/versions/%s/runexperiments' % (protocol.pk, commit_protocol.sha)) assert response.status_code == 200 assert response.context['object_list'] == [{'id': model.pk, 'name': 'mymodel1', - 'versions': [{'commit': commit2, 'tags': ['v1'], 'latest': True}, - {'commit': commit1, 'tags': [], 'latest': False}]}, + 'versions': [{'commit': version2, 'tags': ['v1'], 'latest': True}, + {'commit': version1, 'tags': [], 'latest': False}]}, ] # Test post returns correct response data = {'model_protocol_list[]': ['%d:%s' % (model.pk, commit1.sha), '%d:%s' % (model.pk, commit2.sha)], @@ -2471,6 +2510,7 @@ def test_view_run_experiment_post_protocol_multiple_users(self, client, helpers, commit1 = helpers.add_version(model, visibility='public') commit2 = helpers.add_version(model, visibility='public') model.add_tag('v1', commit2.sha) + protocol = recipes.protocol.make(author=logged_in_user) commit_protocol = helpers.add_version(protocol, visibility='public') @@ -2479,19 +2519,24 @@ def test_view_run_experiment_post_protocol_multiple_users(self, client, helpers, other_commit2 = helpers.add_version(other_model, visibility='public') other_model.add_tag('v1', other_commit2.sha) + version1 = model.repocache.get_version(commit1.sha) + version2 = model.repocache.get_version(commit2.sha) + other_version1 = other_model.repocache.get_version(other_commit1.sha) + other_version2 = other_model.repocache.get_version(other_commit2.sha) + response = client.get( '/entities/protocols/%d/versions/%s/runexperiments' % (protocol.pk, commit_protocol.sha)) assert response.status_code == 200 assert response.context['object_list'] == [{'id': model.pk, 'name': 'mymodel1', - 'versions': [{'commit': commit2, 'tags': ['v1'], 'latest': True}, - {'commit': commit1, 'tags': [], 'latest': False}]}, + 'versions': [{'commit': version2, 'tags': ['v1'], 'latest': True}, + {'commit': version1, 'tags': [], 'latest': False}]}, ] assert response.context['other_object_list'] == [ {'id': other_model.pk, 'name': 'mymodel2', - 'versions': [{'commit': other_commit2, 'tags': ['v1'], 'latest': True}, - {'commit': other_commit1, 'tags': [], 'latest': False}]}, + 'versions': [{'commit': other_version2, 'tags': ['v1'], 'latest': True}, + {'commit': other_version1, 'tags': [], 'latest': False}]}, ] # Test post returns correct response data = {'model_protocol_list[]': ['%d:%s' % (model.pk, commit1.sha), @@ -2526,14 +2571,17 @@ def test_view_run_experiment_none_checked(self, client, helpers, logged_in_user) commit2 = helpers.add_version(protocol, visibility='public') protocol.add_tag('v1', commit2.sha) + version1 = protocol.repocache.get_version(commit1.sha) + version2 = protocol.repocache.get_version(commit2.sha) + # Test context has correct information response = client.get( '/entities/models/%d/versions/%s/runexperiments' % (model.pk, commit_model.sha)) assert response.status_code == 200 assert response.context['object_list'] == [{'id': protocol.pk, 'name': 'myprotocol1', - 'versions': [{'commit': commit2, 'tags': ['v1'], 'latest': True}, - {'commit': commit1, 'tags': [], 'latest': False}]}, + 'versions': [{'commit': version2, 'tags': ['v1'], 'latest': True}, + {'commit': version1, 'tags': [], 'latest': False}]}, ] # Test post returns correct response data = {'model_protocol_list[]': [], @@ -2570,14 +2618,17 @@ def test_view_run_experiment_protocol_not_latest(self, client, helpers, logged_i protocol.add_tag('p1', proto_commit1.sha) protocol.add_tag('p2', proto_commit2.sha) + version1 = model.repocache.get_version(commit1.sha) + version2 = model.repocache.get_version(commit2.sha) + # display using sha response = client.get( '/entities/protocols/%d/versions/%s/runexperiments' % (protocol.pk, proto_commit1.sha)) assert response.status_code == 200 assert response.context['object_list'] == [{'id': model.pk, 'name': 'mymodel1', - 'versions': [{'commit': commit2, 'tags': ['v1'], 'latest': True}, - {'commit': commit1, 'tags': [], 'latest': False}]}, + 'versions': [{'commit': version2, 'tags': ['v1'], 'latest': True}, + {'commit': version1, 'tags': [], 'latest': False}]}, ] assert response.context['preposition'] == 'on' diff --git a/weblab/entities/views.py b/weblab/entities/views.py index 5d7753855..daab60dd7 100644 --- a/weblab/entities/views.py +++ b/weblab/entities/views.py @@ -658,7 +658,7 @@ def get_context_data(self, **kwargs): kwargs.update(**{ 'versions': list( (list(version.tags.values_list('tag', flat=True)), - entity.repo.get_commit(version.sha)) + entity.repocache.get_version(version.sha)) for version in versions.prefetch_related('tags') ) }) @@ -1028,9 +1028,9 @@ def get_context_data(self, **kwargs): version_info = [] for version in versions.prefetch_related('tags'): tag_list = list(version.tags.values_list('tag', flat=True)) - commit = item.repo.get_commit(version.sha) - latest = item.repo.latest_commit - version_info.append({'commit': commit, 'tags': tag_list, 'latest': latest == commit}) + ver = item.repocache.get_version(version.sha) + latest_version = item.repocache.latest_version + version_info.append({'commit': ver, 'tags': tag_list, 'latest': latest_version == ver}) if item.author == self.request.user: context['object_list'].append({'id': item.id, 'name': item.name, 'versions': version_info}) else: diff --git a/weblab/templates/entities/entity_runexperiments.html b/weblab/templates/entities/entity_runexperiments.html index 49602bec9..161d2758c 100644 --- a/weblab/templates/entities/entity_runexperiments.html +++ b/weblab/templates/entities/entity_runexperiments.html @@ -91,9 +91,7 @@

Other {{ other_type }}s

by {{ entity_version.commit.author }}
created - {% with entity_version.commit.filenames|length as numfiles %} - containing {{ numfiles }} file{{ numfiles|pluralize }} - {% endwith %} + containing {{ commit.numfiles }} file{{ commit.numfiles|pluralize }}
{{ entity_version.commit.message|linebreaksbr }} diff --git a/weblab/templates/entities/entity_versions.html b/weblab/templates/entities/entity_versions.html index 60a8ceb34..98a6be929 100644 --- a/weblab/templates/entities/entity_versions.html +++ b/weblab/templates/entities/entity_versions.html @@ -28,9 +28,7 @@

Versions

by {{ commit.author }}
created - {% with commit.filenames|length as numfiles %} - containing {{ numfiles }} file{{ numfiles|pluralize}} - {% endwith %} + containing {{ commit.numfiles }} file{{ commit.numfiles|pluralize}}
{{ commit.message|linebreaksbr }} From b8991a8f25d1b71901c22832c067e39a6037cd1e Mon Sep 17 00:00:00 2001 From: Steve Roderick Date: Tue, 28 Jan 2020 15:01:30 +0000 Subject: [PATCH 25/30] Correctly prefix numfiles in template --- weblab/templates/entities/entity_runexperiments.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weblab/templates/entities/entity_runexperiments.html b/weblab/templates/entities/entity_runexperiments.html index 161d2758c..223ae0dae 100644 --- a/weblab/templates/entities/entity_runexperiments.html +++ b/weblab/templates/entities/entity_runexperiments.html @@ -91,7 +91,7 @@

Other {{ other_type }}s

by {{ entity_version.commit.author }}
created - containing {{ commit.numfiles }} file{{ commit.numfiles|pluralize }} + containing {{ entity_version.commit.numfiles }} file{{ entity_version.commit.numfiles|pluralize }}
{{ entity_version.commit.message|linebreaksbr }} From b12368d2253e8f9d0693a1a09506ddf096deb990 Mon Sep 17 00:00:00 2001 From: Jonathan Cooper Date: Tue, 28 Jan 2020 16:56:45 +0000 Subject: [PATCH 26/30] Cope if local dev system is missing repo folders! --- weblab/repocache/management/commands/populate_entity_cache.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/weblab/repocache/management/commands/populate_entity_cache.py b/weblab/repocache/management/commands/populate_entity_cache.py index 439d2c8bd..c71809c7a 100644 --- a/weblab/repocache/management/commands/populate_entity_cache.py +++ b/weblab/repocache/management/commands/populate_entity_cache.py @@ -17,4 +17,5 @@ def handle(self, *args, **options): entities = entities.filter(id__in=options['entity_id']) for entity in entities: - populate_entity_cache(entity) + if entity.repo_abs_path.exists(): + populate_entity_cache(entity) From a4ef2baae1ab0fc1300bf55a39562607197d2c05 Mon Sep 17 00:00:00 2001 From: Steve Roderick Date: Wed, 29 Jan 2020 17:20:23 +0000 Subject: [PATCH 27/30] Change url friendly method to use cache --- weblab/entities/templatetags/entities.py | 12 ++++++---- weblab/entities/tests/test_templatetags.py | 26 +++++++++++++++------- 2 files changed, 26 insertions(+), 12 deletions(-) diff --git a/weblab/entities/templatetags/entities.py b/weblab/entities/templatetags/entities.py index 2ceceb368..46ae7be0e 100644 --- a/weblab/entities/templatetags/entities.py +++ b/weblab/entities/templatetags/entities.py @@ -134,10 +134,14 @@ def _url_friendly_label(entity, commit): :param entity: Entity the commit belongs to :param commit: `git.Commit` object """ - last_tag = str(entity.repo.tag_dict.get(commit.sha, ['/'])[-1]) - if '/' in last_tag or last_tag in ['new', 'latest']: - last_tag = commit.sha - return last_tag + tags = commit.tags.last() + if tags is not None: + tag = tags.tag + if tag is None or tag in ['new', 'latest']: + return commit.sha + else: + return tag + return commit.sha @register.filter diff --git a/weblab/entities/tests/test_templatetags.py b/weblab/entities/tests/test_templatetags.py index 79e812547..405a8c4c5 100644 --- a/weblab/entities/tests/test_templatetags.py +++ b/weblab/entities/tests/test_templatetags.py @@ -35,7 +35,7 @@ def test_file_type(): @pytest.mark.django_db def test_model_urls(model_with_version): model = model_with_version - model_version = model.repo.latest_commit + model_version = model.repocache.latest_version context = {'current_namespace': 'entities'} assert entity_tags.ns_url(context, 'new', 'model') == '/entities/models/new' @@ -65,7 +65,7 @@ def test_model_urls(model_with_version): @pytest.mark.django_db def test_protocol_urls(protocol_with_version): protocol = protocol_with_version - protocol_version = protocol.repo.latest_commit + protocol_version = protocol.repocache.latest_version context = {'current_namespace': 'entities'} assert entity_tags.ns_url(context, 'new', 'protocol') == '/entities/protocols/new' @@ -117,29 +117,39 @@ def test_name_of_entity_linked_to_experiment(model_with_version, protocol_with_v @pytest.mark.django_db def test_url_friendly_label(model_with_version, helpers): commit = model_with_version.repo.latest_commit - assert entity_tags._url_friendly_label(model_with_version, commit) == commit.sha + version = model_with_version.repocache.get_version(commit.sha) + + assert entity_tags._url_friendly_label(model_with_version, version) == commit.sha model_with_version.repo.tag('v1') - assert entity_tags._url_friendly_label(model_with_version, commit) == 'v1' + populate_entity_cache(model_with_version) + + assert entity_tags._url_friendly_label(model_with_version, version) == 'v1' commit2 = helpers.add_version(model_with_version) model_with_version.repo.tag('new') - assert entity_tags._url_friendly_label(model_with_version, commit2) == commit2.sha + populate_entity_cache(model_with_version) + version2 = model_with_version.repocache.get_version(commit2.sha) + + assert entity_tags._url_friendly_label(model_with_version, version2) == commit2.sha commit3 = helpers.add_version(model_with_version) model_with_version.repo.tag('latest') - assert entity_tags._url_friendly_label(model_with_version, commit3) == commit3.sha + populate_entity_cache(model_with_version) + version3 = model_with_version.repocache.get_version(commit3.sha) + + assert entity_tags._url_friendly_label(model_with_version, version3) == commit3.sha @pytest.mark.django_db def test_url_runexperiments(model_with_version, protocol_with_version): model = model_with_version - model_commit = model.repo.latest_commit + model_commit = model.repocache.latest_version assert (entity_tags.url_run_experiments(model, model_commit) == '/entities/models/%d/versions/%s/runexperiments' % (model.pk, model_commit.sha)) protocol = protocol_with_version - protocol_commit = protocol.repo.latest_commit + protocol_commit = protocol.repocache.latest_version assert (entity_tags.url_run_experiments(protocol, protocol_commit) == '/entities/protocols/%d/versions/%s/runexperiments' % (protocol.pk, protocol_commit.sha)) From 7e91e7cb1ad2053806f36a012731bf7ff6beb1f1 Mon Sep 17 00:00:00 2001 From: Jonathan Cooper Date: Thu, 30 Jan 2020 09:31:20 +0000 Subject: [PATCH 28/30] Cache the repo property to reduce file handle count --- weblab/entities/models.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/weblab/entities/models.py b/weblab/entities/models.py index bbea62de3..0b5daaeef 100644 --- a/weblab/entities/models.py +++ b/weblab/entities/models.py @@ -6,6 +6,7 @@ from django.core.urlresolvers import reverse from django.core.validators import MinLengthValidator from django.db import models +from django.utils.functional import cached_property from guardian.shortcuts import get_objects_for_user from core.filetypes import get_file_type @@ -72,11 +73,12 @@ class Meta: def __str__(self): return self.name - @property + @cached_property def repo(self): """This entity's git repository wrapper. - Note that we do not cache this property as this can lead to too many open files. + Caching this property actually reduces the number of open files, but there's still a risk + of running out of file handles eventually. See also https://gitpython.readthedocs.io/en/stable/intro.html#leakage-of-system-resources """ return Repository(self.repo_abs_path) From 6ce08ec053f1b6e082931c99549c9bdf3dac8f8c Mon Sep 17 00:00:00 2001 From: Jonathan Cooper Date: Thu, 30 Jan 2020 09:40:43 +0000 Subject: [PATCH 29/30] Fix version links in run experiments view --- weblab/entities/tests/test_views.py | 17 +++++++++++++++++ weblab/entities/views.py | 6 ++++-- .../entities/entity_runexperiments.html | 4 ++-- 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/weblab/entities/tests/test_views.py b/weblab/entities/tests/test_views.py index 1ca9ee3df..8eefff7b4 100644 --- a/weblab/entities/tests/test_views.py +++ b/weblab/entities/tests/test_views.py @@ -2111,6 +2111,7 @@ def test_view_run_experiment_model(self, client, helpers, logged_in_user): '/entities/models/%d/versions/%s/runexperiments' % (model.pk, 'latest')) assert response.status_code == 200 assert response.context['object_list'] == [{'id': protocol.pk, + 'entity': protocol, 'name': 'myprotocol1', 'versions': [{'commit': version2, 'tags': ['v1'], 'latest': True}, {'commit': version1, 'tags': [], 'latest': False}]}, @@ -2142,12 +2143,14 @@ def test_view_run_experiment_model_multiple_users(self, client, helpers, logged_ '/entities/models/%d/versions/%s/runexperiments' % (model.pk, 'latest')) assert response.status_code == 200 assert response.context['object_list'] == [{'id': protocol.pk, + 'entity': protocol, 'name': 'myprotocol1', 'versions': [{'commit': version2, 'tags': ['v1'], 'latest': True}, {'commit': version1, 'tags': [], 'latest': False}]}, ] assert response.context['other_object_list'] == [ {'id': other_protocol.pk, + 'entity': other_protocol, 'name': 'myprotocol2', 'versions': [{'commit': other_version2, 'tags': ['v1'], 'latest': True}, {'commit': other_version1, 'tags': [], 'latest': False}]}, @@ -2172,6 +2175,7 @@ def test_view_run_experiment_model_post(self, client, helpers, logged_in_user): '/entities/models/%d/versions/%s/runexperiments' % (model.pk, commit_model.sha)) assert response.status_code == 200 assert response.context['object_list'] == [{'id': protocol.pk, + 'entity': protocol, 'name': 'myprotocol1', 'versions': [{'commit': version2, 'tags': ['v1'], 'latest': True}, {'commit': version1, 'tags': [], 'latest': False}]}, @@ -2220,6 +2224,7 @@ def test_view_run_experiment_model_post_exclude_existing(self, client, helpers, '/entities/models/%d/versions/%s/runexperiments' % (model.pk, commit_model.sha)) assert response.status_code == 200 assert response.context['object_list'] == [{'id': protocol.pk, + 'entity': protocol, 'name': 'myprotocol1', 'versions': [{'commit': version2, 'tags': ['v1'], 'latest': True}, {'commit': version1, 'tags': [], 'latest': False}]}, @@ -2270,12 +2275,14 @@ def test_view_run_experiment_post_model_multiple_users(self, client, helpers, lo '/entities/models/%d/versions/%s/runexperiments' % (model.pk, commit_model.sha)) assert response.status_code == 200 assert response.context['object_list'] == [{'id': protocol.pk, + 'entity': protocol, 'name': 'myprotocol1', 'versions': [{'commit': version2, 'tags': ['v1'], 'latest': True}, {'commit': version1, 'tags': [], 'latest': False}]}, ] assert response.context['other_object_list'] == [ {'id': other_protocol.pk, + 'entity': other_protocol, 'name': 'myprotocol2', 'versions': [{'commit': other_version2, 'tags': ['v1'], 'latest': True}, {'commit': other_version1, 'tags': [], 'latest': False}]}, @@ -2324,6 +2331,7 @@ def test_view_run_experiment_model_not_latest(self, client, helpers, logged_in_u '/entities/models/%d/versions/%s/runexperiments' % (model.pk, 'model_v1')) assert response.status_code == 200 assert response.context['object_list'] == [{'id': protocol.pk, + 'entity': protocol, 'name': 'myprotocol1', 'versions': [{'commit': version2, 'tags': ['v1'], 'latest': True}, {'commit': version1, 'tags': [], 'latest': False}]}, @@ -2368,6 +2376,7 @@ def test_view_run_experiment_protocol(self, client, helpers, logged_in_user): '/entities/protocols/%d/versions/%s/runexperiments' % (protocol.pk, 'latest')) assert response.status_code == 200 assert response.context['object_list'] == [{'id': model.pk, + 'entity': model, 'name': 'mymodel1', 'versions': [{'commit': version2, 'tags': ['v1'], 'latest': True}, {'commit': version1, 'tags': [], 'latest': False}]}, @@ -2399,12 +2408,14 @@ def test_view_run_experiment_protocol_multiple_users(self, client, helpers, logg '/entities/protocols/%d/versions/%s/runexperiments' % (protocol.pk, 'latest')) assert response.status_code == 200 assert response.context['object_list'] == [{'id': model.pk, + 'entity': model, 'name': 'mymodel1', 'versions': [{'commit': version2, 'tags': ['v1'], 'latest': True}, {'commit': version1, 'tags': [], 'latest': False}]}, ] assert response.context['other_object_list'] == [ {'id': other_model.pk, + 'entity': other_model, 'name': 'mymodel2', 'versions': [{'commit': other_version2, 'tags': ['v1'], 'latest': True}, {'commit': other_version1, 'tags': [], 'latest': False}]}, @@ -2427,6 +2438,7 @@ def test_view_run_experiment_protocol_post(self, client, helpers, logged_in_user '/entities/protocols/%d/versions/%s/runexperiments' % (protocol.pk, commit_protocol.sha)) assert response.status_code == 200 assert response.context['object_list'] == [{'id': model.pk, + 'entity': model, 'name': 'mymodel1', 'versions': [{'commit': version2, 'tags': ['v1'], 'latest': True}, {'commit': version1, 'tags': [], 'latest': False}]}, @@ -2481,6 +2493,7 @@ def test_view_run_experiment_protocol_post_exclude_existing(self, client, helper '/entities/protocols/%d/versions/%s/runexperiments' % (protocol.pk, commit_protocol.sha)) assert response.status_code == 200 assert response.context['object_list'] == [{'id': model.pk, + 'entity': model, 'name': 'mymodel1', 'versions': [{'commit': version2, 'tags': ['v1'], 'latest': True}, {'commit': version1, 'tags': [], 'latest': False}]}, @@ -2528,12 +2541,14 @@ def test_view_run_experiment_post_protocol_multiple_users(self, client, helpers, '/entities/protocols/%d/versions/%s/runexperiments' % (protocol.pk, commit_protocol.sha)) assert response.status_code == 200 assert response.context['object_list'] == [{'id': model.pk, + 'entity': model, 'name': 'mymodel1', 'versions': [{'commit': version2, 'tags': ['v1'], 'latest': True}, {'commit': version1, 'tags': [], 'latest': False}]}, ] assert response.context['other_object_list'] == [ {'id': other_model.pk, + 'entity': other_model, 'name': 'mymodel2', 'versions': [{'commit': other_version2, 'tags': ['v1'], 'latest': True}, {'commit': other_version1, 'tags': [], 'latest': False}]}, @@ -2579,6 +2594,7 @@ def test_view_run_experiment_none_checked(self, client, helpers, logged_in_user) '/entities/models/%d/versions/%s/runexperiments' % (model.pk, commit_model.sha)) assert response.status_code == 200 assert response.context['object_list'] == [{'id': protocol.pk, + 'entity': protocol, 'name': 'myprotocol1', 'versions': [{'commit': version2, 'tags': ['v1'], 'latest': True}, {'commit': version1, 'tags': [], 'latest': False}]}, @@ -2626,6 +2642,7 @@ def test_view_run_experiment_protocol_not_latest(self, client, helpers, logged_i '/entities/protocols/%d/versions/%s/runexperiments' % (protocol.pk, proto_commit1.sha)) assert response.status_code == 200 assert response.context['object_list'] == [{'id': model.pk, + 'entity': model, 'name': 'mymodel1', 'versions': [{'commit': version2, 'tags': ['v1'], 'latest': True}, {'commit': version1, 'tags': [], 'latest': False}]}, diff --git a/weblab/entities/views.py b/weblab/entities/views.py index daab60dd7..269566792 100644 --- a/weblab/entities/views.py +++ b/weblab/entities/views.py @@ -1032,9 +1032,11 @@ def get_context_data(self, **kwargs): latest_version = item.repocache.latest_version version_info.append({'commit': ver, 'tags': tag_list, 'latest': latest_version == ver}) if item.author == self.request.user: - context['object_list'].append({'id': item.id, 'name': item.name, 'versions': version_info}) + context['object_list'].append( + {'entity': item, 'id': item.id, 'name': item.name, 'versions': version_info}) else: - context['other_object_list'].append({'id': item.id, 'name': item.name, 'versions': version_info}) + context['other_object_list'].append( + {'entity': item, 'id': item.id, 'name': item.name, 'versions': version_info}) return context def post(self, request, *args, **kwargs): diff --git a/weblab/templates/entities/entity_runexperiments.html b/weblab/templates/entities/entity_runexperiments.html index 223ae0dae..0ccf728bf 100644 --- a/weblab/templates/entities/entity_runexperiments.html +++ b/weblab/templates/entities/entity_runexperiments.html @@ -46,7 +46,7 @@

Your {{ other_type }}s

name="model_protocol_list[]"/> {% endif %} - + {% include "./includes/version_name.html" with tags=entity_version.tags version=entity_version.commit only %} @@ -84,7 +84,7 @@

Other {{ other_type }}s

name="model_protocol_list[]"/> {% endif %} - + {% include "./includes/version_name.html" with tags=entity_version.tags version=entity_version.commit only %} From 385c6f1cb3077361a50f0de4986b34c981d25264 Mon Sep 17 00:00:00 2001 From: Steve Roderick Date: Thu, 30 Jan 2020 10:06:17 +0000 Subject: [PATCH 30/30] Changed docstring in url friendly method --- weblab/entities/templatetags/entities.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/weblab/entities/templatetags/entities.py b/weblab/entities/templatetags/entities.py index 46ae7be0e..40b168d81 100644 --- a/weblab/entities/templatetags/entities.py +++ b/weblab/entities/templatetags/entities.py @@ -127,21 +127,21 @@ def name_of_protocol(experiment): return '%s @ %s' % (protocol.name, protocol_version) -def _url_friendly_label(entity, commit): +def _url_friendly_label(entity, version): """ Get URL-friendly version label for a commit :param entity: Entity the commit belongs to - :param commit: `git.Commit` object + :param version: CachedEntityVersion object """ - tags = commit.tags.last() + tags = version.tags.last() if tags is not None: tag = tags.tag if tag is None or tag in ['new', 'latest']: - return commit.sha + return version.sha else: return tag - return commit.sha + return version.sha @register.filter