diff --git a/.travis.yml b/.travis.yml
index cee0575d4..e083b6ed7 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -15,7 +15,7 @@ before_script:
install:
pip install -r requirements/test.txt
script:
- - pytest --cov=weblab --cov-config=weblab/.coveragerc weblab
+ - pytest --fail-on-template-vars --cov=weblab --cov-config=weblab/.coveragerc weblab
- flake8 weblab
- isort --verbose --check-only --diff --recursive weblab
after_success:
diff --git a/weblab/config/settings/dev.py b/weblab/config/settings/dev.py
index 58f31e423..057aa2496 100644
--- a/weblab/config/settings/dev.py
+++ b/weblab/config/settings/dev.py
@@ -80,3 +80,14 @@
app.split('.')[0]: copy.deepcopy(local_logger_conf)
for app in LOCAL_APPS
})
+
+
+class InvalidStringShowWarning(str):
+ def __mod__(self, other):
+ import logging
+ logger = logging.getLogger(__name__)
+ logger.warning("In template, undefined variable or unknown value for: '%s'" % (other,))
+ return ""
+
+
+TEMPLATES[0]['OPTIONS']['string_if_invalid'] = InvalidStringShowWarning("%s")
diff --git a/weblab/entities/models.py b/weblab/entities/models.py
index e375b3c9c..0b5daaeef 100644
--- a/weblab/entities/models.py
+++ b/weblab/entities/models.py
@@ -6,6 +6,7 @@
from django.core.urlresolvers import reverse
from django.core.validators import MinLengthValidator
from django.db import models
+from django.utils.functional import cached_property
from guardian.shortcuts import get_objects_for_user
from core.filetypes import get_file_type
@@ -72,11 +73,12 @@ class Meta:
def __str__(self):
return self.name
- @property
+ @cached_property
def repo(self):
"""This entity's git repository wrapper.
- Note that we do not cache this property as this can lead to too many open files.
+ Caching this property actually reduces the number of open files, but there's still a risk
+ of running out of file handles eventually.
See also https://gitpython.readthedocs.io/en/stable/intro.html#leakage-of-system-resources
"""
return Repository(self.repo_abs_path)
@@ -92,11 +94,17 @@ def repo_abs_path(self):
self.author.get_storage_dir('repo'), '%ss' % self.entity_type, str(self.id)
)
- def nice_version(self, commit):
- version = self.repo.get_name_for_commit(commit)
- if len(version) > 20:
- version = version[:8] + '...'
- return version
+ def nice_version(self, sha_or_tag):
+ """
+ Returns tag/sha with ellipses
+
+ :param sha_or_tag: version sha or tag string
+ :return version_name: string with the sha_or_tag formatted
+ """
+ version_name = self.repocache.get_name_for_version(sha_or_tag)
+ if len(version_name) > 20:
+ version_name = version_name[:8] + '...'
+ return version_name
def get_visibility_from_repo(self, commit):
"""
@@ -296,7 +304,7 @@ def get_version_json(self, commit, ns):
'visibility': self.get_version_visibility(commit.sha, default=self.DEFAULT_VISIBILITY),
'created': commit.timestamp,
'name': self.name,
- 'version': self.repo.get_name_for_commit(commit.sha), # TODO #191 use repocache instead
+ 'version': self.repocache.get_name_for_version(commit.sha),
'files': files,
'commitMessage': commit.message,
'numFiles': len(files),
diff --git a/weblab/entities/repository.py b/weblab/entities/repository.py
index 1e8fed3a5..35b8fc3da 100644
--- a/weblab/entities/repository.py
+++ b/weblab/entities/repository.py
@@ -138,19 +138,6 @@ def tag_dict(self):
tags.setdefault(tag.commit.hexsha, []).append(tag)
return tags
- def get_name_for_commit(self, version):
- """Get a human-friendly display name for the given version
-
- :param version: Revision specification (sha, branch name, tag etc.)
- or 'latest' to get latest revision
- :return: tag for this commit, if any, or version if not
- """
- commit = self.get_commit(version)
- for tag in self._repo.tags:
- if tag.commit == commit._commit:
- return tag.name
- return version
-
def hard_reset(self):
"""
Reset the working tree
diff --git a/weblab/entities/templatetags/entities.py b/weblab/entities/templatetags/entities.py
index 6e2e13598..40b168d81 100644
--- a/weblab/entities/templatetags/entities.py
+++ b/weblab/entities/templatetags/entities.py
@@ -116,28 +116,32 @@ def url_entity_diff_base(context, entity_type):
@register.filter
def name_of_model(experiment):
model = experiment.model
- model_version = model.repo.get_name_for_commit(experiment.model_version)
+ model_version = model.repocache.get_name_for_version(experiment.model_version)
return '%s @ %s' % (model.name, model_version)
@register.filter
def name_of_protocol(experiment):
protocol = experiment.protocol
- protocol_version = protocol.repo.get_name_for_commit(experiment.protocol_version)
+ protocol_version = protocol.repocache.get_name_for_version(experiment.protocol_version)
return '%s @ %s' % (protocol.name, protocol_version)
-def _url_friendly_label(entity, commit):
+def _url_friendly_label(entity, version):
"""
Get URL-friendly version label for a commit
:param entity: Entity the commit belongs to
- :param commit: `git.Commit` object
+ :param version: CachedEntityVersion object
"""
- last_tag = str(entity.repo.tag_dict.get(commit.sha, ['/'])[-1])
- if '/' in last_tag or last_tag in ['new', 'latest']:
- last_tag = commit.sha
- return last_tag
+ tags = version.tags.last()
+ if tags is not None:
+ tag = tags.tag
+ if tag is None or tag in ['new', 'latest']:
+ return version.sha
+ else:
+ return tag
+ return version.sha
@register.filter
diff --git a/weblab/entities/tests/test_models.py b/weblab/entities/tests/test_models.py
index ebfe1c9ad..58d57a929 100644
--- a/weblab/entities/tests/test_models.py
+++ b/weblab/entities/tests/test_models.py
@@ -97,6 +97,7 @@ def test_nice_version(self, model_with_version):
assert model_with_version.nice_version(commit) == '%s...' % commit[:8]
model_with_version.repo.tag('v1')
+ populate_entity_cache(model_with_version)
assert model_with_version.nice_version(commit) == 'v1'
def test_set_and_get_version_visibility(self, model_with_version):
diff --git a/weblab/entities/tests/test_repository.py b/weblab/entities/tests/test_repository.py
index df22dbb73..959bca818 100644
--- a/weblab/entities/tests/test_repository.py
+++ b/weblab/entities/tests/test_repository.py
@@ -77,19 +77,6 @@ def test_tag(self, repo, repo_file, author):
with pytest.raises(GitCommandError):
repo.tag('v1')
- def test_name_for_commit(self, repo, repo_file, author):
- repo.add_file(repo_file)
- commit = repo.commit('commit_message', author)
-
- assert repo.get_name_for_commit(commit.sha) == commit.sha
- assert repo.get_name_for_commit('latest') == 'latest'
-
- repo.tag('v1')
-
- assert repo.get_name_for_commit(commit.sha) == 'v1'
- assert repo.get_name_for_commit('v1') == 'v1'
- assert repo.get_name_for_commit('latest') == 'v1'
-
def test_has_changes(self, repo, repo_file, author):
assert not repo.has_changes
repo.add_file(repo_file)
diff --git a/weblab/entities/tests/test_templatetags.py b/weblab/entities/tests/test_templatetags.py
index ae3c9191f..405a8c4c5 100644
--- a/weblab/entities/tests/test_templatetags.py
+++ b/weblab/entities/tests/test_templatetags.py
@@ -2,6 +2,7 @@
import entities.templatetags.entities as entity_tags
from core import recipes
+from repocache.populate import populate_entity_cache
def test_human_readable_bytes():
@@ -34,7 +35,7 @@ def test_file_type():
@pytest.mark.django_db
def test_model_urls(model_with_version):
model = model_with_version
- model_version = model.repo.latest_commit
+ model_version = model.repocache.latest_version
context = {'current_namespace': 'entities'}
assert entity_tags.ns_url(context, 'new', 'model') == '/entities/models/new'
@@ -64,7 +65,7 @@ def test_model_urls(model_with_version):
@pytest.mark.django_db
def test_protocol_urls(protocol_with_version):
protocol = protocol_with_version
- protocol_version = protocol.repo.latest_commit
+ protocol_version = protocol.repocache.latest_version
context = {'current_namespace': 'entities'}
assert entity_tags.ns_url(context, 'new', 'protocol') == '/entities/protocols/new'
@@ -98,6 +99,8 @@ def test_protocol_urls(protocol_with_version):
def test_name_of_entity_linked_to_experiment(model_with_version, protocol_with_version):
model_with_version.repo.tag('v1')
protocol_with_version.repo.tag('v2')
+ populate_entity_cache(model_with_version)
+ populate_entity_cache(protocol_with_version)
exp = recipes.experiment_version.make(
status='SUCCESS',
@@ -114,29 +117,39 @@ def test_name_of_entity_linked_to_experiment(model_with_version, protocol_with_v
@pytest.mark.django_db
def test_url_friendly_label(model_with_version, helpers):
commit = model_with_version.repo.latest_commit
- assert entity_tags._url_friendly_label(model_with_version, commit) == commit.sha
+ version = model_with_version.repocache.get_version(commit.sha)
+
+ assert entity_tags._url_friendly_label(model_with_version, version) == commit.sha
model_with_version.repo.tag('v1')
- assert entity_tags._url_friendly_label(model_with_version, commit) == 'v1'
+ populate_entity_cache(model_with_version)
+
+ assert entity_tags._url_friendly_label(model_with_version, version) == 'v1'
commit2 = helpers.add_version(model_with_version)
model_with_version.repo.tag('new')
- assert entity_tags._url_friendly_label(model_with_version, commit2) == commit2.sha
+ populate_entity_cache(model_with_version)
+ version2 = model_with_version.repocache.get_version(commit2.sha)
+
+ assert entity_tags._url_friendly_label(model_with_version, version2) == commit2.sha
commit3 = helpers.add_version(model_with_version)
model_with_version.repo.tag('latest')
- assert entity_tags._url_friendly_label(model_with_version, commit3) == commit3.sha
+ populate_entity_cache(model_with_version)
+ version3 = model_with_version.repocache.get_version(commit3.sha)
+
+ assert entity_tags._url_friendly_label(model_with_version, version3) == commit3.sha
@pytest.mark.django_db
def test_url_runexperiments(model_with_version, protocol_with_version):
model = model_with_version
- model_commit = model.repo.latest_commit
+ model_commit = model.repocache.latest_version
assert (entity_tags.url_run_experiments(model, model_commit) ==
'/entities/models/%d/versions/%s/runexperiments' % (model.pk, model_commit.sha))
protocol = protocol_with_version
- protocol_commit = protocol.repo.latest_commit
+ protocol_commit = protocol.repocache.latest_version
assert (entity_tags.url_run_experiments(protocol, protocol_commit) ==
'/entities/protocols/%d/versions/%s/runexperiments' % (protocol.pk, protocol_commit.sha))
diff --git a/weblab/entities/tests/test_views.py b/weblab/entities/tests/test_views.py
index b1fc4ed6f..8eefff7b4 100644
--- a/weblab/entities/tests/test_views.py
+++ b/weblab/entities/tests/test_views.py
@@ -1,5 +1,6 @@
import io
import json
+import os
import uuid
import zipfile
from datetime import timedelta
@@ -19,6 +20,7 @@
from entities.models import AnalysisTask, ModelEntity, ProtocolEntity
from experiments.models import Experiment, PlannedExperiment
from repocache.models import ProtocolInterface
+from repocache.populate import populate_entity_cache
@pytest.fixture
@@ -152,43 +154,41 @@ def check(self, client, url, version, tags):
def test_view_entity_version(self, client, logged_in_user, helpers):
model = recipes.model.make()
helpers.add_version(model, visibility='public')
- commit = model.repo.latest_commit
- self.check(client, '/entities/models/%d/versions/%s' % (model.pk, commit.sha),
- commit, [])
+ version = model.repocache.latest_version
+ self.check(client, '/entities/models/%d/versions/%s' % (model.pk, version.sha),
+ version, [])
self.check(client, '/entities/models/%d/versions/latest' % model.pk,
- commit, [])
+ version, [])
# Now add a second version with tag
- assert len(list(model.repo.commits)) == 1
- commit2 = helpers.add_version(model, visibility='public')
- model.add_tag('my_tag', commit2.sha)
+ assert model.repocache.versions.count() == 1
+ version2 = helpers.add_version(model, visibility='public')
+ model.add_tag('my_tag', version2.sha)
# Commits are yielded newest first
- assert len(list(model.repo.commits)) == 2
- assert commit == list(model.repo.commits)[-1]
- commit = model.repo.latest_commit
+ assert model.repocache.versions.count() == 2
+ assert version == model.repocache.versions.last()
+ version = model.repocache.latest_version
- self.check(client, '/entities/models/%d/versions/%s' % (model.pk, commit.sha),
- commit, ['my_tag'])
+ self.check(client, '/entities/models/%d/versions/%s' % (model.pk, version.sha),
+ version, ['my_tag'])
self.check(client, '/entities/models/%d/versions/%s' % (model.pk, 'my_tag'),
- commit, ['my_tag'])
+ version, ['my_tag'])
self.check(client, '/entities/models/%d/versions/latest' % model.pk,
- commit, ['my_tag'])
+ version, ['my_tag'])
def test_version_with_two_tags(self, client, helpers):
model = recipes.model.make()
helpers.add_version(model, visibility='public')
- commit = model.repo.latest_commit
- model.add_tag('tag1', commit.sha)
- model.add_tag('tag2', commit.sha)
- self.check(client, '/entities/models/%d/versions/%s' % (model.pk, commit.sha),
- commit, ['tag1', 'tag2'])
- self.check(client, '/entities/models/%d/versions/%s' % (model.pk, 'tag1'),
- commit, ['tag1', 'tag2'])
+ version = model.repocache.latest_version
+ model.add_tag('tag1', version.sha)
+ model.add_tag('tag2', version.sha)
+ self.check(client, '/entities/models/%d/versions/%s' % (model.pk, version.sha), version, ['tag1', 'tag2'])
+ self.check(client, '/entities/models/%d/versions/%s' % (model.pk, 'tag1'), version, ['tag1', 'tag2'])
self.check(client, '/entities/models/%d/versions/%s' % (model.pk, 'tag2'),
- commit, ['tag1', 'tag2'])
+ version, ['tag1', 'tag2'])
self.check(client, '/entities/models/%d/versions/latest' % model.pk,
- commit, ['tag1', 'tag2'])
+ version, ['tag1', 'tag2'])
def test_shows_correct_visibility(self, client, logged_in_user, model_with_version):
model = model_with_version
@@ -305,6 +305,7 @@ def test_version_json(self, client, logged_in_user, helpers, can_create_expt, is
version = helpers.add_version(model)
model.set_version_visibility(version.sha, 'public')
model.repo.tag('v1')
+ populate_entity_cache(model)
planned_expt = PlannedExperiment(
model=model, model_version=version.sha,
protocol=recipes.protocol.make(), protocol_version=uuid.uuid4(),
@@ -805,19 +806,18 @@ def test_view_entity_version_list(self, client, helpers):
commit1 = helpers.add_version(model, visibility='public')
commit2 = helpers.add_version(model, visibility='moderated')
model.add_tag('v1', commit2.sha)
-
response = client.get('/entities/models/%d/versions/' % model.pk)
assert response.status_code == 200
assert response.context['versions'] == [
- (['v1'], commit2),
- ([], commit1),
+ (['v1'], model.repocache.get_version(commit2.sha)),
+ ([], model.repocache.get_version(commit1.sha)),
]
def test_only_shows_visible_versions(self, client, helpers):
model = recipes.model.make()
- helpers.add_version(model, visibility='private')
- commit2 = helpers.add_version(model, visibility='public')
- helpers.add_version(model, visibility='private')
+ helpers.add_fake_version(model, visibility='private')
+ commit2 = helpers.add_fake_version(model, visibility='public')
+ helpers.add_fake_version(model, visibility='private')
response = client.get('/entities/models/%d/versions/' % model.pk)
assert response.status_code == 200
@@ -1967,8 +1967,8 @@ def test_bives_diff(self, mock_post, client, helpers):
'https://bives.bio.informatik.uni-rostock.de/',
json={
'files': [
- 'v1 contents\n',
- 'v2 contents\n'
+ 'v1 contents' + os.linesep,
+ 'v2 contents' + os.linesep
],
'commands': [
'compHierarchyJson',
@@ -2104,13 +2104,17 @@ def test_view_run_experiment_model(self, client, helpers, logged_in_user):
commit2 = helpers.add_version(protocol, visibility='public')
protocol.add_tag('v1', commit2.sha)
+ version1 = protocol.repocache.get_version(commit1.sha)
+ version2 = protocol.repocache.get_version(commit2.sha)
+
response = client.get(
'/entities/models/%d/versions/%s/runexperiments' % (model.pk, 'latest'))
assert response.status_code == 200
assert response.context['object_list'] == [{'id': protocol.pk,
+ 'entity': protocol,
'name': 'myprotocol1',
- 'versions': [{'commit': commit2, 'tags': ['v1'], 'latest': True},
- {'commit': commit1, 'tags': [], 'latest': False}]},
+ 'versions': [{'commit': version2, 'tags': ['v1'], 'latest': True},
+ {'commit': version1, 'tags': [], 'latest': False}]},
]
assert response.context['preposition'] == 'under'
@@ -2129,19 +2133,27 @@ def test_view_run_experiment_model_multiple_users(self, client, helpers, logged_
other_commit2 = helpers.add_version(other_protocol, visibility='public')
other_protocol.add_tag('v1', other_commit2.sha)
+ version1 = protocol.repocache.get_version(commit1.sha)
+ version2 = protocol.repocache.get_version(commit2.sha)
+
+ other_version1 = other_protocol.repocache.get_version(other_commit1.sha)
+ other_version2 = other_protocol.repocache.get_version(other_commit2.sha)
+
response = client.get(
'/entities/models/%d/versions/%s/runexperiments' % (model.pk, 'latest'))
assert response.status_code == 200
assert response.context['object_list'] == [{'id': protocol.pk,
+ 'entity': protocol,
'name': 'myprotocol1',
- 'versions': [{'commit': commit2, 'tags': ['v1'], 'latest': True},
- {'commit': commit1, 'tags': [], 'latest': False}]},
+ 'versions': [{'commit': version2, 'tags': ['v1'], 'latest': True},
+ {'commit': version1, 'tags': [], 'latest': False}]},
]
assert response.context['other_object_list'] == [
{'id': other_protocol.pk,
+ 'entity': other_protocol,
'name': 'myprotocol2',
- 'versions': [{'commit': other_commit2, 'tags': ['v1'], 'latest': True},
- {'commit': other_commit1, 'tags': [], 'latest': False}]},
+ 'versions': [{'commit': other_version2, 'tags': ['v1'], 'latest': True},
+ {'commit': other_version1, 'tags': [], 'latest': False}]},
]
assert response.context['preposition'] == 'under'
@@ -2155,14 +2167,18 @@ def test_view_run_experiment_model_post(self, client, helpers, logged_in_user):
commit2 = helpers.add_version(protocol, visibility='public')
protocol.add_tag('v1', commit2.sha)
+ version1 = protocol.repocache.get_version(commit1.sha)
+ version2 = protocol.repocache.get_version(commit2.sha)
+
# Test context has correct information
response = client.get(
'/entities/models/%d/versions/%s/runexperiments' % (model.pk, commit_model.sha))
assert response.status_code == 200
assert response.context['object_list'] == [{'id': protocol.pk,
+ 'entity': protocol,
'name': 'myprotocol1',
- 'versions': [{'commit': commit2, 'tags': ['v1'], 'latest': True},
- {'commit': commit1, 'tags': [], 'latest': False}]},
+ 'versions': [{'commit': version2, 'tags': ['v1'], 'latest': True},
+ {'commit': version1, 'tags': [], 'latest': False}]},
]
# Test post returns correct response
data = {'model_protocol_list[]': ['%d:%s' % (protocol.pk, commit2.sha)],
@@ -2193,6 +2209,9 @@ def test_view_run_experiment_model_post_exclude_existing(self, client, helpers,
commit2 = helpers.add_version(protocol, visibility='public')
protocol.add_tag('v1', commit2.sha)
+ version1 = protocol.repocache.get_version(commit1.sha)
+ version2 = protocol.repocache.get_version(commit2.sha)
+
recipes.experiment_version.make(
status='SUCCESS',
experiment__model=model,
@@ -2205,9 +2224,10 @@ def test_view_run_experiment_model_post_exclude_existing(self, client, helpers,
'/entities/models/%d/versions/%s/runexperiments' % (model.pk, commit_model.sha))
assert response.status_code == 200
assert response.context['object_list'] == [{'id': protocol.pk,
+ 'entity': protocol,
'name': 'myprotocol1',
- 'versions': [{'commit': commit2, 'tags': ['v1'], 'latest': True},
- {'commit': commit1, 'tags': [], 'latest': False}]},
+ 'versions': [{'commit': version2, 'tags': ['v1'], 'latest': True},
+ {'commit': version1, 'tags': [], 'latest': False}]},
]
# Test post returns correct response
data = {'model_protocol_list[]': ['%d:%s' % (protocol.pk, commit1.sha),
@@ -2244,20 +2264,28 @@ def test_view_run_experiment_post_model_multiple_users(self, client, helpers, lo
other_commit2 = helpers.add_version(other_protocol, visibility='public')
other_protocol.add_tag('v1', other_commit2.sha)
+ version1 = protocol.repocache.get_version(commit1.sha)
+ version2 = protocol.repocache.get_version(commit2.sha)
+
+ other_version1 = other_protocol.repocache.get_version(other_commit1.sha)
+ other_version2 = other_protocol.repocache.get_version(other_commit2.sha)
+
# check context
response = client.get(
'/entities/models/%d/versions/%s/runexperiments' % (model.pk, commit_model.sha))
assert response.status_code == 200
assert response.context['object_list'] == [{'id': protocol.pk,
+ 'entity': protocol,
'name': 'myprotocol1',
- 'versions': [{'commit': commit2, 'tags': ['v1'], 'latest': True},
- {'commit': commit1, 'tags': [], 'latest': False}]},
+ 'versions': [{'commit': version2, 'tags': ['v1'], 'latest': True},
+ {'commit': version1, 'tags': [], 'latest': False}]},
]
assert response.context['other_object_list'] == [
{'id': other_protocol.pk,
+ 'entity': other_protocol,
'name': 'myprotocol2',
- 'versions': [{'commit': other_commit2, 'tags': ['v1'], 'latest': True},
- {'commit': other_commit1, 'tags': [], 'latest': False}]},
+ 'versions': [{'commit': other_version2, 'tags': ['v1'], 'latest': True},
+ {'commit': other_version1, 'tags': [], 'latest': False}]},
]
# Test post returns correct response
data = {'model_protocol_list[]': ['%d:%s' % (protocol.pk, commit2.sha),
@@ -2295,14 +2323,18 @@ def test_view_run_experiment_model_not_latest(self, client, helpers, logged_in_u
commit2 = helpers.add_version(protocol, visibility='public')
protocol.add_tag('v1', commit2.sha)
+ version1 = protocol.repocache.get_version(commit1.sha)
+ version2 = protocol.repocache.get_version(commit2.sha)
+
# display page using tag
response = client.get(
'/entities/models/%d/versions/%s/runexperiments' % (model.pk, 'model_v1'))
assert response.status_code == 200
assert response.context['object_list'] == [{'id': protocol.pk,
+ 'entity': protocol,
'name': 'myprotocol1',
- 'versions': [{'commit': commit2, 'tags': ['v1'], 'latest': True},
- {'commit': commit1, 'tags': [], 'latest': False}]},
+ 'versions': [{'commit': version2, 'tags': ['v1'], 'latest': True},
+ {'commit': version1, 'tags': [], 'latest': False}]},
]
assert response.context['preposition'] == 'under'
@@ -2337,13 +2369,17 @@ def test_view_run_experiment_protocol(self, client, helpers, logged_in_user):
protocol = recipes.protocol.make(author=logged_in_user)
helpers.add_version(protocol, visibility='private')
+ version1 = model.repocache.get_version(commit1.sha)
+ version2 = model.repocache.get_version(commit2.sha)
+
response = client.get(
'/entities/protocols/%d/versions/%s/runexperiments' % (protocol.pk, 'latest'))
assert response.status_code == 200
assert response.context['object_list'] == [{'id': model.pk,
+ 'entity': model,
'name': 'mymodel1',
- 'versions': [{'commit': commit2, 'tags': ['v1'], 'latest': True},
- {'commit': commit1, 'tags': [], 'latest': False}]},
+ 'versions': [{'commit': version2, 'tags': ['v1'], 'latest': True},
+ {'commit': version1, 'tags': [], 'latest': False}]},
]
assert response.context['preposition'] == 'on'
@@ -2362,19 +2398,27 @@ def test_view_run_experiment_protocol_multiple_users(self, client, helpers, logg
protocol = recipes.protocol.make(author=logged_in_user)
helpers.add_version(protocol, visibility='private')
+ version1 = model.repocache.get_version(commit1.sha)
+ version2 = model.repocache.get_version(commit2.sha)
+
+ other_version1 = other_model.repocache.get_version(other_commit1.sha)
+ other_version2 = other_model.repocache.get_version(other_commit2.sha)
+
response = client.get(
'/entities/protocols/%d/versions/%s/runexperiments' % (protocol.pk, 'latest'))
assert response.status_code == 200
assert response.context['object_list'] == [{'id': model.pk,
+ 'entity': model,
'name': 'mymodel1',
- 'versions': [{'commit': commit2, 'tags': ['v1'], 'latest': True},
- {'commit': commit1, 'tags': [], 'latest': False}]},
+ 'versions': [{'commit': version2, 'tags': ['v1'], 'latest': True},
+ {'commit': version1, 'tags': [], 'latest': False}]},
]
assert response.context['other_object_list'] == [
{'id': other_model.pk,
+ 'entity': other_model,
'name': 'mymodel2',
- 'versions': [{'commit': other_commit2, 'tags': ['v1'], 'latest': True},
- {'commit': other_commit1, 'tags': [], 'latest': False}]},
+ 'versions': [{'commit': other_version2, 'tags': ['v1'], 'latest': True},
+ {'commit': other_version1, 'tags': [], 'latest': False}]},
]
assert response.context['preposition'] == 'on'
@@ -2387,13 +2431,17 @@ def test_view_run_experiment_protocol_post(self, client, helpers, logged_in_user
protocol = recipes.protocol.make(author=logged_in_user)
commit_protocol = helpers.add_version(protocol, visibility='public')
+ version1 = model.repocache.get_version(commit1.sha)
+ version2 = model.repocache.get_version(commit2.sha)
+
response = client.get(
'/entities/protocols/%d/versions/%s/runexperiments' % (protocol.pk, commit_protocol.sha))
assert response.status_code == 200
assert response.context['object_list'] == [{'id': model.pk,
+ 'entity': model,
'name': 'mymodel1',
- 'versions': [{'commit': commit2, 'tags': ['v1'], 'latest': True},
- {'commit': commit1, 'tags': [], 'latest': False}]},
+ 'versions': [{'commit': version2, 'tags': ['v1'], 'latest': True},
+ {'commit': version1, 'tags': [], 'latest': False}]},
]
# Test post returns correct response
data = {'model_protocol_list[]': ['%d:%s' % (model.pk, commit1.sha), '%d:%s' % (model.pk, commit2.sha)],
@@ -2437,14 +2485,18 @@ def test_view_run_experiment_protocol_post_exclude_existing(self, client, helper
protocol=protocol,
protocol_version=commit_protocol.sha)
+ version1 = model.repocache.get_version(commit1.sha)
+ version2 = model.repocache.get_version(commit2.sha)
+
# Test context has correct information
response = client.get(
'/entities/protocols/%d/versions/%s/runexperiments' % (protocol.pk, commit_protocol.sha))
assert response.status_code == 200
assert response.context['object_list'] == [{'id': model.pk,
+ 'entity': model,
'name': 'mymodel1',
- 'versions': [{'commit': commit2, 'tags': ['v1'], 'latest': True},
- {'commit': commit1, 'tags': [], 'latest': False}]},
+ 'versions': [{'commit': version2, 'tags': ['v1'], 'latest': True},
+ {'commit': version1, 'tags': [], 'latest': False}]},
]
# Test post returns correct response
data = {'model_protocol_list[]': ['%d:%s' % (model.pk, commit1.sha), '%d:%s' % (model.pk, commit2.sha)],
@@ -2471,6 +2523,7 @@ def test_view_run_experiment_post_protocol_multiple_users(self, client, helpers,
commit1 = helpers.add_version(model, visibility='public')
commit2 = helpers.add_version(model, visibility='public')
model.add_tag('v1', commit2.sha)
+
protocol = recipes.protocol.make(author=logged_in_user)
commit_protocol = helpers.add_version(protocol, visibility='public')
@@ -2479,19 +2532,26 @@ def test_view_run_experiment_post_protocol_multiple_users(self, client, helpers,
other_commit2 = helpers.add_version(other_model, visibility='public')
other_model.add_tag('v1', other_commit2.sha)
+ version1 = model.repocache.get_version(commit1.sha)
+ version2 = model.repocache.get_version(commit2.sha)
+ other_version1 = other_model.repocache.get_version(other_commit1.sha)
+ other_version2 = other_model.repocache.get_version(other_commit2.sha)
+
response = client.get(
'/entities/protocols/%d/versions/%s/runexperiments' % (protocol.pk, commit_protocol.sha))
assert response.status_code == 200
assert response.context['object_list'] == [{'id': model.pk,
+ 'entity': model,
'name': 'mymodel1',
- 'versions': [{'commit': commit2, 'tags': ['v1'], 'latest': True},
- {'commit': commit1, 'tags': [], 'latest': False}]},
+ 'versions': [{'commit': version2, 'tags': ['v1'], 'latest': True},
+ {'commit': version1, 'tags': [], 'latest': False}]},
]
assert response.context['other_object_list'] == [
{'id': other_model.pk,
+ 'entity': other_model,
'name': 'mymodel2',
- 'versions': [{'commit': other_commit2, 'tags': ['v1'], 'latest': True},
- {'commit': other_commit1, 'tags': [], 'latest': False}]},
+ 'versions': [{'commit': other_version2, 'tags': ['v1'], 'latest': True},
+ {'commit': other_version1, 'tags': [], 'latest': False}]},
]
# Test post returns correct response
data = {'model_protocol_list[]': ['%d:%s' % (model.pk, commit1.sha),
@@ -2526,14 +2586,18 @@ def test_view_run_experiment_none_checked(self, client, helpers, logged_in_user)
commit2 = helpers.add_version(protocol, visibility='public')
protocol.add_tag('v1', commit2.sha)
+ version1 = protocol.repocache.get_version(commit1.sha)
+ version2 = protocol.repocache.get_version(commit2.sha)
+
# Test context has correct information
response = client.get(
'/entities/models/%d/versions/%s/runexperiments' % (model.pk, commit_model.sha))
assert response.status_code == 200
assert response.context['object_list'] == [{'id': protocol.pk,
+ 'entity': protocol,
'name': 'myprotocol1',
- 'versions': [{'commit': commit2, 'tags': ['v1'], 'latest': True},
- {'commit': commit1, 'tags': [], 'latest': False}]},
+ 'versions': [{'commit': version2, 'tags': ['v1'], 'latest': True},
+ {'commit': version1, 'tags': [], 'latest': False}]},
]
# Test post returns correct response
data = {'model_protocol_list[]': [],
@@ -2570,14 +2634,18 @@ def test_view_run_experiment_protocol_not_latest(self, client, helpers, logged_i
protocol.add_tag('p1', proto_commit1.sha)
protocol.add_tag('p2', proto_commit2.sha)
+ version1 = model.repocache.get_version(commit1.sha)
+ version2 = model.repocache.get_version(commit2.sha)
+
# display using sha
response = client.get(
'/entities/protocols/%d/versions/%s/runexperiments' % (protocol.pk, proto_commit1.sha))
assert response.status_code == 200
assert response.context['object_list'] == [{'id': model.pk,
+ 'entity': model,
'name': 'mymodel1',
- 'versions': [{'commit': commit2, 'tags': ['v1'], 'latest': True},
- {'commit': commit1, 'tags': [], 'latest': False}]},
+ 'versions': [{'commit': version2, 'tags': ['v1'], 'latest': True},
+ {'commit': version1, 'tags': [], 'latest': False}]},
]
assert response.context['preposition'] == 'on'
diff --git a/weblab/entities/views.py b/weblab/entities/views.py
index 63670de61..269566792 100644
--- a/weblab/entities/views.py
+++ b/weblab/entities/views.py
@@ -15,7 +15,7 @@
PermissionRequiredMixin,
UserPassesTestMixin,
)
-from django.core.exceptions import PermissionDenied
+from django.core.exceptions import ObjectDoesNotExist, PermissionDenied
from django.core.urlresolvers import reverse
from django.db.models import (
Count,
@@ -123,6 +123,27 @@ def get_commit(self):
return self._commit
+ def get_version(self):
+ """
+ Get the cached entity for this version
+
+ :return: CachedEntityVersion object
+ :raise: Http404 if version not found
+ """
+ if hasattr(self, '_version'):
+ return self._version
+
+ sha_or_tag = self.kwargs['sha']
+ cached_entity = self.object.cachedentity
+ try:
+ self._version = cached_entity.get_version(sha_or_tag)
+ except RepoCacheMiss:
+ try:
+ self._version = cached_entity.tags.get(tag=sha_or_tag).version
+ except ObjectDoesNotExist:
+ raise Http404
+ return self._version
+
def _get_object(self):
if not hasattr(self, 'object'):
self.object = self.get_object()
@@ -130,12 +151,12 @@ def _get_object(self):
def get_context_data(self, **kwargs):
entity = self._get_object()
- commit = self.get_commit()
+ version = self.get_version()
kwargs.update(**{
- 'version': commit,
+ 'version': version,
'visibility': self.get_visibility(),
- 'tags': entity.get_tags(commit.sha),
- 'master_filename': commit.master_filename,
+ 'tags': entity.get_tags(version.sha),
+ 'master_filename': version.master_filename,
})
return super().get_context_data(**kwargs)
@@ -192,7 +213,7 @@ class EntityVersionView(EntityTypeMixin, EntityVersionMixin, DetailView):
def get_context_data(self, **kwargs):
entity = self._get_object()
- visibility = entity.get_version_visibility(self.get_commit().sha)
+ visibility = entity.get_version_visibility(self.get_version().sha)
kwargs['form'] = EntityChangeVisibilityForm(
user=self.request.user,
initial={
@@ -249,7 +270,7 @@ class EntityCompareExperimentsView(EntityTypeMixin, EntityVersionMixin, DetailVi
def get_context_data(self, **kwargs):
entity = self._get_object()
- commit = self.get_commit()
+ commit = self.get_version()
entity_type = entity.entity_type
other_type = entity.other_type
@@ -637,7 +658,7 @@ def get_context_data(self, **kwargs):
kwargs.update(**{
'versions': list(
(list(version.tags.values_list('tag', flat=True)),
- entity.repo.get_commit(version.sha))
+ entity.repocache.get_version(version.sha))
for version in versions.prefetch_related('tags')
)
})
@@ -1007,13 +1028,15 @@ def get_context_data(self, **kwargs):
version_info = []
for version in versions.prefetch_related('tags'):
tag_list = list(version.tags.values_list('tag', flat=True))
- commit = item.repo.get_commit(version.sha)
- latest = item.repo.latest_commit
- version_info.append({'commit': commit, 'tags': tag_list, 'latest': latest == commit})
+ ver = item.repocache.get_version(version.sha)
+ latest_version = item.repocache.latest_version
+ version_info.append({'commit': ver, 'tags': tag_list, 'latest': latest_version == ver})
if item.author == self.request.user:
- context['object_list'].append({'id': item.id, 'name': item.name, 'versions': version_info})
+ context['object_list'].append(
+ {'entity': item, 'id': item.id, 'name': item.name, 'versions': version_info})
else:
- context['other_object_list'].append({'id': item.id, 'name': item.name, 'versions': version_info})
+ context['other_object_list'].append(
+ {'entity': item, 'id': item.id, 'name': item.name, 'versions': version_info})
return context
def post(self, request, *args, **kwargs):
@@ -1021,7 +1044,7 @@ def post(self, request, *args, **kwargs):
# in get context self.object was the entity being worked with
# here we have to retrieve it
this_entity = self.get_object()
- this_version = self.get_commit().sha
+ this_version = self.get_version().sha
is_latest = (this_version == this_entity.repocache.latest_version.sha)
exclude_existing = 'rerun_expts' not in request.POST
experiments_to_run = request.POST.getlist('model_protocol_list[]')
diff --git a/weblab/experiments/tests/test_models.py b/weblab/experiments/tests/test_models.py
index 11209f417..4cf2734bb 100644
--- a/weblab/experiments/tests/test_models.py
+++ b/weblab/experiments/tests/test_models.py
@@ -7,6 +7,7 @@
from core import recipes
from experiments.models import Experiment, ExperimentVersion
+from repocache.populate import populate_entity_cache
@pytest.fixture
@@ -56,9 +57,12 @@ def test_nice_versions(self, experiment_version):
assert exp.nice_protocol_version == exp.protocol.repo.latest_commit.sha[:8] + '...'
exp.model.repo.tag('v1')
+ populate_entity_cache(exp.model)
assert exp.nice_model_version == 'v1'
exp.protocol.repo.tag('v2')
+ populate_entity_cache(exp.protocol)
+
assert exp.nice_protocol_version == 'v2'
def test_visibility(self, helpers):
diff --git a/weblab/experiments/tests/test_views.py b/weblab/experiments/tests/test_views.py
index 04da9d536..0e8eec778 100644
--- a/weblab/experiments/tests/test_views.py
+++ b/weblab/experiments/tests/test_views.py
@@ -21,6 +21,7 @@
PlannedExperiment,
RunningExperiment,
)
+from repocache.populate import populate_entity_cache
def generate_response(template='%s succ celery-task-id'):
@@ -1168,6 +1169,7 @@ def test_compare_experiments(self, client, experiment_version, helpers):
protocol = recipes.protocol.make()
protocol_commit = helpers.add_version(protocol, visibility='public')
exp.protocol.repo.tag('v1')
+ populate_entity_cache(exp.protocol)
version2 = recipes.experiment_version.make(
status='SUCCESS',
diff --git a/weblab/experiments/views.py b/weblab/experiments/views.py
index cb024adb0..2c9f7cab4 100644
--- a/weblab/experiments/views.py
+++ b/weblab/experiments/views.py
@@ -450,8 +450,8 @@ def _version_json(self, version, model_version_in_name, protocol_version_in_name
'versionId': version.id,
'modelName': exp.model.name,
'protoName': exp.protocol.name,
- 'modelVersion': exp.model.repo.get_name_for_commit(exp.model_version), # TODO #191: Use repocache instead
- 'protoVersion': exp.protocol.repo.get_name_for_commit(exp.protocol_version),
+ 'modelVersion': exp.model.repocache.get_name_for_version(exp.model_version),
+ 'protoVersion': exp.protocol.repocache.get_name_for_version(exp.protocol_version),
'runNumber': version.run_number,
})
return details
diff --git a/weblab/repocache/management/commands/populate_entity_cache.py b/weblab/repocache/management/commands/populate_entity_cache.py
index 439d2c8bd..c71809c7a 100644
--- a/weblab/repocache/management/commands/populate_entity_cache.py
+++ b/weblab/repocache/management/commands/populate_entity_cache.py
@@ -17,4 +17,5 @@ def handle(self, *args, **options):
entities = entities.filter(id__in=options['entity_id'])
for entity in entities:
- populate_entity_cache(entity)
+ if entity.repo_abs_path.exists():
+ populate_entity_cache(entity)
diff --git a/weblab/repocache/models.py b/weblab/repocache/models.py
index 31cc30500..c2e1e20ca 100644
--- a/weblab/repocache/models.py
+++ b/weblab/repocache/models.py
@@ -67,6 +67,19 @@ def get_version(self, sha):
except ObjectDoesNotExist:
raise RepoCacheMiss("Entity version not found")
+ def get_name_for_version(self, sha):
+ """Get a human-friendly display name for the given version
+
+ :param sha: version sha
+ :return: first cached tag for this version, if any, or sha if not
+ """
+ version = self.get_version(sha)
+
+ first_tag = version.tags.first()
+ if first_tag is not None:
+ return first_tag.tag
+ return sha
+
def add_version(self, sha):
"""
Add an entity version to the cache
diff --git a/weblab/repocache/tests/test_models.py b/weblab/repocache/tests/test_models.py
index 9dd8f6f04..f9ea415c4 100644
--- a/weblab/repocache/tests/test_models.py
+++ b/weblab/repocache/tests/test_models.py
@@ -2,7 +2,9 @@
from django.db.utils import IntegrityError
from core import recipes
+from repocache.exceptions import RepoCacheMiss
from repocache.models import CachedModel, CachedProtocol
+from repocache.populate import populate_entity_cache
@pytest.mark.django_db
@@ -111,3 +113,20 @@ def test_add_version(self, helpers):
assert version.author == commit.author.name
assert version.numfiles == len(commit.filenames)
assert version.visibility == 'public'
+
+ def test_get_name_for_version(self, helpers):
+ model = recipes.model.make()
+ commit = helpers.add_version(model, visibility='public', cache=False)
+ populate_entity_cache(model)
+ assert model.repocache.get_name_for_version(commit.sha) == commit.sha
+ assert model.repocache.get_name_for_version('latest') == 'latest'
+
+ model.repo.tag('v1')
+ assert model.repocache.get_name_for_version(commit.sha) == commit.sha
+ populate_entity_cache(model)
+ assert model.repocache.get_name_for_version(commit.sha) == 'v1'
+ assert model.repocache.get_name_for_version('latest') == 'v1'
+
+ # get_name_for_version must be sha or latest
+ with pytest.raises(RepoCacheMiss):
+ model.repocache.get_name_for_version('v1')
diff --git a/weblab/templates/entities/entity_runexperiments.html b/weblab/templates/entities/entity_runexperiments.html
index 49602bec9..0ccf728bf 100644
--- a/weblab/templates/entities/entity_runexperiments.html
+++ b/weblab/templates/entities/entity_runexperiments.html
@@ -46,7 +46,7 @@ Your {{ other_type }}s
name="model_protocol_list[]"/>
{% endif %}
-
+
{% include "./includes/version_name.html" with tags=entity_version.tags version=entity_version.commit only %}
@@ -84,16 +84,14 @@ Other {{ other_type }}s
name="model_protocol_list[]"/>
{% endif %}
-
+
{% include "./includes/version_name.html" with tags=entity_version.tags version=entity_version.commit only %}
by {{ entity_version.commit.author }}
created
- {% with entity_version.commit.filenames|length as numfiles %}
- containing {{ numfiles }} file{{ numfiles|pluralize }}
- {% endwith %}
+ containing {{ entity_version.commit.numfiles }} file{{ entity_version.commit.numfiles|pluralize }}
{{ entity_version.commit.message|linebreaksbr }}
diff --git a/weblab/templates/entities/entity_versions.html b/weblab/templates/entities/entity_versions.html
index 60a8ceb34..98a6be929 100644
--- a/weblab/templates/entities/entity_versions.html
+++ b/weblab/templates/entities/entity_versions.html
@@ -28,9 +28,7 @@ Versions
by {{ commit.author }}
created
- {% with commit.filenames|length as numfiles %}
- containing {{ numfiles }} file{{ numfiles|pluralize}}
- {% endwith %}
+ containing {{ commit.numfiles }} file{{ commit.numfiles|pluralize}}
{{ commit.message|linebreaksbr }}