From f401dfc4da390802ce316d7eab99cf92a7549c28 Mon Sep 17 00:00:00 2001 From: Romain Keramitas Date: Fri, 15 Jun 2018 10:55:46 +0200 Subject: [PATCH] Add tests Signed-off-by: Romain Keramitas --- modelforge/__main__.py | 35 ++++----- modelforge/backends.py | 10 ++- modelforge/configuration.py | 1 + modelforge/gcs_backend.py | 75 +++++++++++--------- modelforge/index.py | 57 +++++++++------ modelforge/model.py | 2 +- modelforge/registry.py | 113 +++++++++++++++++++----------- modelforge/storage_backend.py | 51 +++++++++----- modelforge/tests/fake_requests.py | 6 ++ modelforge/tests/test_backends.py | 16 ++--- modelforge/tests/test_index.py | 4 +- modelforge/tests/test_model.py | 14 ++++ 12 files changed, 236 insertions(+), 148 deletions(-) diff --git a/modelforge/__main__.py b/modelforge/__main__.py index b088061..a3c7bb5 100644 --- a/modelforge/__main__.py +++ b/modelforge/__main__.py @@ -4,7 +4,6 @@ import os from modelforge.logs import setup_logging -from modelforge.index import DEFAULT_CACHE from modelforge.registry import publish_model, list_models, initialize_registry, dump_model, \ delete_model @@ -29,8 +28,9 @@ def add_index_args(p): help="Username for the Git repository with the index.") p.add_argument("--password", default="", help="Password for the Git repository with the index") - p.add_argument("--index-repo", default="", help="Url of the remote Git repository.") - p.add_argument("--cache", default=DEFAULT_CACHE, + p.add_argument("--index-repo", default=None, + help="Url of the remote Git repository.") + p.add_argument("--cache", default=None, help="Path to the folder where the Git repository will be cached.") def add_templates_args(p): @@ -43,14 +43,22 @@ def add_templates_args(p): default=os.path.join(os.path.dirname(__file__), "templates/template_readme.md.jinja2"), help="Path to the jinja2 template used in the index for the readme.") + # ------------------------------------------------------------------------ + init_parser = subparsers.add_parser("init", help="Initialize the registry.") + init_parser.set_defaults(handler=initialize_registry) + init_parser.add_argument("-f", "--force", action="store_true", + help="Destructively initialize the registry.") + add_index_args(init_parser) + add_backend_args(init_parser) + # ------------------------------------------------------------------------ dump_parser = subparsers.add_parser( "dump", help="Print a brief information about the model to stdout.") dump_parser.set_defaults(handler=dump_model) dump_parser.add_argument( "input", help="Path to the model file, URL or UUID.") - add_backend_args(dump_parser) add_index_args(dump_parser) - + add_backend_args(dump_parser) + # ------------------------------------------------------------------------ publish_parser = subparsers.add_parser( "publish", help="Upload the model and update the registry.") publish_parser.set_defaults(handler=publish_model) @@ -63,30 +71,23 @@ def add_templates_args(p): help="Set this model as the default one.") publish_parser.add_argument("-f", "--force", action="store_true", help="Overwrite existing models.") - add_backend_args(publish_parser) add_index_args(publish_parser) + add_backend_args(publish_parser) add_templates_args(publish_parser) - + # ------------------------------------------------------------------------ list_parser = subparsers.add_parser( "list", help="Lists all the models in the registry.") list_parser.set_defaults(handler=list_models) add_index_args(list_parser) - - init_parser = subparsers.add_parser("init", help="Initialize the registry.") - init_parser.set_defaults(handler=initialize_registry) - init_parser.add_argument("-f", "--force", action="store_true", - help="Destructively initialize the registry.") - add_backend_args(init_parser) - add_index_args(init_parser) - + # ------------------------------------------------------------------------ delete_parser = subparsers.add_parser("delete", help="Delete a model.") delete_parser.set_defaults(handler=delete_model) delete_parser.add_argument( "input", help="UUID of the model to be deleted.") - add_backend_args(delete_parser) add_index_args(delete_parser) + add_backend_args(delete_parser) add_templates_args(delete_parser) - + # ------------------------------------------------------------------------ args = parser.parse_args() args.log_level = logging._nameToLevel[args.log_level] setup_logging(args.log_level) diff --git a/modelforge/backends.py b/modelforge/backends.py index 2d7fad3..8c96791 100644 --- a/modelforge/backends.py +++ b/modelforge/backends.py @@ -52,9 +52,13 @@ def supply_backend_inner(func): @wraps(func) def wrapped_supply_backend(args): log = logging.getLogger(func.__name__) - git_index = GitIndex(index_repo=args.index_repo, username=args.username, - password=args.password, cache=args.cache, - log_level=args.log_level) + try: + git_index = GitIndex(index_repo=args.index_repo, username=args.username, + password=args.password, cache=args.cache, + log_level=args.log_level) + except ValueError: + return 1 + if real_optional and not getattr(args, "backend", False): backend = None else: diff --git a/modelforge/configuration.py b/modelforge/configuration.py index 39fccab..425f702 100644 --- a/modelforge/configuration.py +++ b/modelforge/configuration.py @@ -7,6 +7,7 @@ BACKEND = os.getenv("MODELFORGE_BACKEND", None) BACKEND_ARGS = os.getenv("MODELFORGE_BACKEND_ARGS", "") DEFAULT_REPO = os.getenv("MODELFORGE_DEFAULT_REPO", "") +DEFAULT_CACHE = os.getenv("MODELFORGE_DEFAULT_CACHE", os.path.expanduser("~/.cache")) OVERRIDE_FILE = "modelforgecfg.py" diff --git a/modelforge/gcs_backend.py b/modelforge/gcs_backend.py index a92791b..885d86e 100644 --- a/modelforge/gcs_backend.py +++ b/modelforge/gcs_backend.py @@ -1,16 +1,17 @@ import io -import json import logging import math import os import requests +from typing import BinaryIO, Union from clint.textui import progress from google.cloud.exceptions import NotFound from modelforge.index import GitIndex from modelforge.progress_bar import progress_bar -from modelforge.storage_backend import StorageBackend, TransactionRequiredError +from modelforge.storage_backend import StorageBackend, ExistingBackendError, \ + ModelAlreadyExistsError, BackendRequiredError class Tracker: @@ -18,7 +19,7 @@ class Tracker: Wrapper around a bytes buffer which follows the file position and updates the console progressbar mimicking a file object. """ - def __init__(self, data, logger): + def __init__(self, data: memoryview, logger: logging.Logger): self._file = io.BytesIO(data) self._size = len(data) self._enabled = logger.isEnabledFor(logging.INFO) @@ -27,7 +28,7 @@ def __init__(self, data, logger): else: logger.debug("Progress indication is not enabled") - def read(self, size=None): + def read(self, size: int=None): pos_before = self._file.tell() result = self._file.read(size) if self._enabled: @@ -75,31 +76,41 @@ def bucket_name(self): def credentials(self): return self._credentials - def fetch_model(self, source: str, file: str) -> None: - self._fetch(source, file) - - def connect(self): - log = self._log - log.info("Connecting to the bucket...") - + def create_client(self): # Client should be imported here because grpc starts threads during import # and if you call fork after that, a child process will be hang during exit from google.cloud.storage import Client - if self.credentials: client = Client.from_service_account_json(self.credentials) else: client = Client() + return client + + def connect(self): + log = self._log + log.info("Connecting to the bucket...") + client = self.create_client() return client.lookup_bucket(self.bucket_name) - def upload_model(self, path, meta, force): + def reset(self, force): + client = self.create_client() + bucket = client.lookup_bucket(self.bucket_name) + if bucket is not None: + if not force: + self._log.error("Bucket already exists, aborting.") + raise ExistingBackendError + self._log.info("Bucket already exists, deleting all content.") + bucket.delete(force, client) + client.create_bucket(self.bucket_name) + + def upload_model(self, path: str, meta: dict, force: bool): bucket = self.connect() if bucket is None: - raise TransactionRequiredError + raise BackendRequiredError blob = bucket.blob("models/%s/%s.asdf" % (meta["model"], meta["uuid"])) if blob.exists() and not force: self._log.error("Model %s already exists, aborted.", meta["uuid"]) - return 1 + raise ModelAlreadyExistsError self._log.info("Uploading %s from %s...", meta["model"], os.path.abspath(path)) def tracker(data): @@ -124,23 +135,19 @@ def request_with_progress(method, url, data=None, headers=None, **kwargs): blob.make_public() return blob.public_url - def upload_index(self, index): - bucket = self.connect() - if bucket is None: - raise TransactionRequiredError - blob = bucket.blob(self.INDEX_FILE) - blob.cache_control = "no-cache" - blob.upload_from_string(json.dumps(index, indent=4, sort_keys=True)) - blob.make_public() - - def _fetch(self, url, where, chunk_size=DEFAULT_CHUNK_SIZE): - self._log.info("Fetching %s...", url) - r = requests.get(url, stream=True) - if isinstance(where, str): - os.makedirs(os.path.dirname(where), exist_ok=True) - f = open(where, "wb") + def fetch_model(self, source: str, file: Union[str, BinaryIO], + chunk_size: int=DEFAULT_CHUNK_SIZE) -> None: + self._log.info("Fetching %s...", source) + r = requests.get(source, stream=True) + if r.status_code != 200: + self._log.error( + "An error occurred while fetching the model, with code %s" % r.status_code) + raise ValueError + if isinstance(file, str): + os.makedirs(os.path.dirname(file), exist_ok=True) + f = open(file, "wb") else: - f = where + f = file try: total_length = int(r.headers.get("content-length")) num_chunks = math.ceil(total_length / chunk_size) @@ -153,13 +160,13 @@ def _fetch(self, url, where, chunk_size=DEFAULT_CHUNK_SIZE): if chunk: f.write(chunk) finally: - if isinstance(where, str): + if isinstance(file, str): f.close() - def delete_model(self, meta): + def delete_model(self, meta: dict): bucket = self.connect() if bucket is None: - raise TransactionRequiredError + raise BackendRequiredError blob_name = "models/%s/%s.asdf" % (meta["model"], meta["uuid"]) self._log.info(blob_name) try: diff --git a/modelforge/index.py b/modelforge/index.py index 4e95cd5..dee5b6a 100644 --- a/modelforge/index.py +++ b/modelforge/index.py @@ -9,9 +9,9 @@ from dulwich.errors import HangupException, GitProtocolError, NotGitRepository from urllib.parse import urlparse +import modelforge.configuration as config INDEX_FILE = "index.json" #: Models repository index file name. -DEFAULT_CACHE = os.path.expanduser("~/.cache") REMOTE_URL = "%s://%s%s/%s" #: Remote repo url @@ -23,8 +23,8 @@ class GitIndex: "add": "Add {model}/{uuid}", } - def __init__(self, index_repo: str= "", username: str= "", password: str= "", - cache: str=DEFAULT_CACHE, log_level: int=logging.INFO): + def __init__(self, index_repo: str="", username: str= "", password: str= "", cache: str="", + log_level: int=logging.INFO): """ Initializes a new instance of :class:`GitIndex`. @@ -37,14 +37,21 @@ def __init__(self, index_repo: str= "", username: str= "", password: str= "", """ self._log = logging.getLogger(type(self).__name__) self._log.setLevel(log_level) + if index_repo is None and config.DEFAULT_REPO: + index_repo = config.DEFAULT_REPO + if cache is None: + cache = config.DEFAULT_CACHE parsed_url = urlparse(index_repo) if not parsed_url.scheme or \ parsed_url.scheme not in ("git", "git+ssh", "ssh", "http", "https"): - raise ValueError("Parsed url does not contain a valid protocol.") + self._log.critical("Parsed url does not contain a valid protocol.") + raise ValueError if not parsed_url.netloc: - raise ValueError("Parsed url does not contain a valid domain.") + self._log.critical("Parsed url does not contain a valid domain.") + raise ValueError if not parsed_url.path: - raise ValueError("Parsed url does not contain a valid repository path.") + self._log.critical("Parsed url does not contain a valid repository path.") + raise ValueError self.repo = parsed_url.path if self.repo.startswith("/"): self.repo = self.repo[1:] @@ -55,21 +62,25 @@ def __init__(self, index_repo: str= "", username: str= "", password: str= "", auth = username + ":" + password + "@" self.remote_url = REMOTE_URL % (parsed_url.scheme, auth, parsed_url.netloc, self.repo) elif username or password: - raise ValueError("Both username and password must be supplied to access git with " - "credentials.") + self._log.critical("Both username and password must be supplied to access git with " + "credentials.") + raise ValueError else: self.remote_url = index_repo - self.content = None + self.contents = None try: self.fetch_index() except NotGitRepository as e: - raise ValueError("Repository does not exist: %s" % e) from e + self._log.critical("Repository does not exist: %s" % e) + raise ValueError except HangupException as e: - raise ValueError("Check SSH is configured, or connection is stable: %s" % e) from e + self._log.critical("Check SSH is configured, or connection is stable: %s" % e) + raise ValueError except GitProtocolError as e: - raise ValueError("%s: %s\nCheck your Git credentials." % (type(e), e)) - self.models = self.content["models"] - self.meta = self.content["meta"] + self._log.critical("%s: %s\nCheck your Git credentials." % (type(e), e)) + raise ValueError + self.models = self.contents["models"] + self.meta = self.contents["meta"] def fetch_index(self): os.makedirs(os.path.dirname(self.cached_repo), exist_ok=True) @@ -82,7 +93,7 @@ def fetch_index(self): self._log.info("Cached index is not up to date, pulling %s", self. repo) git.pull(self.cached_repo, self.remote_url) with open(os.path.join(self.cached_repo, INDEX_FILE), encoding="utf-8") as _in: - self.content = json.load(_in) + self.contents = json.load(_in) def remove_model(self, model_uuid: str) -> dict: model_type = None @@ -92,7 +103,8 @@ def remove_model(self, model_uuid: str) -> dict: model_type = key break if model_type is None: - raise ValueError("Model not found, aborted.") + self._log.error("Model not found, aborted.") + raise ValueError model_directory = os.path.join(self.cached_repo, model_type) model_node = self.models[model_type] meta_node = self.meta[model_type] @@ -152,7 +164,7 @@ def initialize_index(self): os.remove(path) elif os.path.isdir(path): shutil.rmtree(path) - self.content = {"models": {}, "meta": {}} + self.contents = {"models": {}, "meta": {}} def upload_index(self, cmd: str, meta: dict): index = os.path.join(self.cached_repo, INDEX_FILE) @@ -160,7 +172,7 @@ def upload_index(self, cmd: str, meta: dict): os.remove(index) self._log.info("Writing the new index.json ...") with open(index, "w") as _out: - json.dump(self.content, _out) + json.dump(self.contents, _out) # implementation of git add --all is pretty bad, changing directory is the easiest way os.chdir(self.cached_repo) git.add() @@ -169,15 +181,18 @@ def upload_index(self, cmd: str, meta: dict): # TODO: change when https://github.com/dulwich/dulwich/issues/631 gets addressed git.push(self.cached_repo, self.remote_url, b"master") if self._are_local_and_remote_heads_different(): - raise ValueError("Push has failed") + self._log.error("Push has failed") + raise ValueError def load_template(self, template: str) -> Template: env = dict(trim_blocks=True, lstrip_blocks=True, keep_trailing_newline=False) jinja2_ext = ".jinja2" if not template.endswith(jinja2_ext): - raise ValueError("Template file name must end with %s" % jinja2_ext) + self._log.error("Template file name must end with %s" % jinja2_ext) + raise ValueError if not template[:-len(jinja2_ext)].endswith(".md"): - raise ValueError("Template file should be a Markdown file.") + self._log.error("Template file should be a Markdown file.") + raise ValueError with open(template, encoding="utf-8") as fin: template_obj = Template(fin.read(), **env) template_obj.filename = template diff --git a/modelforge/model.py b/modelforge/model.py index 7c220a2..790c05a 100644 --- a/modelforge/model.py +++ b/modelforge/model.py @@ -81,7 +81,7 @@ def load(self, source: Union[str, BinaryIO, "Model"]=None, cache_dir: str=None, if backend is None: raise ValueError("The backend must be set to load a UUID or the default " "model.") - index = backend.index.content + index = backend.index.contents config = index["models"] if self.NAME is not None: if not is_uuid: diff --git a/modelforge/registry.py b/modelforge/registry.py index f27b924..977d13a 100644 --- a/modelforge/registry.py +++ b/modelforge/registry.py @@ -6,12 +6,56 @@ from dateutil.parser import parse as parse_datetime from modelforge.models import GenericModel -from modelforge.storage_backend import StorageBackend, ModelExistsError from modelforge.backends import supply_backend +from modelforge.storage_backend import StorageBackend, ModelAlreadyExistsError, \ + ExistingBackendError from modelforge.index import GitIndex from modelforge.meta import extract_model_meta +@supply_backend +def initialize_registry(args: argparse.Namespace, backend: StorageBackend, log: logging.Logger): + """ + Initialize the registry and the index. + :param args: :class:`argparse.Namespace` with "backend", "args", "force" and "log_level". + :param backend: Backend which is responsible for working with model files. + :param log: Logger supplied by supply_backend + :return: None + """ + try: + backend.reset(args.force) + except ExistingBackendError: + return 1 + + log.info("Deleting content of index ...") + backend.index.initialize_index() + try: + backend.index.upload_index("initilialize", {}) + except ValueError: + return 1 + log.info("Successfully initialized") + + +@supply_backend(optional=True) +def dump_model(args: argparse.Namespace, backend: StorageBackend, log: logging.Logger): + """ + Prints the information about the model. + :param args: :class:`argparse.Namespace` with "input", "backend", "args", "username", + "password", "remote_repo" and "log_level". + :param backend: Backend which is responsible for working with model files. + :param log: Logger supplied by supply_backend + :return: None + """ + try: + print(GenericModel(args.input, backend=backend)) + except ValueError as e: + log.critical('"input" must be a path: %s', e) + return 1 + except Exception as e: + log.critical("Failed to load the model: %s: %s" % (type(e).__name__, e)) + return 1 + + @supply_backend def publish_model(args: argparse.Namespace, backend: StorageBackend, log: logging.Logger): """ @@ -35,19 +79,26 @@ def publish_model(args: argparse.Namespace, backend: StorageBackend, log: loggin base_meta = model.meta try: model_url = backend.upload_model(path, base_meta, args.force) - except ModelExistsError: + except ModelAlreadyExistsError: return 1 + log.info("Uploaded as %s", model_url) with open(os.path.join(args.meta), encoding="utf-8") as _in: extra_meta = json.load(_in) meta = extract_model_meta(base_meta, extra_meta, model_url) log.info("Updating the models index...") - template_model = backend.index.load_template(args.template_model) + try: + template_model = backend.index.load_template(args.template_model) + template_readme = backend.index.load_template(args.template_readme) + except ValueError: + return 1 backend.index.add_model(base_meta["model"], base_meta["uuid"], meta, template_model, args.update_default) - template_readme = backend.index.load_template(args.template_readme) backend.index.update_readme(template_readme) - backend.index.upload_index("add", base_meta) + try: + backend.index.upload_index("add", base_meta) + except ValueError: # TODO: replace with PorcelainError, see related TODO in index.py:181 + return 1 log.info("Successfully published.") @@ -58,8 +109,11 @@ def list_models(args: argparse.Namespace): "log_level" :return: None """ - git_index = GitIndex(index_repo=args.index_repo, username=args.username, - password=args.password, cache=args.cache, log_level=args.log_level) + try: + git_index = GitIndex(index_repo=args.index_repo, username=args.username, + password=args.password, cache=args.cache, log_level=args.log_level) + except ValueError: + return 1 for model_type, models in git_index.models.items(): print(model_type) default = git_index.meta[model_type]["default"] @@ -70,36 +124,6 @@ def list_models(args: argparse.Namespace): model["created_at"]) -@supply_backend -def initialize_registry(args: argparse.Namespace, backend: StorageBackend, log: logging.Logger): - """ - Initialize the registry and the index. - :param args: :class:`argparse.Namespace` with "backend", "args", "force" and "log_level". - :param backend: Backend which is responsible for working with model files. - :param log: Logger supplied by supply_backend - :return: None - """ - - # TODO: add backend.initialize_backend(args.force) (Next split) - log.info("Deleting content of index ...") - backend.index.initialize_index() - backend.index.upload_index("initilialize", {}) - log.info("Successfully initialized") - - -@supply_backend(optional=True) -def dump_model(args: argparse.Namespace, backend: StorageBackend, log: logging.Logger): - """ - Prints the information about the model. - :param args: :class:`argparse.Namespace` with "input", "backend", "args", "username", - "password", "remote_repo" and "log_level". - :param backend: Backend which is responsible for working with model files. - :param log: Logger supplied by supply_backend - :return: None - """ - print(GenericModel(args.input, backend=backend)) - - @supply_backend def delete_model(args: argparse.Namespace, backend: StorageBackend, log: logging.Logger): """ @@ -111,11 +135,16 @@ def delete_model(args: argparse.Namespace, backend: StorageBackend, log: logging :param log: Logger supplied by supply_backend :return: None """ - - meta = backend.index.remove_model(args.input) + try: + meta = backend.index.remove_model(args.input) + template_readme = backend.index.load_template(args.template_readme) + backend.index.update_readme(template_readme) + except ValueError: + return 1 backend.delete_model(meta) log.info("Updating the models index...") - template_readme = backend.index.load_template(args.template_readme) - backend.index.update_readme(template_readme) - backend.index.upload_index("delete", meta) + try: + backend.index.upload_index("delete", meta) + except ValueError: # TODO: replace with PorcelainError + return 1 log.info("Successfully deleted.") diff --git a/modelforge/storage_backend.py b/modelforge/storage_backend.py index 7fe7ea1..77e2e29 100644 --- a/modelforge/storage_backend.py +++ b/modelforge/storage_backend.py @@ -1,3 +1,5 @@ +from typing import Union, BinaryIO + from modelforge.index import GitIndex @@ -11,22 +13,22 @@ def __init__(self, index: GitIndex=None, **kwargs): """ self._index = index - def fetch_model(self, source: str, file: str) -> None: - """ - Downloads the model from the remote storage. - - :param source: URL to download. - :param file: PAth to the local file to write. - :return: None - """ - raise NotImplementedError - @property def index(self) -> GitIndex: if self._index: return self._index raise AttributeError("No index was provided.") + def reset(self, force: bool): + """ + Initialize backend. + :param force: If backend is not empty, force must be set to True. + :return: None + :raises BackendRequiredError: If supplied bucket is unusable. + :raises ExistingBackendError: If backend is already initialized, and force set to False. + """ + raise NotImplementedError + def upload_model(self, path: str, meta: dict, force: bool) -> str: """ Puts the given file to the remote storage. @@ -35,17 +37,20 @@ def upload_model(self, path: str, meta: dict, force: bool) -> str: :param meta: Metadata of the model. :param force: Overwrite an existing model. :return: URL of the uploaded model. - :raises TransactionRequiredError: If called not in a lock scope. + :raises BackendRequiredError: If supplied bucket is unusable. + :raises ModelAlreadyExistsError: If model already exists and no forcing. """ raise NotImplementedError - def upload_index(self, index: dict) -> None: + def fetch_model(self, source: str, file: Union[str, BinaryIO]) -> None: """ - Updates the index on the remote storage. + Downloads the model from the remote storage. - :param index: The new index. + :param source: URL to download. + :param file: Path to the local file to write or open file object. :return: None - :raises TransactionRequiredError: If called not in a lock scope. + :raises BackendRequiredError: If supplied bucket is unusable. + :raises NonExistingModelError: If model does not exist. """ raise NotImplementedError @@ -54,19 +59,27 @@ def delete_model(self, meta: dict): Deletes the model associated to the metadata dictionary from the remote storage. :param meta: Metadata of the model. - :raises TransactionRequiredError: If called not in a lock scope. + :raises BackendRequiredError: If supplied bucket is unusable. + :raises NonExistingModelError: If model does not exist. """ raise NotImplementedError -class TransactionRequiredError(Exception): +class ExistingBackendError(Exception): + """ + User tried to initialize a backend that already was initialized without forcing. + """ + pass + + +class BackendRequiredError(Exception): """ - User tried to change the index and did not acquire a lock. + User tried to publish or delete a model, but the supplied bucket parameters were incorrect. """ pass -class ModelExistsError(Exception): +class ModelAlreadyExistsError(Exception): """ User tried to publish a model that already exists without forcing. """ diff --git a/modelforge/tests/fake_requests.py b/modelforge/tests/fake_requests.py index 0023c5c..cdcd557 100644 --- a/modelforge/tests/fake_requests.py +++ b/modelforge/tests/fake_requests.py @@ -6,6 +6,12 @@ def __init__(self, content): def headers(self): return {"content-length": len(self.content)} + @property + def status_code(self): + if self.content == 404: + return 404 + return 200 + def iter_content(self, chunk_size): return [self.content[i:i+chunk_size] for i in range(0, len(self.content), chunk_size)] diff --git a/modelforge/tests/test_backends.py b/modelforge/tests/test_backends.py index 070aa9c..5dbe2c7 100644 --- a/modelforge/tests/test_backends.py +++ b/modelforge/tests/test_backends.py @@ -105,19 +105,17 @@ def test_optional(args, backend, log): self.assertIsNone(test_optional(argparse.Namespace( index_repo=self.default_url, username="", password="", cache=self.cached_path, - log_level="WARNING", template_model="", template_readme=""))) + log_level="WARNING"))) shutil.rmtree(self.cached_path) - self.assertEqual(test_optional(argparse.Namespace( - input=input, backend="none", args="", index_repo=self.default_url, username="", - password="", cache=self.cached_path, log_level="WARNING", template_model="", - template_readme="")), 1) + self.assertEqual(test_optional(self._get_args(index_repo=self.default_url)), 1) shutil.rmtree(self.cached_path) + self.tearDown() + self.assertEqual(1, test_optional(self._get_args(index_repo="any_error_really"))) - def _get_args(self): + def _get_args(self, index_repo): return argparse.Namespace( - input=input, backend="none", args="", index_repo=self.default_url, username="", - password="", cache=self.cached_path, log_level="WARNING", template_model="", - template_readme="") + backend="none", args="", index_repo=index_repo, username="", + password="", cache=self.cached_path, log_level="WARNING") if __name__ == "__main__": diff --git a/modelforge/tests/test_index.py b/modelforge/tests/test_index.py index a80bb49..3444bec 100644 --- a/modelforge/tests/test_index.py +++ b/modelforge/tests/test_index.py @@ -57,7 +57,7 @@ def test_init_base(self): self.assertListEqual(sorted(os.listdir("/tmp/modelforge-test-cache/src-d/models/docfreq")), ["12345678-9abc-def0-1234-56789abcdef0.md", "1e3da42a-28b6-4b33-94a2-a5671f4102f4.md"]) - self.assertEqual(git_index.content, self.default_index) + self.assertEqual(git_index.contents, self.default_index) self.assertEqual(git_index.models, self.default_index["models"]) self.assertEqual(git_index.meta, self.default_index["meta"]) @@ -282,7 +282,7 @@ def test_initialize(self): _out.write("nothing") git_index.initialize_index() empty_index = {"models": {}, "meta": {}} - self.assertDictEqual(empty_index, git_index.content) + self.assertDictEqual(empty_index, git_index.contents) self.assertTrue(os.path.exists(git_index.cached_repo)) self.assertListEqual(os.listdir(git_index.cached_repo), [".gitignore"]) diff --git a/modelforge/tests/test_model.py b/modelforge/tests/test_model.py index 7e69e83..f74498a 100644 --- a/modelforge/tests/test_model.py +++ b/modelforge/tests/test_model.py @@ -206,6 +206,20 @@ def route(url): model = FakeModel(backend=self.backend) self._validate_meta(model) + def test_bad_code(self): + def route(url): + self.assertEqual("https://bad_code", url) + return 404 + + back.requests = FakeRequests(route) + success = True + try: + GenericModel(source="https://bad_code", backend=self.backend) + success = False + except ValueError: + pass + self.assertTrue(success) + def test_init_with_model(self): model1 = Model1().load(source=get_path(self.DOCFREQ_PATH)) # init with correct model