From f1950b6bd198b63759dd5645913076deb0f1b7b7 Mon Sep 17 00:00:00 2001 From: Vin Date: Wed, 6 Apr 2022 02:43:56 -0400 Subject: [PATCH 01/16] Added Soybean SNP support (DB/endpoint python file/pytest) --- api/__init__.py | 2 + api/models/soybean_nssnp.py | 57 ++++++++++ api/resources/snps.py | 46 +++++--- config/BAR_API.cfg | 1 + config/databases/soybean_nssnp.sql | 165 +++++++++++++++++++++++++++++ config/init.sh | 1 + tests/resources/test_snps.py | 61 ++++++++++- 7 files changed, 318 insertions(+), 15 deletions(-) create mode 100644 api/models/soybean_nssnp.py create mode 100644 config/databases/soybean_nssnp.sql diff --git a/api/__init__.py b/api/__init__.py index 84660184..e95e7963 100644 --- a/api/__init__.py +++ b/api/__init__.py @@ -91,6 +91,7 @@ def create_app(): eplant_rice_db.init_app(bar_app) eplant_tomato_db.init_app(bar_app) poplar_nssnp_db.init_app(bar_app) + soybean_nssnp_db.init_app(bar_app) tomato_nssnp_db.init_app(bar_app) tomato_seq_db.init_app(bar_app) single_cell_db.init_app(bar_app) @@ -152,6 +153,7 @@ def create_app(): eplant_tomato_db = SQLAlchemy(metadata=MetaData()) poplar_nssnp_db = SQLAlchemy(metadata=MetaData()) tomato_nssnp_db = SQLAlchemy(metadata=MetaData()) +soybean_nssnp_db = SQLAlchemy(metadata=MetaData()) tomato_seq_db = SQLAlchemy(metadata=MetaData()) single_cell_db = SQLAlchemy(metadata=MetaData()) summarization_db = SQLAlchemy(metadata=MetaData()) diff --git a/api/models/soybean_nssnp.py b/api/models/soybean_nssnp.py new file mode 100644 index 00000000..1cd48770 --- /dev/null +++ b/api/models/soybean_nssnp.py @@ -0,0 +1,57 @@ +from api import soybean_nssnp_db as db + + +class ProteinReference(db.Model): + __bind_key__ = "soybean_nssnp" + __tablename__ = "protein_reference" + __table_args__ = ( + db.Index("protein_gene_id_idx", "gene_identifier"), + ) + protein_reference_id = db.Column(db.Integer(), primary_key=True) + gene_identifier = db.Column(db.String(45), primary_key=False) + gene_name = db.Column(db.String(45), primary_key=False) + proteinsJoin = db.relationship("SnpsToProtein", backref="prot") + + +class SnpsToProtein(db.Model): + __bind_key__ = "soybean_nssnp" + __tablename__ = "snps_to_protein" + snps_reference_id = db.Column( + db.Integer(), + db.ForeignKey("snps_reference.snps_reference_id"), + primary_key=True, + ) + protein_reference_id = db.Column( + db.Integer(), + db.ForeignKey("protein_reference.protein_reference_id"), + primary_key=True, + ) + transcript_pos = db.Column(db.Integer(), primary_key=False) + ref_DNA = db.Column(db.String(1), primary_key=False) + alt_DNA = db.Column(db.String(1), primary_key=False) + aa_pos = db.Column(db.Integer(), primary_key=False) + ref_aa = db.Column(db.String(3), primary_key=False) + alt_aa = db.Column(db.String(3), primary_key=False) + type = db.Column(db.String(50), primary_key=False) + effect_impact = db.Column(db.String(50), primary_key=False) + transcript_biotype = db.Column(db.String(45), primary_key=False) + + +class SnpsReference(db.Model): + __bind_key__ = "soybean_nssnp" + __tablename__ = "snps_reference" + snps_reference_id = db.Column(db.Integer(), primary_key=True) + chromosome = db.Column(db.Integer(), primary_key=False) + chromosomal_loci = db.Column(db.Integer(), primary_key=False) + ref_allele = db.Column(db.String(1), primary_key=False) + alt_allele = db.Column(db.String(1), primary_key=False) + sample_id = db.Column(db.String(45), primary_key=False) + snpsJoin = db.relationship("SnpsToProtein", backref="snp") + + +class SamplesLookup(db.Model): + __bind_key__ = "soybean_nssnp" + __tablename__ = "sample_lookup" + sample_id = db.Column(db.String(45), primary_key=True) + dataset = db.Column(db.String(45), primary_key=False) + dataset_sample = db.Column(db.String(45), primary_key=False) diff --git a/api/resources/snps.py b/api/resources/snps.py index 9709ccc2..e49a4fea 100644 --- a/api/resources/snps.py +++ b/api/resources/snps.py @@ -12,8 +12,14 @@ SnpsReference as TomatoSnpsReference, LinesLookup as TomatoLinesLookup, ) +from api.models.soybean_nssnp import ( + ProteinReference as SoybeanProteinReference, + SnpsToProtein as SoybeanSnpsToProtein, + SnpsReference as SoybeanSnpsReference, + SamplesLookup as SoybeanSampleNames, +) from api.utils.bar_utils import BARUtils -from api import cache, poplar_nssnp_db, tomato_nssnp_db +from api import cache, poplar_nssnp_db, tomato_nssnp_db, soybean_nssnp_db import re import subprocess import requests @@ -87,10 +93,13 @@ class GeneNameAlias(Resource): @snps.param("gene_id", _in="path", default="Potri.019G123900.1") @cache.cached() def get(self, species="", gene_id=""): - """Endpoint returns annotated SNP poplar data in order of (to match A th API format): + """ + Supported species = poplar, soybean, tomato + Endpoint returns annotated SNP poplar data in order of (to match A th API format): AA pos (zero-indexed), sample id, 'missense_variant','MODERATE', 'MISSENSE', codon/DNA base change, AA change (DH), pro length, gene ID, 'protein_coding', 'CODING', transcript id, biotype - values with single quotes are fixed""" + values with single quotes are fixed + """ results_json = [] # Escape input @@ -106,6 +115,11 @@ def get(self, species="", gene_id=""): protein_reference = TomatoProteinReference snps_to_protein = TomatoSnpsToProtein snps_reference = TomatoSnpsReference + elif species == "soybean" and BARUtils.is_soybean_gene_valid(gene_id): + query_db = soybean_nssnp_db + protein_reference = SoybeanProteinReference + snps_to_protein = SoybeanSnpsToProtein + snps_reference = SoybeanSnpsReference else: return BARUtils.error_exit("Invalid gene id"), 400 @@ -159,7 +173,7 @@ def get(self, species="", gene_id=""): @snps.route("//samples") class SampleDefinitions(Resource): @snps.param("species", _in="path", default="tomato") - @cache.cached() + # @cache.cached() def get(self, species=""): """ Endpoint returns sample/individual data for a given dataset(species). @@ -168,15 +182,21 @@ def get(self, species=""): aliases = {} - if species != "tomato": + if species == "tomato": + try: + rows = tomato_nssnp_db.session.query(TomatoLinesLookup).all() + except OperationalError: + return BARUtils.error_exit("An internal error has occurred"), 500 + for row in rows: + aliases[row.lines_id] = {"alias": row.alias, "species": row.species} + elif species == "soybean": + try: + rows = soybean_nssnp_db.session.query(SoybeanSampleNames).all() + except OperationalError: + return BARUtils.error_exit("An internal error has occurred"), 500 + for row in rows: + aliases[row.sample_id] = {"dataset": row.dataset, "PI number": row.dataset_sample} + else: return BARUtils.error_exit("Invalid gene id"), 400 - try: - rows = TomatoLinesLookup.query.all() - except OperationalError: - return BARUtils.error_exit("An internal error has occurred"), 500 - for row in rows: - aliases[row.lines_id] = {"alias": row.alias, "species": row.species} - # [aliases.append(row.alias) for row in rows] - return BARUtils.success_exit(aliases) diff --git a/config/BAR_API.cfg b/config/BAR_API.cfg index 5c452903..775fc9b8 100644 --- a/config/BAR_API.cfg +++ b/config/BAR_API.cfg @@ -16,6 +16,7 @@ SQLALCHEMY_BINDS = { 'summarization': 'mysql://root:root@localhost/summarization', 'poplar_nssnp' : 'mysql://root:root@localhost/poplar_nssnp', 'tomato_nssnp' : 'mysql://root:root@localhost/tomato_nssnp', + 'soybean_nssnp' : 'mysql://root:root@localhost/soybean_nssnp', 'eplant_poplar' : 'mysql://root:root@localhost/eplant_poplar', 'eplant_rice' : 'mysql://root:root@localhost/eplant_rice', 'eplant_tomato' : 'mysql://root:root@localhost/eplant_tomato', diff --git a/config/databases/soybean_nssnp.sql b/config/databases/soybean_nssnp.sql new file mode 100644 index 00000000..7174a893 --- /dev/null +++ b/config/databases/soybean_nssnp.sql @@ -0,0 +1,165 @@ +-- MySQL dump 10.13 Distrib 8.0.23, for Linux (x86_64) +-- +-- Host: localhost Database: soybean_nssnp +-- ------------------------------------------------------ +-- Server version 8.0.23 + +/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; +/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; +/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; +/*!50503 SET NAMES utf8mb4 */; +/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; +/*!40103 SET TIME_ZONE='+00:00' */; +/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; +/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; +/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; +/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; + +-- +-- Current Database: `soybean_nssnp` +-- + +CREATE DATABASE /*!32312 IF NOT EXISTS*/ `soybean_nssnp` /*!40100 DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci */ /*!80016 DEFAULT ENCRYPTION='N' */; + +USE `soybean_nssnp`; + +-- +-- Table structure for table `protein_reference` +-- + +DROP TABLE IF EXISTS `protein_reference`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!50503 SET character_set_client = utf8mb4 */; +CREATE TABLE `protein_reference` ( + `protein_reference_id` int NOT NULL AUTO_INCREMENT, + `gene_identifier` varchar(45) NOT NULL, + `gene_name` varchar(45) DEFAULT NULL, + PRIMARY KEY (`protein_reference_id`), + INDEX `protein_gene_id_idx` (`gene_identifier`) + -- UNIQUE KEY `gene_identifier_UNIQUE` (`gene_identifier`) + -- NB: UNIQUE KEY `gene_identifier_UNIQUE` (`gene_identifier`) is not used, as there are some duplicates in soybean dataset + -- Instead: CREATE INDEX protein_gene_id_idx ON protein_reference (gene_identifier); +) ENGINE=InnoDB AUTO_INCREMENT=88178 DEFAULT CHARSET=utf8mb4; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `protein_reference` +-- + +LOCK TABLES `protein_reference` WRITE; +/*!40000 ALTER TABLE `protein_reference` DISABLE KEYS */; +INSERT INTO `protein_reference` VALUES (18,'GLYMA.01G000100','KRH74114'); +INSERT INTO `protein_reference` VALUES (19,'GLYMA.01G000200','KRH74115'); +INSERT INTO `protein_reference` VALUES (20,'GLYMA.01G000300','KRH74115'); +/*!40000 ALTER TABLE `protein_reference` ENABLE KEYS */; +UNLOCK TABLES; + +-- +-- Table structure for table `snps_reference` +-- + +DROP TABLE IF EXISTS `snps_reference`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!50503 SET character_set_client = utf8mb4 */; +CREATE TABLE `snps_reference` ( + `snps_reference_id` int NOT NULL AUTO_INCREMENT, + `chromosome` int NOT NULL, + `chromosomal_loci` int NOT NULL, + `ref_allele` varchar(1) NOT NULL, + `alt_allele` varchar(1) NOT NULL, + `sample_id` varchar(45) NOT NULL, + PRIMARY KEY (`snps_reference_id`), + UNIQUE KEY `preventdupe` (`chromosome`,`chromosomal_loci`,`ref_allele`,`alt_allele`,`sample_id`), + KEY `index2` (`sample_id`) +) ENGINE=InnoDB AUTO_INCREMENT=22398414 DEFAULT CHARSET=utf8mb4; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `snps_reference` +-- + +LOCK TABLES `snps_reference` WRITE; +/*!40000 ALTER TABLE `snps_reference` DISABLE KEYS */; +INSERT INTO `snps_reference` VALUES (54848,1,27721,'C','A','Gm_H002'); +INSERT INTO `snps_reference` VALUES (54849,1,27721,'C','A','Gm_H003'); +INSERT INTO `snps_reference` VALUES (54850,1,27721,'C','A','Gm_H004'); +/*!40000 ALTER TABLE `snps_reference` ENABLE KEYS */; +UNLOCK TABLES; + +-- +-- Table structure for table `snps_to_protein` +-- + +DROP TABLE IF EXISTS `snps_to_protein`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!50503 SET character_set_client = utf8mb4 */; +CREATE TABLE `snps_to_protein` ( + `snps_reference_id` int NOT NULL, + `protein_reference_id` int NOT NULL, + `transcript_pos` int NOT NULL, + `ref_DNA` varchar(1) NOT NULL, + `alt_DNA` varchar(45) NOT NULL, + `aa_pos` int NOT NULL, + `ref_aa` varchar(3) NOT NULL, + `alt_aa` varchar(3) NOT NULL, + `type` varchar(50) NOT NULL, + `effect_impact` varchar(50) NOT NULL, + `transcript_biotype` varchar(45) DEFAULT NULL, + PRIMARY KEY (`snps_reference_id`,`protein_reference_id`), + KEY `protein_fk_idx` (`protein_reference_id`), + CONSTRAINT `protein_fk` FOREIGN KEY (`protein_reference_id`) REFERENCES `protein_reference` (`protein_reference_id`), + CONSTRAINT `snp_fk` FOREIGN KEY (`snps_reference_id`) REFERENCES `snps_reference` (`snps_reference_id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `snps_to_protein` +-- + +LOCK TABLES `snps_to_protein` WRITE; +/*!40000 ALTER TABLE `snps_to_protein` DISABLE KEYS */; +INSERT INTO `snps_to_protein` VALUES (54848,18,250,'G','T',84,'Val','Phe','transcript','MODERATE',NULL); +INSERT INTO `snps_to_protein` VALUES (54849,18,250,'G','T',84,'Val','Phe','transcript','MODERATE',NULL); +INSERT INTO `snps_to_protein` VALUES (54850,18,250,'G','T',84,'Val','Phe','transcript','MODERATE',NULL); +/*!40000 ALTER TABLE `snps_to_protein` ENABLE KEYS */; +UNLOCK TABLES; +/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; + +-- +-- Table structure for table `sample_lookup` +-- + +DROP TABLE IF EXISTS `sample_lookup`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!50503 SET character_set_client = utf8mb4 */; +CREATE TABLE `sample_lookup` ( + `sample_id` varchar(45) NOT NULL, + `dataset` varchar(45) DEFAULT NULL, + `dataset_sample` varchar(45) DEFAULT NULL, + PRIMARY KEY (`sample_id`), + CONSTRAINT `sample_id` FOREIGN KEY (`sample_id`) REFERENCES `snps_reference` (`sample_id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `sample_lookup` +-- + +LOCK TABLES `sample_lookup` WRITE; +/*!40000 ALTER TABLE `sample_lookup` DISABLE KEYS */; +INSERT INTO `sample_lookup` VALUES ('Gm_H002', 'Torkamaneh_Laroche_2019', 'X5302-1-52-3-2-B'); +INSERT INTO `sample_lookup` VALUES ('Gm_H003', 'Torkamaneh_Laroche_2019', 'OACInwood'); +INSERT INTO `sample_lookup` VALUES ('Gm_H004', 'Torkamaneh_Laroche_2019', 'OACDrayton'); +/*!40000 ALTER TABLE `sample_lookup` ENABLE KEYS */; +UNLOCK TABLES; +/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; + +/*!40101 SET SQL_MODE=@OLD_SQL_MODE */; +/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; +/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */; +/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; +/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; +/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; +/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; + +-- Dump completed on 2022-03-22 13:22:26 diff --git a/config/init.sh b/config/init.sh index a8f1de8e..b04e7ea9 100755 --- a/config/init.sh +++ b/config/init.sh @@ -15,6 +15,7 @@ mysql -u $DB_USER -p$DB_PASS < ./config/databases/eplant2.sql mysql -u $DB_USER -p$DB_PASS < ./config/databases/summarization.sql mysql -u $DB_USER -p$DB_PASS < ./config/databases/poplar_nssnp.sql mysql -u $DB_USER -p$DB_PASS < ./config/databases/tomato_nssnp.sql +mysql -u $DB_USER -p$DB_PASS < ./config/databases/soybean_nssnp.sql mysql -u $DB_USER -p$DB_PASS < ./config/databases/eplant_poplar.sql mysql -u $DB_USER -p$DB_PASS < ./config/databases/eplant_tomato.sql mysql -u $DB_USER -p$DB_PASS < ./config/databases/eplant_rice.sql diff --git a/tests/resources/test_snps.py b/tests/resources/test_snps.py index 593fdbf2..c92649b1 100644 --- a/tests/resources/test_snps.py +++ b/tests/resources/test_snps.py @@ -46,8 +46,8 @@ def test_get_phenix(self): expected = {"wasSuccessful": False, "error": "Invalid moving pdb gene id"} self.assertEqual(response.json, expected) - def test_get_gene_alias(self): - """This function test gene alias. + def test_get_snps(self): + """This function will test retrieving SNPs for several supported species. Note: This is using proof of principle database with only one row. Testing on the BAR will fail for now. """ @@ -101,6 +101,63 @@ def test_get_gene_alias(self): } self.assertEqual(response.json, expected) + # Valid request soybean + response = self.app_client.get("/snps/soybean/GLYMA.01G000100") + expected = { + "wasSuccessful": True, + "data": [ + [ + 1, + 83, + "Gm_H002", + "missense_variant", + "MODERATE", + "MISSENSE", + "250G>T", + "ValPhe", + None, + "GLYMA.01G0001", + "protein_coding", + "CODING", + "GLYMA.01G000100", + None + ], + [ + 1, + 83, + "Gm_H003", + "missense_variant", + "MODERATE", + "MISSENSE", + "250G>T", + "ValPhe", + None, + "GLYMA.01G0001", + "protein_coding", + "CODING", + "GLYMA.01G000100", + None + ], + [ + 1, + 83, + "Gm_H004", + "missense_variant", + "MODERATE", + "MISSENSE", + "250G>T", + "ValPhe", + None, + "GLYMA.01G0001", + "protein_coding", + "CODING", + "GLYMA.01G000100", + None + ] + ] + } + self.assertEqual(response.json, expected) + # Invalid gene id response = self.app_client.get("/snps/poplar/abc") expected = {"wasSuccessful": False, "error": "Invalid gene id"} From f7a721d18c87800bc360e9cc3c109e05af0df5ac Mon Sep 17 00:00:00 2001 From: Vin Date: Fri, 8 Apr 2022 15:57:17 -0400 Subject: [PATCH 02/16] added soybean isoform db, api logic, and testing --- api/__init__.py | 2 + api/models/eplant_soybean.py | 18 +++++ api/resources/gene_annotation.py | 2 + api/resources/gene_information.py | 22 +++++++ config/BAR_API.cfg | 1 + config/databases/eplant_soybean.sql | 84 ++++++++++++++++++++++++ config/init.sh | 1 + tests/resources/test_gene_information.py | 59 +++++++++++++++++ 8 files changed, 189 insertions(+) create mode 100644 api/models/eplant_soybean.py create mode 100644 config/databases/eplant_soybean.sql diff --git a/api/__init__.py b/api/__init__.py index e95e7963..53eeb003 100644 --- a/api/__init__.py +++ b/api/__init__.py @@ -88,6 +88,7 @@ def create_app(): annotations_lookup_db.init_app(bar_app) eplant2_db.init_app(bar_app) eplant_poplar_db.init_app(bar_app) + eplant_soybean_db.init_app(bar_app) eplant_rice_db.init_app(bar_app) eplant_tomato_db.init_app(bar_app) poplar_nssnp_db.init_app(bar_app) @@ -150,6 +151,7 @@ def create_app(): eplant2_db = SQLAlchemy(metadata=MetaData()) eplant_poplar_db = SQLAlchemy(metadata=MetaData()) eplant_rice_db = SQLAlchemy(metadata=MetaData()) +eplant_soybean_db = SQLAlchemy(metadata=MetaData()) eplant_tomato_db = SQLAlchemy(metadata=MetaData()) poplar_nssnp_db = SQLAlchemy(metadata=MetaData()) tomato_nssnp_db = SQLAlchemy(metadata=MetaData()) diff --git a/api/models/eplant_soybean.py b/api/models/eplant_soybean.py new file mode 100644 index 00000000..3c9aa119 --- /dev/null +++ b/api/models/eplant_soybean.py @@ -0,0 +1,18 @@ +from api import eplant_soybean_db as db + + +class Isoforms(db.Model): + __bind_key__ = "eplant_soybean" + __tablename__ = "isoforms" + __table_args__ = (db.Index("idx_gene_isoform", "gene", "isoform"),) + + gene = db.Column(db.String(20), nullable=False, primary_key=True) + isoform = db.Column(db.String(24), nullable=False, primary_key=True) + + +class GeneAnnotation(db.Model): + __bind_key__ = "eplant_soybean" + __tablename__ = "gene_annotation" + + gene = db.Column(db.String(20), nullable=False, primary_key=True) + annotation = db.Column(db.String(64000), nullable=False, primary_key=False) diff --git a/api/resources/gene_annotation.py b/api/resources/gene_annotation.py index df27a912..731e29e5 100644 --- a/api/resources/gene_annotation.py +++ b/api/resources/gene_annotation.py @@ -5,6 +5,7 @@ from api.models.eplant_rice import GeneAnnotation as EplantRiceAnnotation from api.models.eplant_poplar import GeneAnnotation as EplantPoplarAnnotation from api.models.eplant_tomato import GeneAnnotation as EplantTomatoAnnotation +from api.models.eplant_soybean import GeneAnnotation as EplantSoybeanAnnotation from api.models.eplant2 import AgiAnnotation, TAIR10, GeneRIFs from api.utils.bar_utils import BARUtils from marshmallow import Schema, ValidationError, fields as marshmallow_fields @@ -26,6 +27,7 @@ def get(self, query=""): "tomato": EplantTomatoAnnotation, "poplar": EplantPoplarAnnotation, "rice": EplantRiceAnnotation, + "soybean": EplantSoybeanAnnotation, "arabidopsis": [AgiAnnotation, TAIR10, GeneRIFs], } diff --git a/api/resources/gene_information.py b/api/resources/gene_information.py index b53a683c..96932edd 100644 --- a/api/resources/gene_information.py +++ b/api/resources/gene_information.py @@ -6,6 +6,7 @@ from api.models.eplant2 import Isoforms as eplant2_isoforms from api.models.eplant_poplar import Isoforms as eplant_poplar_isoforms from api.models.eplant_tomato import Isoforms as eplant_tomato_isoforms +from api.models.eplant_soybean import Isoforms as eplant_soybean_isoforms from api.utils.bar_utils import BARUtils from marshmallow import Schema, ValidationError, fields as marshmallow_fields from api import cache @@ -108,6 +109,12 @@ def get(self, species="", gene_id=""): if not BARUtils.is_tomato_gene_valid(gene_id, False): return BARUtils.error_exit("Invalid gene id"), 400 + + elif species == "soybean": + database = eplant_soybean_isoforms + + if not BARUtils.is_soybean_gene_valid(gene_id): + return BARUtils.error_exit("Invalid gene id"), 400 else: return BARUtils.error_exit("No data for the given species") @@ -189,6 +196,21 @@ def post(self): except OperationalError: return BARUtils.error_exit("An internal error has occurred."), 500 + elif species == "soybean": + database = eplant_soybean_isoforms() + + for gene in genes: + # Check if gene is valid + if not BARUtils.is_soybean_gene_valid(gene): + return BARUtils.error_exit("Invalid gene id"), 400 + + try: + rows = database.query.filter( + eplant_soybean_isoforms.gene.in_(genes) + ).all() + except OperationalError: + return BARUtils.error_exit("An internal error has occurred."), 500 + else: return BARUtils.error_exit("Invalid species"), 400 diff --git a/config/BAR_API.cfg b/config/BAR_API.cfg index 775fc9b8..7a298475 100644 --- a/config/BAR_API.cfg +++ b/config/BAR_API.cfg @@ -19,6 +19,7 @@ SQLALCHEMY_BINDS = { 'soybean_nssnp' : 'mysql://root:root@localhost/soybean_nssnp', 'eplant_poplar' : 'mysql://root:root@localhost/eplant_poplar', 'eplant_rice' : 'mysql://root:root@localhost/eplant_rice', + 'eplant_soybean' : 'mysql://root:root@localhost/eplant_soybean', 'eplant_tomato' : 'mysql://root:root@localhost/eplant_tomato', 'tomato_sequence' : 'mysql://root:root@localhost/tomato_sequence', 'rice_interactions': 'mysql://root:root@localhost/rice_interactions' diff --git a/config/databases/eplant_soybean.sql b/config/databases/eplant_soybean.sql new file mode 100644 index 00000000..a61100aa --- /dev/null +++ b/config/databases/eplant_soybean.sql @@ -0,0 +1,84 @@ +-- MySQL dump 10.13 Distrib 8.0.23, for Linux (x86_64) +-- +-- Host: localhost Database: eplant_soybean +-- ------------------------------------------------------ +-- Server version 8.0.23-3 + +/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; +/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; +/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; +/*!50503 SET NAMES utf8mb4 */; +/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; +/*!40103 SET TIME_ZONE='+00:00' */; +/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; +/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; +/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; +/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; + +-- +-- Current Database: `eplant_soybean` +-- + +CREATE DATABASE /*!32312 IF NOT EXISTS*/ `eplant_soybean` /*!40100 DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci */ /*!80016 DEFAULT ENCRYPTION='N' */; + +USE `eplant_soybean`; + +-- +-- Table structure for table `isoforms` +-- + +DROP TABLE IF EXISTS `isoforms`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!50503 SET character_set_client = utf8mb4 */; +CREATE TABLE `isoforms` ( + `gene` varchar(20) NOT NULL, + `isoform` varchar(24) NOT NULL, + KEY `idx_gene_isoform` (`gene`,`isoform`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `isoforms` +-- + +LOCK TABLES `isoforms` WRITE; +/*!40000 ALTER TABLE `isoforms` DISABLE KEYS */; +INSERT INTO `isoforms` VALUES ('Glyma.01G000100','Glyma.01G000100'),('Glyma.01G000200','Glyma.01G000200'),('Glyma.001G000400','Glyma.001G000400'); +/*!40000 ALTER TABLE `isoforms` ENABLE KEYS */; +UNLOCK TABLES; + +-- +-- Table structure for table `gene_annotation` +-- + +DROP TABLE IF EXISTS `gene_annotation`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!50503 SET character_set_client = utf8mb4 */; +CREATE TABLE `gene_annotation` ( + `gene` varchar(20) NOT NULL, + `annotation` mediumtext NOT NULL, + PRIMARY KEY (`gene`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `gene_annotation` +-- NOTE: The native eplant DB has '.1' transcript extensions for soybean hence it is here below +-- + +LOCK TABLES `gene_annotation` WRITE; +/*!40000 ALTER TABLE `gene_annotation` DISABLE KEYS */; +INSERT INTO `gene_annotation` VALUES ('Glyma.01G000100.1', '2.2.1.9//4.2.1.113//4.2.99.20 - 2-succinyl-5-enolpyruvyl-6-hydroxy-3-cyclohexene-1-carboxylic-acid synthase / SEPHCHC synthase // o-succinylbenzoate synthase / OSBS // 2-succinyl-6-hydroxy-2,4-cyclohexadiene-1-carboxylate synthase / SHCHC synthase (1 of 10)'); +/*!40000 ALTER TABLE `gene_annotation` ENABLE KEYS */; +UNLOCK TABLES; +/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; + +/*!40101 SET SQL_MODE=@OLD_SQL_MODE */; +/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; +/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */; +/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; +/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; +/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; +/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; + +-- Dump completed on 2021-06-26 19:08:20 diff --git a/config/init.sh b/config/init.sh index b04e7ea9..ebed8aae 100755 --- a/config/init.sh +++ b/config/init.sh @@ -18,6 +18,7 @@ mysql -u $DB_USER -p$DB_PASS < ./config/databases/tomato_nssnp.sql mysql -u $DB_USER -p$DB_PASS < ./config/databases/soybean_nssnp.sql mysql -u $DB_USER -p$DB_PASS < ./config/databases/eplant_poplar.sql mysql -u $DB_USER -p$DB_PASS < ./config/databases/eplant_tomato.sql +mysql -u $DB_USER -p$DB_PASS < ./config/databases/eplant_soybean.sql mysql -u $DB_USER -p$DB_PASS < ./config/databases/eplant_rice.sql mysql -u $DB_USER -p$DB_PASS < ./config/databases/tomato_sequence.sql mysql -u $DB_USER -p$DB_PASS < ./config/databases/rice_interactions.sql diff --git a/tests/resources/test_gene_information.py b/tests/resources/test_gene_information.py index 21824139..4946f93c 100644 --- a/tests/resources/test_gene_information.py +++ b/tests/resources/test_gene_information.py @@ -70,6 +70,12 @@ def test_get_arabidopsis_gene_isoform(self): expected = {"wasSuccessful": True, "data": ["Solyc00g005000.3.1"]} self.assertEqual(response.json, expected) + response = self.app_client.get( + "/gene_information/gene_isoforms/soybean/Glyma.01G000100" + ) + expected = {"wasSuccessful": True, "data": ["Glyma.01G000100"]} + self.assertEqual(response.json, expected) + # Data not found, but gene is valid response = self.app_client.get( "/gene_information/gene_isoforms/arabidopsis/At3g24651" @@ -99,6 +105,15 @@ def test_get_arabidopsis_gene_isoform(self): } self.assertEqual(response.json, expected) + response = self.app_client.get( + "/gene_information/gene_isoforms/soybean/Glyma.01G000102" + ) + expected = { + "wasSuccessful": False, + "error": "There are no data found for the given gene", + } + self.assertEqual(response.json, expected) + # Invalid Gene response = self.app_client.get( "/gene_information/gene_isoforms/arabidopsis/At3g2465x" @@ -118,6 +133,12 @@ def test_get_arabidopsis_gene_isoform(self): expected = {"wasSuccessful": False, "error": "Invalid gene id"} self.assertEqual(response.json, expected) + response = self.app_client.get( + "/gene_information/gene_isoforms/soybean/Glyma.01G00010x" + ) + expected = {"wasSuccessful": False, "error": "Invalid gene id"} + self.assertEqual(response.json, expected) + # Invalid Species response = self.app_client.get("/gene_information/gene_isoforms/x/At3g24650") expected = {"wasSuccessful": False, "error": "No data for the given species"} @@ -155,6 +176,14 @@ def test_post_arabidopsis_gene_isoform(self): } self.assertEqual(response.json, expected) + data = {"species": "soybean", "genes": ["Glyma.01G000100", "Glyma.01G000200"]} + response = self.app_client.post("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/gene_information/gene_isoforms/", json=data) + expected = { + "wasSuccessful": True, + "data": {"Glyma.01G000100": ["Glyma.01G000100"], "Glyma.01G000200": ["Glyma.01G000200"]}, + } + self.assertEqual(response.json, expected) + data = {"species": "tomato", "genes": ["Solyc00g005000"]} response = self.app_client.post("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/gene_information/gene_isoforms/", json=data) expected = { @@ -182,6 +211,15 @@ def test_post_arabidopsis_gene_isoform(self): expected = {"wasSuccessful": False, "error": {"abc": ["Unknown field."]}} self.assertEqual(response.json, expected) + data = { + "species": "soybean", + "genes": ["Glyma.01G000100", "Glyma.01G000200"], + "abc": "xyz", + } + response = self.app_client.post("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/gene_information/gene_isoforms/", json=data) + expected = {"wasSuccessful": False, "error": {"abc": ["Unknown field."]}} + self.assertEqual(response.json, expected) + # Data not found for a valid gene data = {"species": "arabidopsis", "genes": ["AT1G01011", "AT1G01020"]} response = self.app_client.post("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/gene_information/gene_isoforms/", json=data) @@ -199,6 +237,14 @@ def test_post_arabidopsis_gene_isoform(self): } self.assertEqual(response.json, expected) + data = {"species": "soybean", "genes": ["Glyma.01G000100", "Glyma.01G000101"]} + response = self.app_client.post("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/gene_information/gene_isoforms/", json=data) + expected = { + "wasSuccessful": True, + "data": {"Glyma.01G000100": ["Glyma.01G000100"]}, + } + self.assertEqual(response.json, expected) + # Check if arabidopsis gene is valid data = {"species": "arabidopsis", "genes": ["AT1G01011", "AT1G01020"]} response = self.app_client.post("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/gene_information/gene_isoforms/", json=data) @@ -224,6 +270,11 @@ def test_post_arabidopsis_gene_isoform(self): expected = {"wasSuccessful": False, "error": "Invalid gene id"} self.assertEqual(response.json, expected) + data = {"species": "soybean", "genes": ["abc"]} + response = self.app_client.post("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/gene_information/gene_isoforms/", json=data) + expected = {"wasSuccessful": False, "error": "Invalid gene id"} + self.assertEqual(response.json, expected) + # Check if there is data for the given gene data = {"species": "arabidopsis", "genes": ["AT1G01011"]} response = self.app_client.post("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/gene_information/gene_isoforms/", json=data) @@ -263,3 +314,11 @@ def test_post_arabidopsis_gene_isoform(self): "error": "No data for the given species/genes", } self.assertEqual(response.json, expected) + + data = {"species": "soybean", "genes": ["Glyma.01G000101"]} + response = self.app_client.post("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/gene_information/gene_isoforms/", json=data) + expected = { + "wasSuccessful": False, + "error": "No data for the given species/genes", + } + self.assertEqual(response.json, expected) From 6acc7e32e565f72f515fe278899e3f3b598242cc Mon Sep 17 00:00:00 2001 From: Bruno Pereira Date: Tue, 12 Apr 2022 12:34:24 -0400 Subject: [PATCH 03/16] Add median endpoints --- .../summarization_gene_expression.py | 79 +++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/api/resources/summarization_gene_expression.py b/api/resources/summarization_gene_expression.py index b2dc7286..999d9ca8 100644 --- a/api/resources/summarization_gene_expression.py +++ b/api/resources/summarization_gene_expression.py @@ -11,7 +11,9 @@ from flask_restx import Namespace, Resource from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.inspection import inspect +from sqlalchemy import func from scour.scour import scourString +from math import floor from cryptography.fernet import Fernet from api.utils.summarization_gene_expression_utils import ( SummarizationGeneExpressionUtils, @@ -506,3 +508,80 @@ def post(self): return BARUtils.success_exit(out_string) else: return BARUtils.error_exit("Invalid API key") + + +@summarization_gene_expression.route("/add_median_ctrls") +class SummarizationGeneExpressionAddMedianCtrls(Resource): + def post(self): + if request.method == "POST": + api_key = request.headers.get("x-api-key") + if api_key is None: + return BARUtils.error_exit("Invalid API key"), 403 + elif SummarizationGeneExpressionUtils.decrement_uses(api_key): + con = db.get_engine(bind="summarization") + tbl = SummarizationGeneExpressionUtils.get_table_object(api_key) + values = [] + try: + rows = con.execute(db.select([tbl.c.data_probeset_id]).distinct()) + except SQLAlchemyError as e: + print(e) + return BARUtils.error_exit("Internal server error"), 500 + # Get all genes + [values.append(row.data_probeset_id) for row in rows] + for gene in values: + signals = [] + print(gene) + try: + rows = con.execute(db.select([tbl.c.data_signal]).where(tbl.c.data_probeset_id == gene)) + except SQLAlchemyError as e: + print(e) + return BARUtils.error_exit("Internal server error"), 500 + # Get values for this gene + [signals.append(row.data_signal) for row in rows] + # Sort values + signals.sort() + # Get middle value(s) + if len(signals) % 2 == 0: + median = (signals[int(len(signals)/2)-1] + signals[int(len(signals)/2)]) / 2 + else: + median = signals[floor(len(signals)/2)] + # Insert as CTRL_Median + try: + last_index = con.execute(db.select(func.max(tbl.c.sample_id))).scalar() + print(last_index) + con.execute(db.insert(tbl).values(index=last_index+1, proj_id=1, sample_id=last_index+1, data_probeset_id=gene, data_bot_id="CTRL_Median", data_signal=median)) + last_index = last_index + 1 + except SQLAlchemyError as e: + print(e) + return BARUtils.error_exit("Internal server error"), 500 + return BARUtils.success_exit("CTRL_Medians added") + + +@summarization_gene_expression.route("/get_median/") +class SummarizationGeneExpressionAddMedianCtrls(Resource): + @summarization_gene_expression.param("gene", _in="path", default="AT1G01010") + def get(self, gene): + if request.method == "GET": + api_key = request.headers.get("x-api-key") + if api_key is None: + return BARUtils.error_exit("Invalid API key"), 403 + elif SummarizationGeneExpressionUtils.decrement_uses(api_key): + con = db.get_engine(bind="summarization") + tbl = SummarizationGeneExpressionUtils.get_table_object(api_key) + signals = [] + try: + rows = con.execute(db.select([tbl.c.data_signal]).where(tbl.c.data_probeset_id == gene)) + except SQLAlchemyError: + return BARUtils.error_exit("Internal server error"), 500 + # Get values for this gene + [signals.append(row.data_signal) for row in rows] + # Sort values + signals.sort() + # Get middle value(s) + if len(signals) % 2 == 0: + median = (signals[int(len(signals)/2)-1] + signals[int(len(signals)/2)]) / 2 + else: + median = signals[floor(len(signals)/2)] + # Insert as CTRL_Median + return BARUtils.success_exit(median) + return BARUtils.error_exit("Internal server error"), 500 \ No newline at end of file From 982aff0c44e915e544025ed85b287891ea562217 Mon Sep 17 00:00:00 2001 From: Bruno Pereira Date: Tue, 12 Apr 2022 12:37:24 -0400 Subject: [PATCH 04/16] Fix flake8 errors --- api/resources/summarization_gene_expression.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/resources/summarization_gene_expression.py b/api/resources/summarization_gene_expression.py index 999d9ca8..5a0a1feb 100644 --- a/api/resources/summarization_gene_expression.py +++ b/api/resources/summarization_gene_expression.py @@ -558,7 +558,7 @@ def post(self): @summarization_gene_expression.route("/get_median/") -class SummarizationGeneExpressionAddMedianCtrls(Resource): +class SummarizationGeneExpressionGetMedian(Resource): @summarization_gene_expression.param("gene", _in="path", default="AT1G01010") def get(self, gene): if request.method == "GET": @@ -584,4 +584,4 @@ def get(self, gene): median = signals[floor(len(signals)/2)] # Insert as CTRL_Median return BARUtils.success_exit(median) - return BARUtils.error_exit("Internal server error"), 500 \ No newline at end of file + return BARUtils.error_exit("Internal server error"), 500 From d05cec8bb5f192aa0cbef49c32cbbe75d2b9cb9e Mon Sep 17 00:00:00 2001 From: asherpasha Date: Mon, 18 Apr 2022 17:18:32 -0400 Subject: [PATCH 05/16] Requirement updates. --- requirements.txt | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/requirements.txt b/requirements.txt index 57b3b7d9..0fc7eb56 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,26 +7,27 @@ certifi==2021.10.8 cffi==1.15.0 chardet==4.0.0 charset-normalizer==2.0.12 -click==8.0.4 +click==8.1.2 coverage==6.3.2 cryptography==36.0.2 Deprecated==1.2.13 docopt==0.6.2 flake8==4.0.1 -Flask==2.1.0 +Flask==2.1.1 Flask-Caching==1.10.1 Flask-Cors==3.0.10 -Flask-Limiter==2.2.0 +Flask-Limiter==2.3.2 flask-marshmallow==0.14.0 flask-restx==0.5.1 Flask-SQLAlchemy==2.5.1 greenlet==1.1.2 idna==3.3 +importlib-metadata==4.11.3 iniconfig==1.1.1 itsdangerous==2.1.2 Jinja2==3.1.1 jsonschema==4.4.0 -limits==2.4.0 +limits==2.5.2 MarkupSafe==2.1.1 marshmallow==3.15.0 mccabe==0.6.1 @@ -37,29 +38,29 @@ numpy==1.21.5 packaging==21.3 pandas==1.3.5 pathspec==0.9.0 -platformdirs==2.5.1 +platformdirs==2.5.2 pluggy==1.0.0 py==1.11.0 pycodestyle==2.8.0 pycparser==2.21 pyflakes==2.4.0 -pyparsing==3.0.7 +pyparsing==3.0.8 pyrsistent==0.18.1 pytest==7.1.1 python-dateutil==2.8.2 pytz==2022.1 -redis==4.2.0 +redis==4.2.2 regex==2022.3.15 requests==2.27.1 scour==0.38.2 six==1.16.0 -SQLAlchemy==1.4.32 +SQLAlchemy==1.4.35 toml==0.10.2 tomli==2.0.1 -typed-ast==1.5.2 -typing_extensions==4.1.1 +typed-ast==1.5.3 +typing_extensions==4.2.0 urllib3==1.26.9 wcwidth==0.2.5 -Werkzeug==2.1.0 +Werkzeug==2.1.1 wrapt==1.14.0 -zipp==3.7.0 +zipp==3.8.0 From 15477d5e5d268fd4136b71cefe80227bd484d3da Mon Sep 17 00:00:00 2001 From: asherpasha Date: Mon, 18 Apr 2022 17:22:44 -0400 Subject: [PATCH 06/16] Requirement updates. --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 0fc7eb56..29c8c104 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,7 +22,7 @@ flask-restx==0.5.1 Flask-SQLAlchemy==2.5.1 greenlet==1.1.2 idna==3.3 -importlib-metadata==4.11.3 +importlib-metadata==4.2.0 iniconfig==1.1.1 itsdangerous==2.1.2 Jinja2==3.1.1 From 2eaa5fac65814538b4a204b449f856c97969f924 Mon Sep 17 00:00:00 2001 From: asherpasha Date: Mon, 18 Apr 2022 17:39:36 -0400 Subject: [PATCH 07/16] More updates. --- .github/workflows/bar-api.yml | 2 +- Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/bar-api.yml b/.github/workflows/bar-api.yml index 1161b4b1..f3fcb62d 100644 --- a/.github/workflows/bar-api.yml +++ b/.github/workflows/bar-api.yml @@ -13,7 +13,7 @@ jobs: runs-on: Ubuntu-20.04 strategy: matrix: - python-version: [3.7, 3.8, 3.9, 3.10.2] + python-version: [3.7, 3.8, 3.9, 3.10] services: redis: diff --git a/Dockerfile b/Dockerfile index 6ff62519..fe8d3478 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.10.2-bullseye +FROM python:3.10.4-bullseye WORKDIR /usr/src/app From deddae03417b5a3f5e9dbab0eaf668130e815368 Mon Sep 17 00:00:00 2001 From: asherpasha Date: Mon, 18 Apr 2022 17:41:20 -0400 Subject: [PATCH 08/16] More updates. --- .github/workflows/bar-api.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/bar-api.yml b/.github/workflows/bar-api.yml index f3fcb62d..2884d104 100644 --- a/.github/workflows/bar-api.yml +++ b/.github/workflows/bar-api.yml @@ -13,7 +13,7 @@ jobs: runs-on: Ubuntu-20.04 strategy: matrix: - python-version: [3.7, 3.8, 3.9, 3.10] + python-version: [3.7, 3.8, 3.9, 3.10.4] services: redis: From a0f41172d4952f22ae453dbeb4e0266052907a30 Mon Sep 17 00:00:00 2001 From: Vin Date: Wed, 20 Apr 2022 23:00:05 -0400 Subject: [PATCH 09/16] Added Yue Wei Wang's pymol endpoint code with some modifications from myself (Vincent): - fixed flake8 formatting syntax problems - Redid REST structure for endpoint (moved PDB to resource as a path param versus a query param) - Also no longer user parser() in endpoint, bug with updated flask complaining - Added rate limiter - Added limit to number of SNPs - Simplified and added commentary --- api/resources/pymol_script.py | 102 +++++++++++++++++++ api/resources/snps.py | 187 ++++++++++++++++++++++++++++++++-- tests/resources/test_snps.py | 140 ++++++++++++------------- 3 files changed, 346 insertions(+), 83 deletions(-) create mode 100644 api/resources/pymol_script.py diff --git a/api/resources/pymol_script.py b/api/resources/pymol_script.py new file mode 100644 index 00000000..22f8440c --- /dev/null +++ b/api/resources/pymol_script.py @@ -0,0 +1,102 @@ +from pymol import cmd, stored, CmdException +from sys import argv + +protein_letters = {'A': 'ALA', 'C': 'CYS', 'D': 'ASP', 'E': 'GLU', + 'F': 'PHE', 'G': 'GLY', 'H': 'HIS', 'I': 'ILE', + 'K': 'LYS', 'L': 'LEU', 'M': 'MET', 'N': 'ASN', + 'P': 'PRO', 'Q': 'GLN', 'R': 'ARG', 'S': 'SER', + 'T': 'THR', 'V': 'VAL', 'W': 'TRP', 'Y': 'TYR'} + + +def checkResidueValidation(model, chain, snps): + cmd.load("https:"+model, "target") + query_string = "i. " + if chain.upper() != 'NONE': # multimer, introduce the c. for chain selection + query_string = "c. " + chain.upper() + " & " + query_string + + for each in snps: + print('each', each) + try: + locus_selected = cmd.select(query_string + each[1:-1]) # select by residue postion + resn_selected = cmd.select( + query_string + each[1:-1] + " & resn " + protein_letters[each[0]]) # select by residue position + name + except CmdException: + print("CmdException error for select") + else: + if (locus_selected == 0): # empty select by residue position, wrong locus + rangeInfo = __findRange(chain) # get sequence start and end information + print('out of range;' + each[1:-1] + ';' + rangeInfo) + return + if (resn_selected == 0): # empty select by residue position + name, unmatch original AA + cmd.select("curr", query_string + each[1:-1]) # select the residue at the postion, named "curr" + ori = cmd.get_fastastr('curr').strip()[-1] # get the residue name for "curr" + print('invalid:' + each[1:-1] + " ori:%s" % ori) # print the correct original AA + return + + +def computeMutation(model, filename, chain, snps): + cmd.load("https:"+model, "target") + + # init mutagenesis + cmd.wizard("mutagenesis") + + # check if chain is altered + chain_str = "/target//" + if chain.upper() != "NONE": + chain_str += chain.upper() + "/" + else: + chain_str += "/" + + # looping all snps inputs + for each in snps: + cmd.get_wizard().do_select(chain_str + each[1:-1] + "/") + mut = protein_letters[each[-1].upper()] + cmd.get_wizard().set_mode(mut) + cmd.frame(1) + cmd.get_wizard().apply() + cmd.save(filename, "target", -1) + + +def __findRange(chain): + """ + helper function + return the message of input range residue position and name + """ + if chain == 'NONE': + chain_str = '' # monomer, so chain arg is empty + chain = '' + else: + chain_str = "and c. " + chain + sequence = cmd.get_fastastr("/target//"+chain) + startResidue = sequence.split("\n")[1][0] # get residue name of the first AA + endResidue = sequence.strip()[-1] # get residue name of the last AA + cmd.select("start", "(first resn {start} {chain})".format(start=protein_letters[startResidue], chain=chain_str)) # select the first + cmd.select("end", "(last resn {end} {chain})".format(end=protein_letters[endResidue], chain=chain_str)) # select the last + stored.residues = [] # place holder array + cmd.iterate("start", 'stored.residues.append(resv)') # append the first residue postion int to place holder + cmd.iterate("end", 'stored.residues.append(resv)') # append the last residue position int + if chain == '': + return "residues range start from {start}({startRes}) to {end}({endRes})".format( + start=stored.residues[0], + startRes=startResidue, end=stored.residues[1], endRes=endResidue + ) + else: + return "residues range in chain {c} start from {start}({startRes}) to {end}({endRes})".format( + c=chain, start=stored.residues[0], + startRes=startResidue, end=stored.residues[1], endRes=endResidue + ) + + +""" +argv[2]: loading pdb url +argv[3]: filename for export +argv[4]: chain selector (for multimers, monomer is none) +argv[5:]: snps""" +# select +if argv[1] == "check_residue": + checkResidueValidation(argv[2], argv[3], argv[4:]) +elif argv[1] == 'mutate_snps': + computeMutation(argv[2], argv[3], argv[4], argv[5:]) + +cmd.extend('checkResidueValidation', checkResidueValidation) +cmd.extend('computeMutation', computeMutation) diff --git a/api/resources/snps.py b/api/resources/snps.py index e49a4fea..fc8debb8 100644 --- a/api/resources/snps.py +++ b/api/resources/snps.py @@ -19,7 +19,8 @@ SamplesLookup as SoybeanSampleNames, ) from api.utils.bar_utils import BARUtils -from api import cache, poplar_nssnp_db, tomato_nssnp_db, soybean_nssnp_db +from api import cache, poplar_nssnp_db, tomato_nssnp_db, soybean_nssnp_db, limiter +from flask import request import re import subprocess import requests @@ -93,13 +94,10 @@ class GeneNameAlias(Resource): @snps.param("gene_id", _in="path", default="Potri.019G123900.1") @cache.cached() def get(self, species="", gene_id=""): - """ - Supported species = poplar, soybean, tomato - Endpoint returns annotated SNP poplar data in order of (to match A th API format): + """Endpoint returns annotated SNP poplar data in order of (to match A th API format): AA pos (zero-indexed), sample id, 'missense_variant','MODERATE', 'MISSENSE', codon/DNA base change, AA change (DH), pro length, gene ID, 'protein_coding', 'CODING', transcript id, biotype - values with single quotes are fixed - """ + values with single quotes are fixed""" results_json = [] # Escape input @@ -173,7 +171,7 @@ def get(self, species="", gene_id=""): @snps.route("//samples") class SampleDefinitions(Resource): @snps.param("species", _in="path", default="tomato") - # @cache.cached() + @cache.cached() def get(self, species=""): """ Endpoint returns sample/individual data for a given dataset(species). @@ -200,3 +198,178 @@ def get(self, species=""): return BARUtils.error_exit("Invalid gene id"), 400 return BARUtils.success_exit(aliases) + + +parser = snps.parser() +parser.add_argument('snps', type=str, action='append', required=True, help='SNP locations, format: OriLocMut i.e. V25L', default=["V25L", "E26A"]) +parser.add_argument('chain', type=str, + help='[Optional]\n For multimers, enter chain ID only (i.e. A)\n For monomers, remain \'None\' as default.', + default='None') + + +@snps.route("/pymol/") +class Pymol(Resource): + decorators = [ + limiter.limit("5/minute") + ] + + @snps.param("model", _in="path", default="Potri.016G107900.1", description="gene ID for PDB") + @snps.expect(parser) + def get(self, model): + """ + This end point returns the SNP mutated PDB of the canonical structure. + Supported Species = 'Arabidopsis' (AGIs), Poplar (Potri), Tomato (Solyc) + Enter the gene ID, chain ID (if the structure is multimer) and substitution locations. + Click 'Add string item' button and enter the SNP (format: [AA ref letter][Loci Num][AA mutant letter] - e.g. E25A) if the task contains multiple substitution locations. + """ + chain = request.args.get("chain").upper() + snps = request.args.getlist("snps") + + arabidopsis_pdb_path = "/var/www/html/eplant_legacy/java/Phyre2-Models/Phyre2_" + poplar_pdb_path = "/var/www/html/eplant_poplar/pdb/" + tomato_pdb_path = "/var/www/html/eplant_tomato/pdb/" + pymol_path = "/var/www/html/pymol-mutated-pdbs/" + pymol_link = "//bar.utoronto.ca/pymol-mutated-pdbs/" + protein_letters = "ACDEFGHIKLMNPQRSTVWY" + arabidopsis_pdb_id_link = '//bar.utoronto.ca/eplant_legacy/java/Phyre2-Models/' # new for Arabidopsis pdb id (i.e. 2wtb) + arabidopsis_pdb_id_path = "/var/www/html/eplant_legacy/java/Phyre2-Models/" # new + + # Check if too many mutations + if len(snps) > 25: + return BARUtils.error_exit("Too many mutations, limit is 25"), 400 + + # Check if gene input is valid + if BARUtils.is_arabidopsis_gene_valid(model): + gene_pdb_path = arabidopsis_pdb_path + model.upper() + ".pdb" + elif BARUtils.is_poplar_gene_valid(model): + gene_pdb_path = ( + poplar_pdb_path + BARUtils.format_poplar(model) + ".pdb" + ) + elif BARUtils.is_tomato_gene_valid(model, True): + gene_pdb_path = tomato_pdb_path + model.capitalize() + ".pdb" + + # new: check pdb id inputs + elif len(model) == 4: # pdb id + # check if local has the pdb file already + arabidopsis_response = requests.get('https:' + arabidopsis_pdb_id_link + model.lower() + '.pdb') + + # the file cannot be found in both directory + if arabidopsis_response.status_code == 200: + gene_pdb_path = arabidopsis_pdb_id_path + model.lower() + '.pdb' # lower case + + # conduct rcsb request to check if the pdb id input is valid + else: + url = '//files.rcsb.org/download/' + model.upper()+'.pdb' + rcsb_response = requests.get('https:'+url, allow_redirects=True) + + # valid, then set the rcsb url as file input url + if rcsb_response.status_code == 200: + gene_pdb_path = url + else: + return BARUtils.error_exit("Invalid PDB id"), 400 + else: + return BARUtils.error_exit("Invalid gene id"), 400 + + # Check if all elements in snps are valid format of string + snps = [x.upper() for x in snps] + formatted_snps = [] + for each in snps: + if re.match('^[a-zA-Z][1-9][0-9]*[a-zA-Z]$', each) is None: + return BARUtils.error_exit("Invalid SNP string format"), 400 + elif each[-1] not in protein_letters or each[0] not in protein_letters: + return BARUtils.error_exit("Invalid SNP string for protein letters"), 400 + else: + formatted_snps.append(each) + + # Check any conflict duplicates (i.e. V25A, V25L) + no_duplicated_snps = list(set(formatted_snps)) # set to remove dups + loci = [re.sub("[^0-9]", "", x) for x in no_duplicated_snps] + conflict_snps_loc = list(set([x for x in loci if loci.count(x) > 1])) + list_len = len(conflict_snps_loc) + if list_len > 0: + return BARUtils.error_exit("Conflict SNPs input at loci: %s" + % [int(x) for x in conflict_snps_loc]), 400 + + # Sort snps in location in order and generate pdb filename + no_duplicated_snps.sort(key=lambda x: int(x[1:-1])) + snps_string = "" + for each in no_duplicated_snps: + snps_string += "-" + each + + # new: filename with chain name for multimers0 + if chain != 'NONE': + filename = model.upper() + '-' + chain + snps_string + ".pdb" + else: + filename = model.upper() + snps_string + ".pdb" + + # Check if all snps are within sequence range + # wd_path = os.getcwd() # the wd for all later pymol tasks. Should be root (/var/www/html) during PROD + wd_path = "/var/www/html" + + # new: separate the loading url from rcsb and from bar + if 'rcsb' in gene_pdb_path: + loading_url = gene_pdb_path + else: # bar.utoronto.ca server files + loading_url = str(gene_pdb_path).replace("/var/www/html/", "//bar.utoronto.ca/") + pymol_script_path = "./api/resources/pymol_script.py" # the wd for pymol_script.py + + # 1. chain validation + # new: checking pdb file instead of running pymol_script.py + try: + file = requests.get('https:'+loading_url, allow_redirects=True) + content = re.sub('\n', '', file.content.decode("utf-8")) + first_atom_row = re.search('\nATOM(.*)\n', file.content.decode("utf-8")).group(1) + except AttributeError: + return BARUtils.error_exit("Invalid entity id"), 400 + + alphabet = re.findall('[A-Z]+', first_atom_row.strip()) # Vincent Fix + if len(alphabet) == 3: # monomer + if chain != 'NONE': # but chain input is not none + return BARUtils.error_exit("Invalid chain input, the model is monomer"), 400 + else: + chain_string_index = re.search(r'CHAIN:[\s+\S+]*?;', content).span() # Looking for a CHAIN header e.g. "COMPND 3 CHAIN: A, B;" + sliced_chains = content[chain_string_index[0]+6:chain_string_index[1]-1].split(",") # e.g. ['A', 'B', 'C'] + chains = [] + for each in sliced_chains: + chains.append(each[-1]) + if chain not in chains: + return BARUtils.error_exit("Invalid chain input, chains in the model are %s" % chains), 400 + + # 2. original AAs match the model: + check_snps_command = "pymol -cr " + pymol_script_path + " -- check_residue " \ + + loading_url + " " \ + + chain\ + + snps_string.replace('-', ' ') + + check_res_output = subprocess.run([check_snps_command], shell=True, stdout=subprocess.PIPE) + + check_res_message = (check_res_output.stdout.splitlines()[-1]).decode("utf-8") + print(check_res_message) + if 'invalid' in check_res_message: + loc = check_res_message.split(' ')[0].split(':')[1] + ori = check_res_message.split(' ')[1].split(':')[1] + return BARUtils.error_exit("Invalid SNP input, residue {loc} of the model is {ori}".format( + loc=loc, ori=ori)), 400 + elif 'out of range' in check_res_message: + loc = check_res_message.split(';')[1] + range_info = check_res_message.split(';')[2] + return BARUtils.error_exit("Invalid SNP input, locus {loc} out of range, {info}".format( + loc=loc, info=range_info)), 400 + elif "pymol.CmdException" in check_res_message: + return BARUtils.error_exit("Internal error in checking residues"), 500 + + # Search if the query already exists + response = requests.get("https:" + pymol_link + filename) + + if response.status_code != 200: + # Execute mutate_snps, saving at wd_path: /var/www/html/pymol/ + execute_command = "pymol -cr " + pymol_script_path + " -- mutate_snps " \ + + loading_url + " " \ + + wd_path + pymol_path + filename + " " \ + + chain \ + + snps_string.replace('-', ' ') + subprocess.run([execute_command], shell=True, cwd=wd_path) + # currently return url of local folder: wd_path/var/www/html/pymol + # return BARUtils.success_exit(wd_path + pymol_path + filename) + # should use pymol_link in API: + return BARUtils.success_exit(pymol_link + filename) diff --git a/tests/resources/test_snps.py b/tests/resources/test_snps.py index c92649b1..1f55152b 100644 --- a/tests/resources/test_snps.py +++ b/tests/resources/test_snps.py @@ -28,14 +28,6 @@ def test_get_phenix(self): } self.assertEqual(response.json, expected) - # Valid request - response = self.app_client.get("/snps/phenix/SOLYC01G097110.2.1/AT5G01040.1") - expected = { - "wasSuccessful": True, - "data": "//bar.utoronto.ca/phenix-pdbs/SOLYC01G097110.2.1-AT5G01040.1-phenix.pdb", - } - self.assertEqual(response.json, expected) - # Invalid fixed gene response = self.app_client.get("/snps/phenix/abc/AT5G01040.1") expected = {"wasSuccessful": False, "error": "Invalid fixed pdb gene id"} @@ -101,63 +93,6 @@ def test_get_snps(self): } self.assertEqual(response.json, expected) - # Valid request soybean - response = self.app_client.get("/snps/soybean/GLYMA.01G000100") - expected = { - "wasSuccessful": True, - "data": [ - [ - 1, - 83, - "Gm_H002", - "missense_variant", - "MODERATE", - "MISSENSE", - "250G>T", - "ValPhe", - None, - "GLYMA.01G0001", - "protein_coding", - "CODING", - "GLYMA.01G000100", - None - ], - [ - 1, - 83, - "Gm_H003", - "missense_variant", - "MODERATE", - "MISSENSE", - "250G>T", - "ValPhe", - None, - "GLYMA.01G0001", - "protein_coding", - "CODING", - "GLYMA.01G000100", - None - ], - [ - 1, - 83, - "Gm_H004", - "missense_variant", - "MODERATE", - "MISSENSE", - "250G>T", - "ValPhe", - None, - "GLYMA.01G0001", - "protein_coding", - "CODING", - "GLYMA.01G000100", - None - ] - ] - } - self.assertEqual(response.json, expected) - # Invalid gene id response = self.app_client.get("/snps/poplar/abc") expected = {"wasSuccessful": False, "error": "Invalid gene id"} @@ -171,17 +106,70 @@ def test_get_snps(self): } self.assertEqual(response.json, expected) - def test_get_samples(self): - """This functions test sample. Maybe this is place holder code?""" - # Valid data - response = self.app_client.get("/snps/tomato/samples") - expected = { - "wasSuccessful": True, - "data": {"001": {"alias": "Moneymaker", "species": "Solanum lycopersicum"}}, - } + def test_pymol_snps(self): + """ + Test for class of Pymol: + test_1: valid input + successful response + test_2: valid input + ignore cases + repeated identical SNP string + test_3: invalid input for gene name + test_4: invalid input for snps + incorrect protein letter code + test_5: invalid input for snps + incorrect format + test_6: invalid input for snps + incorrect locus + test_7: invalid input for snps + incorrect residue name + test_8: invalid input for snps + conflict strings + test_9: invalid input for chain + """ + + # test_1: valid input + successful response + response = self.app_client.get("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/snps/pymol?model=Potri.016G107900.1&snps=V25L&snps=E26A&chain=None") + expected = {"wasSuccessful": True, + "data": "//bar.utoronto.ca/pymol-mutated-pdbs/POTRI.016G107900.1-V25L-E26A.pdb"} self.assertEqual(response.json, expected) - # Invalid data - response = self.app_client.get("/snps/abc/samples") - expected = {"wasSuccessful": False, "error": "Invalid gene id"} + # test_2: valid input + ignore cases + repeated identical SNP string + response = self.app_client.get("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/snps/pymol?model=Potri.016G107900.1&snps=V25L&snps=v25l&chain=None") + expected = {"wasSuccessful": True, + "data": "//bar.utoronto.ca/pymol-mutated-pdbs/POTRI.016G107900.1-V25L.pdb"} + self.assertEqual(response.json, expected) + + # test_3: invalid input for gene name + response = self.app_client.get("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/snps/pymol?model=aaa&snps=V25L&chain=None") + expected = {"wasSuccessful": False, + "error": "Invalid gene id"} + self.assertEqual(response.json, expected) + + # test_4: invalid input for snps + incorrect protein letter code + response = self.app_client.get("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/snps/pymol?model=Potri.016G107900.1&snps=B25A&chain=None") + expected = {"wasSuccessful": False, + "error": "Invalid SNP string for protein letters"} + self.assertEqual(response.json, expected) + + # test_5: invalid input for snps + incorrect format + response = self.app_client.get("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/snps/pymol?model=Potri.016G107900.1&snps=25l&chain=None") + expected = {"wasSuccessful": False, + "error": "Invalid SNP string format"} + self.assertEqual(response.json, expected) + + # test_6: invalid input for snps + incorrect locus + response = self.app_client.get("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/snps/pymol?model=Potri.016G107900.1&snps=V1L&chain=None") + expected = {"wasSuccessful": False, + "error": "Invalid SNP input, locus 1 out of range, residues range start from 24(I) to 569(C)"} + self.assertEqual(response.json, expected) + + # test_7: invalid input for snps + incorrect residue name + response = self.app_client.get("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/snps/pymol?model=Potri.016G107900.1&snps=K25L&chain=None") + expected = {"wasSuccessful": False, + "error": "Invalid SNP input, residue 25 of the model is V"} + self.assertEqual(response.json, expected) + + # test_8: invalid input for snps + conflict strings + response = self.app_client.get("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/snps/pymol?model=Potri.016G107900.1&snps=V25L&snps=V25A&chain=None") + expected = {"wasSuccessful": False, + "error": "Conflict SNPs input at loci: [25]"} + self.assertEqual(response.json, expected) + + # test_9: invalid input for chain + response = self.app_client.get("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/snps/pymol?model=Potri.016G107900.1&snps=V25L&chain=A") + expected = {"wasSuccessful": False, + "error": "Invalid chain input, the model is monomer"} self.assertEqual(response.json, expected) From e9e69839fa05cecbd7031575091254972d6f69a8 Mon Sep 17 00:00:00 2001 From: Bruno Pereira Date: Fri, 6 May 2022 10:44:24 -0400 Subject: [PATCH 10/16] Add submit endpoint --- api/resources/api_manager.py | 7 +- .../summarization_gene_expression.py | 96 +++++++++---------- api/utils/api_manager_utils.py | 10 +- 3 files changed, 56 insertions(+), 57 deletions(-) diff --git a/api/resources/api_manager.py b/api/resources/api_manager.py index 627be14b..64836729 100644 --- a/api/resources/api_manager.py +++ b/api/resources/api_manager.py @@ -63,7 +63,12 @@ def post(self): if row_req is None and row_users is None: df.to_sql("requests", con, if_exists="append", index=False) - ApiManagerUtils.send_email_notification() + subject = "[Bio-Analytic Resource] New API key request" + text = """\ + There is a new API key request. + You can approve or reject it at http://bar.utoronto.ca/~bpereira/webservices/bar-request-manager/build/index.html + """ + ApiManagerUtils.send_email_notification(subject, text) return BARUtils.success_exit("Data added") else: return BARUtils.error_exit("E-mail already in use"), 409 diff --git a/api/resources/summarization_gene_expression.py b/api/resources/summarization_gene_expression.py index 5a0a1feb..fa68d6d6 100644 --- a/api/resources/summarization_gene_expression.py +++ b/api/resources/summarization_gene_expression.py @@ -510,53 +510,6 @@ def post(self): return BARUtils.error_exit("Invalid API key") -@summarization_gene_expression.route("/add_median_ctrls") -class SummarizationGeneExpressionAddMedianCtrls(Resource): - def post(self): - if request.method == "POST": - api_key = request.headers.get("x-api-key") - if api_key is None: - return BARUtils.error_exit("Invalid API key"), 403 - elif SummarizationGeneExpressionUtils.decrement_uses(api_key): - con = db.get_engine(bind="summarization") - tbl = SummarizationGeneExpressionUtils.get_table_object(api_key) - values = [] - try: - rows = con.execute(db.select([tbl.c.data_probeset_id]).distinct()) - except SQLAlchemyError as e: - print(e) - return BARUtils.error_exit("Internal server error"), 500 - # Get all genes - [values.append(row.data_probeset_id) for row in rows] - for gene in values: - signals = [] - print(gene) - try: - rows = con.execute(db.select([tbl.c.data_signal]).where(tbl.c.data_probeset_id == gene)) - except SQLAlchemyError as e: - print(e) - return BARUtils.error_exit("Internal server error"), 500 - # Get values for this gene - [signals.append(row.data_signal) for row in rows] - # Sort values - signals.sort() - # Get middle value(s) - if len(signals) % 2 == 0: - median = (signals[int(len(signals)/2)-1] + signals[int(len(signals)/2)]) / 2 - else: - median = signals[floor(len(signals)/2)] - # Insert as CTRL_Median - try: - last_index = con.execute(db.select(func.max(tbl.c.sample_id))).scalar() - print(last_index) - con.execute(db.insert(tbl).values(index=last_index+1, proj_id=1, sample_id=last_index+1, data_probeset_id=gene, data_bot_id="CTRL_Median", data_signal=median)) - last_index = last_index + 1 - except SQLAlchemyError as e: - print(e) - return BARUtils.error_exit("Internal server error"), 500 - return BARUtils.success_exit("CTRL_Medians added") - - @summarization_gene_expression.route("/get_median/") class SummarizationGeneExpressionGetMedian(Resource): @summarization_gene_expression.param("gene", _in="path", default="AT1G01010") @@ -579,9 +532,56 @@ def get(self, gene): signals.sort() # Get middle value(s) if len(signals) % 2 == 0: - median = (signals[int(len(signals)/2)-1] + signals[int(len(signals)/2)]) / 2 + median = float(signals[int(len(signals)/2)-1] + signals[int(len(signals)/2)]) / 2 else: median = signals[floor(len(signals)/2)] # Insert as CTRL_Median return BARUtils.success_exit(median) return BARUtils.error_exit("Internal server error"), 500 + + +@summarization_gene_expression.route("/submit", methods=["POST"], doc=False) +class SummarizationGeneExpressionSubmit(Resource): + decorators = [limiter.limit("1/minute")] + + def post(self): + print(request.files) + svg_file = request.files.get("svg") + xml_file = request.files.get("xml") + user = request.get_json()["user"] + svg_filename = secure_filename(svg_file.filename) + xml_filename = secure_filename(xml_file.filename) + key = request.headers.get("X-Api-Key") + dir_name = os.path.join("/DATA/users/www-data/", secure_filename(key)) + if not os.path.exists(dir_name): + os.makedirs(dir_name) + svg_file.save(os.path.join(dir_name, secure_filename(svg_filename))) + xml_file.save(os.path.join(dir_name, secure_filename(xml_filename))) + if SummarizationGeneExpressionUtils.decrement_uses(key): + inputs = ( + """ + { + "finalSubmissionEmail.id": """ + + key + + """, + "finalSubmissionEmail.user": """ + + user + + """, + "finalSubmissionEmail.svg": """ + + os.path.join(dir_name, secure_filename(svg_filename)) + + """, + "finalSubmissionEmail.xml": """ + + os.path.join(dir_name, secure_filename(xml_filename)) + + """ + } + """ + ) + path = os.path.join(SUMMARIZATION_FILES_PATH, "finalSubmissionEmail.wdl") + files = { + "workflowSource": ("finalSubmissionEmail.wdl", open(path, "rb")), + "workflowInputs": ("inputs.json", inputs), + } + requests.post(CROMWELL_URL + "/api/workflows/v1", files=files) + return BARUtils.success_exit(key) + else: + return BARUtils.error_exit("Invalid API key") diff --git a/api/utils/api_manager_utils.py b/api/utils/api_manager_utils.py index 2caa5577..db55a79b 100644 --- a/api/utils/api_manager_utils.py +++ b/api/utils/api_manager_utils.py @@ -48,7 +48,7 @@ def validate_captcha(value): return True @staticmethod - def send_email_notification(): + def send_email_notification(subject, msg): if os.environ.get("BAR"): with open(os.environ.get("ADMIN_EMAIL"), "r") as f: for line in f: @@ -66,14 +66,8 @@ def send_email_notification(): password = bytes(decipher_text).decode("utf-8") context = create_default_context() smtp_server = "smtp.gmail.com" - sender_email = "bar.summarization@gmail.com" - subject = "[Bio-Analytic Resource] New API key request" - text = """\ - There is a new API key request. - You can approve or reject it at http://bar.utoronto.ca/~bpereira/webservices/bar-request-manager/build/index.html - """ - + text = msg m_text = MIMEText(text, _subtype="plain", _charset="UTF-8") msg = MIMEMultipart() msg["From"] = sender_email From a83b2fb9a4e593a6b3e9d44e848d18b244fca2af Mon Sep 17 00:00:00 2001 From: Bruno Pereira Date: Fri, 6 May 2022 11:08:59 -0400 Subject: [PATCH 11/16] Fix flake errors --- api/resources/summarization_gene_expression.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/api/resources/summarization_gene_expression.py b/api/resources/summarization_gene_expression.py index fa68d6d6..3cdde7d4 100644 --- a/api/resources/summarization_gene_expression.py +++ b/api/resources/summarization_gene_expression.py @@ -11,7 +11,6 @@ from flask_restx import Namespace, Resource from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.inspection import inspect -from sqlalchemy import func from scour.scour import scourString from math import floor from cryptography.fernet import Fernet @@ -562,17 +561,17 @@ def post(self): """ { "finalSubmissionEmail.id": """ - + key - + """, + + key + + """, "finalSubmissionEmail.user": """ - + user - + """, + + user + + """, "finalSubmissionEmail.svg": """ - + os.path.join(dir_name, secure_filename(svg_filename)) - + """, + + os.path.join(dir_name, secure_filename(svg_filename)) + + """, "finalSubmissionEmail.xml": """ - + os.path.join(dir_name, secure_filename(xml_filename)) - + """ + + os.path.join(dir_name, secure_filename(xml_filename)) + + """ } """ ) From 0a014b30d5e1599e664f06641d136e4c134c24d2 Mon Sep 17 00:00:00 2001 From: Vin Date: Tue, 17 May 2022 23:50:21 -0400 Subject: [PATCH 12/16] Changes to YueWei's Pymol endpoint: - Changed how error handling was parsed. Previously was parsing standard output and creating error messages. I have refactored to make it so the error message is directly created in pymol functions and then passed to be error 404'd. No more STDOUT/STDERR parsing - Added all the pymol functions to a class method for export (Asher's request). - Changed some URLs/Error messages to be more clear - Changed unit tests for pymol endpoint to not always run as pymol will not be installed by default in CI environment The pymol unit tests can still be run by pytest --key= --- api/resources/pymol_script.py | 102 --------------------------------- api/resources/snps.py | 43 ++++---------- api/utils/pymol_script.py | 103 ++++++++++++++++++++++++++++++++++ pytest.ini | 3 + tests/resources/test_snps.py | 66 ++++++++++++---------- 5 files changed, 153 insertions(+), 164 deletions(-) delete mode 100644 api/resources/pymol_script.py create mode 100644 api/utils/pymol_script.py create mode 100644 pytest.ini diff --git a/api/resources/pymol_script.py b/api/resources/pymol_script.py deleted file mode 100644 index 22f8440c..00000000 --- a/api/resources/pymol_script.py +++ /dev/null @@ -1,102 +0,0 @@ -from pymol import cmd, stored, CmdException -from sys import argv - -protein_letters = {'A': 'ALA', 'C': 'CYS', 'D': 'ASP', 'E': 'GLU', - 'F': 'PHE', 'G': 'GLY', 'H': 'HIS', 'I': 'ILE', - 'K': 'LYS', 'L': 'LEU', 'M': 'MET', 'N': 'ASN', - 'P': 'PRO', 'Q': 'GLN', 'R': 'ARG', 'S': 'SER', - 'T': 'THR', 'V': 'VAL', 'W': 'TRP', 'Y': 'TYR'} - - -def checkResidueValidation(model, chain, snps): - cmd.load("https:"+model, "target") - query_string = "i. " - if chain.upper() != 'NONE': # multimer, introduce the c. for chain selection - query_string = "c. " + chain.upper() + " & " + query_string - - for each in snps: - print('each', each) - try: - locus_selected = cmd.select(query_string + each[1:-1]) # select by residue postion - resn_selected = cmd.select( - query_string + each[1:-1] + " & resn " + protein_letters[each[0]]) # select by residue position + name - except CmdException: - print("CmdException error for select") - else: - if (locus_selected == 0): # empty select by residue position, wrong locus - rangeInfo = __findRange(chain) # get sequence start and end information - print('out of range;' + each[1:-1] + ';' + rangeInfo) - return - if (resn_selected == 0): # empty select by residue position + name, unmatch original AA - cmd.select("curr", query_string + each[1:-1]) # select the residue at the postion, named "curr" - ori = cmd.get_fastastr('curr').strip()[-1] # get the residue name for "curr" - print('invalid:' + each[1:-1] + " ori:%s" % ori) # print the correct original AA - return - - -def computeMutation(model, filename, chain, snps): - cmd.load("https:"+model, "target") - - # init mutagenesis - cmd.wizard("mutagenesis") - - # check if chain is altered - chain_str = "/target//" - if chain.upper() != "NONE": - chain_str += chain.upper() + "/" - else: - chain_str += "/" - - # looping all snps inputs - for each in snps: - cmd.get_wizard().do_select(chain_str + each[1:-1] + "/") - mut = protein_letters[each[-1].upper()] - cmd.get_wizard().set_mode(mut) - cmd.frame(1) - cmd.get_wizard().apply() - cmd.save(filename, "target", -1) - - -def __findRange(chain): - """ - helper function - return the message of input range residue position and name - """ - if chain == 'NONE': - chain_str = '' # monomer, so chain arg is empty - chain = '' - else: - chain_str = "and c. " + chain - sequence = cmd.get_fastastr("/target//"+chain) - startResidue = sequence.split("\n")[1][0] # get residue name of the first AA - endResidue = sequence.strip()[-1] # get residue name of the last AA - cmd.select("start", "(first resn {start} {chain})".format(start=protein_letters[startResidue], chain=chain_str)) # select the first - cmd.select("end", "(last resn {end} {chain})".format(end=protein_letters[endResidue], chain=chain_str)) # select the last - stored.residues = [] # place holder array - cmd.iterate("start", 'stored.residues.append(resv)') # append the first residue postion int to place holder - cmd.iterate("end", 'stored.residues.append(resv)') # append the last residue position int - if chain == '': - return "residues range start from {start}({startRes}) to {end}({endRes})".format( - start=stored.residues[0], - startRes=startResidue, end=stored.residues[1], endRes=endResidue - ) - else: - return "residues range in chain {c} start from {start}({startRes}) to {end}({endRes})".format( - c=chain, start=stored.residues[0], - startRes=startResidue, end=stored.residues[1], endRes=endResidue - ) - - -""" -argv[2]: loading pdb url -argv[3]: filename for export -argv[4]: chain selector (for multimers, monomer is none) -argv[5:]: snps""" -# select -if argv[1] == "check_residue": - checkResidueValidation(argv[2], argv[3], argv[4:]) -elif argv[1] == 'mutate_snps': - computeMutation(argv[2], argv[3], argv[4], argv[5:]) - -cmd.extend('checkResidueValidation', checkResidueValidation) -cmd.extend('computeMutation', computeMutation) diff --git a/api/resources/snps.py b/api/resources/snps.py index fc8debb8..3edf976d 100644 --- a/api/resources/snps.py +++ b/api/resources/snps.py @@ -24,6 +24,9 @@ import re import subprocess import requests +from api.utils.pymol_script import PymolCmds +import sys + snps = Namespace("SNPs", description="Information about SNPs", path="/snps") @@ -210,7 +213,7 @@ def get(self, species=""): @snps.route("/pymol/") class Pymol(Resource): decorators = [ - limiter.limit("5/minute") + limiter.limit("6/minute") ] @snps.param("model", _in="path", default="Potri.016G107900.1", description="gene ID for PDB") @@ -302,16 +305,13 @@ def get(self, model): else: filename = model.upper() + snps_string + ".pdb" - # Check if all snps are within sequence range - # wd_path = os.getcwd() # the wd for all later pymol tasks. Should be root (/var/www/html) during PROD - wd_path = "/var/www/html" + # pymol_path = "/var/www/html" + pymol_path the wd for all later pymol tasks. Should be root (/var/www/html) during PROD # new: separate the loading url from rcsb and from bar if 'rcsb' in gene_pdb_path: loading_url = gene_pdb_path else: # bar.utoronto.ca server files loading_url = str(gene_pdb_path).replace("/var/www/html/", "//bar.utoronto.ca/") - pymol_script_path = "./api/resources/pymol_script.py" # the wd for pymol_script.py # 1. chain validation # new: checking pdb file instead of running pymol_script.py @@ -336,39 +336,18 @@ def get(self, model): return BARUtils.error_exit("Invalid chain input, chains in the model are %s" % chains), 400 # 2. original AAs match the model: - check_snps_command = "pymol -cr " + pymol_script_path + " -- check_residue " \ - + loading_url + " " \ - + chain\ - + snps_string.replace('-', ' ') - - check_res_output = subprocess.run([check_snps_command], shell=True, stdout=subprocess.PIPE) - - check_res_message = (check_res_output.stdout.splitlines()[-1]).decode("utf-8") - print(check_res_message) - if 'invalid' in check_res_message: - loc = check_res_message.split(' ')[0].split(':')[1] - ori = check_res_message.split(' ')[1].split(':')[1] - return BARUtils.error_exit("Invalid SNP input, residue {loc} of the model is {ori}".format( - loc=loc, ori=ori)), 400 - elif 'out of range' in check_res_message: - loc = check_res_message.split(';')[1] - range_info = check_res_message.split(';')[2] - return BARUtils.error_exit("Invalid SNP input, locus {loc} out of range, {info}".format( - loc=loc, info=range_info)), 400 - elif "pymol.CmdException" in check_res_message: - return BARUtils.error_exit("Internal error in checking residues"), 500 + print(snps_string, 'snps string', file=sys.stderr) + validate_aas = PymolCmds.residue_validation(loading_url, chain, snps_string.split('-')[1:]) + if validate_aas["status"] is False: + return BARUtils.error_exit(validate_aas["msg"]), 400 # Search if the query already exists response = requests.get("https:" + pymol_link + filename) if response.status_code != 200: # Execute mutate_snps, saving at wd_path: /var/www/html/pymol/ - execute_command = "pymol -cr " + pymol_script_path + " -- mutate_snps " \ - + loading_url + " " \ - + wd_path + pymol_path + filename + " " \ - + chain \ - + snps_string.replace('-', ' ') - subprocess.run([execute_command], shell=True, cwd=wd_path) + PymolCmds.compute_mutation(loading_url, pymol_path + filename, chain, snps_string.split('-')[1:]) + # currently return url of local folder: wd_path/var/www/html/pymol # return BARUtils.success_exit(wd_path + pymol_path + filename) # should use pymol_link in API: diff --git a/api/utils/pymol_script.py b/api/utils/pymol_script.py new file mode 100644 index 00000000..6ba6fcf3 --- /dev/null +++ b/api/utils/pymol_script.py @@ -0,0 +1,103 @@ +try: # try block needed for those w/o native pymol installation; pymol not in PIP + from pymol import cmd, stored, CmdException +except ImportError: + pass + +protein_letters = {'A': 'ALA', 'C': 'CYS', 'D': 'ASP', 'E': 'GLU', + 'F': 'PHE', 'G': 'GLY', 'H': 'HIS', 'I': 'ILE', + 'K': 'LYS', 'L': 'LEU', 'M': 'MET', 'N': 'ASN', + 'P': 'PRO', 'Q': 'GLN', 'R': 'ARG', 'S': 'SER', + 'T': 'THR', 'V': 'VAL', 'W': 'TRP', 'Y': 'TYR'} + + +def findRange(chain): + """ + helper function + return the message of input range residue position and name + """ + if chain == 'NONE': + chain_str = '' # monomer, so chain arg is empty + chain = '' + else: + chain_str = "and c. " + chain + sequence = cmd.get_fastastr("/target//"+chain) + startResidue = sequence.split("\n")[1][0] # get residue name of the first AA + endResidue = sequence.strip()[-1] # get residue name of the last AA + cmd.select("start", "(first resn {start} {chain})".format(start=protein_letters[startResidue], chain=chain_str)) # select the first + cmd.select("end", "(last resn {end} {chain})".format(end=protein_letters[endResidue], chain=chain_str)) # select the last + stored.residues = [] # place holder array + cmd.iterate("start", 'stored.residues.append(resv)') # append the first residue postion int to place holder + cmd.iterate("end", 'stored.residues.append(resv)') # append the last residue position int + if chain == '': + return "residues range start from {start}({startRes}) to {end}({endRes})".format( + start=stored.residues[0], + startRes=startResidue, end=stored.residues[1], endRes=endResidue + ) + else: + return "residues range in chain {c} start from {start}({startRes}) to {end}({endRes})".format( + c=chain, start=stored.residues[0], + startRes=startResidue, end=stored.residues[1], endRes=endResidue + ) + + +class PymolCmds: + @staticmethod + def residue_validation(model, chain, snps): + """ Check if AA submitted to pymol are valid in PDB model + :param model: Gene model URI e.g. //bar.utoronto.ca/eplant_poplar/pdb/Potri.016G107900.1.pdb + :param chain: Chain if available, e.g. 'NONE' + :param snps: List of SNPs, e.g. ['V25L', 'E26A'] + """ + cmd.load("https:"+model, "target") + query_string = "i. " + if chain.upper() != 'NONE': # multimer, introduce the c. for chain selection + query_string = "c. " + chain.upper() + " & " + query_string + + for each in snps: + print('each', each) + try: + locus_selected = cmd.select(query_string + each[1:-1]) # select by residue postion + resn_selected = cmd.select( + query_string + each[1:-1] + " & resn " + protein_letters[each[0]]) # select by residue position + name + except CmdException: + return {"status": False, "msg": "CmdException error for select"} + else: + if (locus_selected == 0): # empty select by residue position, wrong locus + rangeInfo = findRange(chain) # get sequence start and end information + # print('out of range;' + each[1:-1] + ';' + rangeInfo) + return {"status": False, "msg": f'Invalid SNP input range, see locus {each[1:-1]}; {rangeInfo}'} + if (resn_selected == 0): # empty select by residue position + name, unmatch original AA + cmd.select("curr", query_string + each[1:-1]) # select the residue at the postion, named "curr" + ori = cmd.get_fastastr('curr').strip()[-1] # get the residue name for "curr" + # print('invalid:' + each[1:-1] + " ori:%s" % ori) # print the correct original AA + return {"status": False, "msg": f'Invalid SNP residue, residue {each[1:-1]} of the model is {ori}'} + return {"status": True} + + @staticmethod + def compute_mutation(model, filename, chain, snps): + """Use pymol mutagensis wizard, along with pymol commands to get most predicted mutated PDB + :model: Gene model URI e.g. //bar.utoronto.ca/eplant_poplar/pdb/Potri.016G107900.1.pdb + :filename: Path of target PDB filename (includes directory, i.e. you need write access) e.g. var/www/html/pymol-mutated-pdbs/POTRI.016G107900.1-V25L-R27A.pdb + :chain: Chain if available, e.g. 'NONE' + :param snps: List of SNPs, e.g. ['V25L', 'E26A'] + """ + cmd.load("https:"+model, "target") + + # init mutagenesis + cmd.wizard("mutagenesis") + + # check if chain is altered + chain_str = "/target//" + if chain.upper() != "NONE": + chain_str += chain.upper() + "/" + else: + chain_str += "/" + + # looping all snps inputs + for each in snps: + cmd.get_wizard().do_select(chain_str + each[1:-1] + "/") + mut = protein_letters[each[-1].upper()] + cmd.get_wizard().set_mode(mut) + cmd.frame(1) + cmd.get_wizard().apply() + cmd.save(filename, "target", -1) diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 00000000..9313441a --- /dev/null +++ b/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +addopts = + -m "not pymolneeded" \ No newline at end of file diff --git a/tests/resources/test_snps.py b/tests/resources/test_snps.py index 1f55152b..39c4fb44 100644 --- a/tests/resources/test_snps.py +++ b/tests/resources/test_snps.py @@ -1,5 +1,6 @@ from api import app from unittest import TestCase +import pytest class TestIntegrations(TestCase): @@ -106,70 +107,75 @@ def test_get_snps(self): } self.assertEqual(response.json, expected) + @pytest.mark.pymolneeded def test_pymol_snps(self): """ Test for class of Pymol: test_1: valid input + successful response test_2: valid input + ignore cases + repeated identical SNP string - test_3: invalid input for gene name - test_4: invalid input for snps + incorrect protein letter code - test_5: invalid input for snps + incorrect format - test_6: invalid input for snps + incorrect locus - test_7: invalid input for snps + incorrect residue name - test_8: invalid input for snps + conflict strings - test_9: invalid input for chain + test_3: invalid input for snps + incorrect locus + test_4: invalid input for snps + incorrect residue name + test_5: invalid input for snps + conflict strings + test_6: invalid input for chain """ # test_1: valid input + successful response - response = self.app_client.get("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/snps/pymol?model=Potri.016G107900.1&snps=V25L&snps=E26A&chain=None") + response = self.app_client.get("/snps/pymol/Potri.016G107900.1?snps=V25L&snps=E26A&chain=None") expected = {"wasSuccessful": True, "data": "//bar.utoronto.ca/pymol-mutated-pdbs/POTRI.016G107900.1-V25L-E26A.pdb"} self.assertEqual(response.json, expected) # test_2: valid input + ignore cases + repeated identical SNP string - response = self.app_client.get("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/snps/pymol?model=Potri.016G107900.1&snps=V25L&snps=v25l&chain=None") + response = self.app_client.get("/snps/pymol/Potri.016G107900.1?snps=V25L&snps=v25l&chain=None") expected = {"wasSuccessful": True, "data": "//bar.utoronto.ca/pymol-mutated-pdbs/POTRI.016G107900.1-V25L.pdb"} self.assertEqual(response.json, expected) - # test_3: invalid input for gene name - response = self.app_client.get("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/snps/pymol?model=aaa&snps=V25L&chain=None") + # test_3: invalid input for snps + incorrect locus + response = self.app_client.get("/snps/pymol/Potri.016G107900.1?snps=V1L&chain=None") expected = {"wasSuccessful": False, - "error": "Invalid gene id"} + "error": "Invalid SNP input range, see locus 1; residues range start from 24(I) to 569(C)"} self.assertEqual(response.json, expected) - # test_4: invalid input for snps + incorrect protein letter code - response = self.app_client.get("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/snps/pymol?model=Potri.016G107900.1&snps=B25A&chain=None") + # test_4: invalid input for snps + incorrect residue name + response = self.app_client.get("/snps/pymol/Potri.016G107900.1?snps=K25L&chain=None") expected = {"wasSuccessful": False, - "error": "Invalid SNP string for protein letters"} + "error": "Invalid SNP residue, residue 25 of the model is V"} self.assertEqual(response.json, expected) - # test_5: invalid input for snps + incorrect format - response = self.app_client.get("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/snps/pymol?model=Potri.016G107900.1&snps=25l&chain=None") + # test_5: invalid input for snps + conflict strings + response = self.app_client.get("/snps/pymol/Potri.016G107900.1?snps=V25L&snps=V25A&chain=None") expected = {"wasSuccessful": False, - "error": "Invalid SNP string format"} + "error": "Conflict SNPs input at loci: [25]"} self.assertEqual(response.json, expected) - # test_6: invalid input for snps + incorrect locus - response = self.app_client.get("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/snps/pymol?model=Potri.016G107900.1&snps=V1L&chain=None") + # test_6: invalid input for chain + response = self.app_client.get("/snps/pymol/Potri.016G107900.1?snps=V25L&chain=A") expected = {"wasSuccessful": False, - "error": "Invalid SNP input, locus 1 out of range, residues range start from 24(I) to 569(C)"} + "error": "Invalid chain input, the model is monomer"} self.assertEqual(response.json, expected) - # test_7: invalid input for snps + incorrect residue name - response = self.app_client.get("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/snps/pymol?model=Potri.016G107900.1&snps=K25L&chain=None") + def test_pymol_snps_pymol_unneeded(self): + """This function test our Pymol endpoint for where Pymol is not needed to be installed. + These unit-tests will thus run on local environments and CI, regardless of Pymol. + test_1: invalid input for gene name + test_2: invalid input for snps + incorrect protein letter code + test_3: invalid input for snps + incorrect format + """ + # test_1: invalid input for gene name + response = self.app_client.get("/snps/pymol/aaa?snps=V25L&chain=None") expected = {"wasSuccessful": False, - "error": "Invalid SNP input, residue 25 of the model is V"} + "error": "Invalid gene id"} self.assertEqual(response.json, expected) - # test_8: invalid input for snps + conflict strings - response = self.app_client.get("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/snps/pymol?model=Potri.016G107900.1&snps=V25L&snps=V25A&chain=None") + # test_2: invalid input for snps + incorrect protein letter code + response = self.app_client.get("/snps/pymol/Potri.016G107900.1?snps=B25A&chain=None") expected = {"wasSuccessful": False, - "error": "Conflict SNPs input at loci: [25]"} + "error": "Invalid SNP string for protein letters"} self.assertEqual(response.json, expected) - # test_9: invalid input for chain - response = self.app_client.get("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/snps/pymol?model=Potri.016G107900.1&snps=V25L&chain=A") + # test_3: invalid input for snps + incorrect format + response = self.app_client.get("/snps/pymol/Potri.016G107900.1?snps=25l&chain=None") expected = {"wasSuccessful": False, - "error": "Invalid chain input, the model is monomer"} + "error": "Invalid SNP string format"} self.assertEqual(response.json, expected) From 230277425ca10f36cff4150fb281f13ffb6a086b Mon Sep 17 00:00:00 2001 From: Vin Date: Wed, 18 May 2022 15:28:08 -0400 Subject: [PATCH 13/16] small fix to REGEX in CHAIN parsing for PDB files for LGTM --- api/resources/snps.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/resources/snps.py b/api/resources/snps.py index 3edf976d..e353a1e4 100644 --- a/api/resources/snps.py +++ b/api/resources/snps.py @@ -327,7 +327,7 @@ def get(self, model): if chain != 'NONE': # but chain input is not none return BARUtils.error_exit("Invalid chain input, the model is monomer"), 400 else: - chain_string_index = re.search(r'CHAIN:[\s+\S+]*?;', content).span() # Looking for a CHAIN header e.g. "COMPND 3 CHAIN: A, B;" + chain_string_index = re.search(r'CHAIN:[\s\S]*?;', content).span() # Looking for a CHAIN header e.g. "COMPND 3 CHAIN: A, B;" sliced_chains = content[chain_string_index[0]+6:chain_string_index[1]-1].split(",") # e.g. ['A', 'B', 'C'] chains = [] for each in sliced_chains: From e53ede9264ed5ce7f2d3f8ff81462c13be93cb4b Mon Sep 17 00:00:00 2001 From: asherpasha Date: Wed, 18 May 2022 16:23:41 -0400 Subject: [PATCH 14/16] Code style. --- api/models/soybean_nssnp.py | 4 +- api/resources/gene_annotation.py | 6 +- api/resources/snps.py | 123 ++++++++++++------ .../summarization_gene_expression.py | 16 ++- api/utils/pymol_script.py | 121 ++++++++++++----- tests/resources/test_gene_annotation.py | 8 +- tests/resources/test_gene_information.py | 5 +- tests/resources/test_snps.py | 80 ++++++++---- 8 files changed, 251 insertions(+), 112 deletions(-) diff --git a/api/models/soybean_nssnp.py b/api/models/soybean_nssnp.py index 1cd48770..65dc2a1a 100644 --- a/api/models/soybean_nssnp.py +++ b/api/models/soybean_nssnp.py @@ -4,9 +4,7 @@ class ProteinReference(db.Model): __bind_key__ = "soybean_nssnp" __tablename__ = "protein_reference" - __table_args__ = ( - db.Index("protein_gene_id_idx", "gene_identifier"), - ) + __table_args__ = (db.Index("protein_gene_id_idx", "gene_identifier"),) protein_reference_id = db.Column(db.Integer(), primary_key=True) gene_identifier = db.Column(db.String(45), primary_key=False) gene_name = db.Column(db.String(45), primary_key=False) diff --git a/api/resources/gene_annotation.py b/api/resources/gene_annotation.py index 731e29e5..efc3879f 100644 --- a/api/resources/gene_annotation.py +++ b/api/resources/gene_annotation.py @@ -157,13 +157,11 @@ def post(self, query=""): try: rows = EplantRiceAnnotation.query.filter( - EplantRiceAnnotation.gene.in_(genes) + EplantRiceAnnotation.gene.in_(genes) ).all() if len(rows) == 0: return ( - BARUtils.error_exit( - "No data for the given species/genes" - ), + BARUtils.error_exit("No data for the given species/genes"), 400, ) else: diff --git a/api/resources/snps.py b/api/resources/snps.py index e353a1e4..012218fc 100644 --- a/api/resources/snps.py +++ b/api/resources/snps.py @@ -196,7 +196,10 @@ def get(self, species=""): except OperationalError: return BARUtils.error_exit("An internal error has occurred"), 500 for row in rows: - aliases[row.sample_id] = {"dataset": row.dataset, "PI number": row.dataset_sample} + aliases[row.sample_id] = { + "dataset": row.dataset, + "PI number": row.dataset_sample, + } else: return BARUtils.error_exit("Invalid gene id"), 400 @@ -204,19 +207,29 @@ def get(self, species=""): parser = snps.parser() -parser.add_argument('snps', type=str, action='append', required=True, help='SNP locations, format: OriLocMut i.e. V25L', default=["V25L", "E26A"]) -parser.add_argument('chain', type=str, - help='[Optional]\n For multimers, enter chain ID only (i.e. A)\n For monomers, remain \'None\' as default.', - default='None') +parser.add_argument( + "snps", + type=str, + action="append", + required=True, + help="SNP locations, format: OriLocMut i.e. V25L", + default=["V25L", "E26A"], +) +parser.add_argument( + "chain", + type=str, + help="[Optional]\n For multimers, enter chain ID only (i.e. A)\n For monomers, remain 'None' as default.", + default="None", +) @snps.route("/pymol/") class Pymol(Resource): - decorators = [ - limiter.limit("6/minute") - ] + decorators = [limiter.limit("6/minute")] - @snps.param("model", _in="path", default="Potri.016G107900.1", description="gene ID for PDB") + @snps.param( + "model", _in="path", default="Potri.016G107900.1", description="gene ID for PDB" + ) @snps.expect(parser) def get(self, model): """ @@ -234,8 +247,10 @@ def get(self, model): pymol_path = "/var/www/html/pymol-mutated-pdbs/" pymol_link = "//bar.utoronto.ca/pymol-mutated-pdbs/" protein_letters = "ACDEFGHIKLMNPQRSTVWY" - arabidopsis_pdb_id_link = '//bar.utoronto.ca/eplant_legacy/java/Phyre2-Models/' # new for Arabidopsis pdb id (i.e. 2wtb) - arabidopsis_pdb_id_path = "/var/www/html/eplant_legacy/java/Phyre2-Models/" # new + arabidopsis_pdb_id_link = "//bar.utoronto.ca/eplant_legacy/java/Phyre2-Models/" # new for Arabidopsis pdb id (i.e. 2wtb) + arabidopsis_pdb_id_path = ( + "/var/www/html/eplant_legacy/java/Phyre2-Models/" # new + ) # Check if too many mutations if len(snps) > 25: @@ -245,25 +260,27 @@ def get(self, model): if BARUtils.is_arabidopsis_gene_valid(model): gene_pdb_path = arabidopsis_pdb_path + model.upper() + ".pdb" elif BARUtils.is_poplar_gene_valid(model): - gene_pdb_path = ( - poplar_pdb_path + BARUtils.format_poplar(model) + ".pdb" - ) + gene_pdb_path = poplar_pdb_path + BARUtils.format_poplar(model) + ".pdb" elif BARUtils.is_tomato_gene_valid(model, True): gene_pdb_path = tomato_pdb_path + model.capitalize() + ".pdb" # new: check pdb id inputs elif len(model) == 4: # pdb id # check if local has the pdb file already - arabidopsis_response = requests.get('https:' + arabidopsis_pdb_id_link + model.lower() + '.pdb') + arabidopsis_response = requests.get( + "https:" + arabidopsis_pdb_id_link + model.lower() + ".pdb" + ) # the file cannot be found in both directory if arabidopsis_response.status_code == 200: - gene_pdb_path = arabidopsis_pdb_id_path + model.lower() + '.pdb' # lower case + gene_pdb_path = ( + arabidopsis_pdb_id_path + model.lower() + ".pdb" + ) # lower case # conduct rcsb request to check if the pdb id input is valid else: - url = '//files.rcsb.org/download/' + model.upper()+'.pdb' - rcsb_response = requests.get('https:'+url, allow_redirects=True) + url = "//files.rcsb.org/download/" + model.upper() + ".pdb" + rcsb_response = requests.get("https:" + url, allow_redirects=True) # valid, then set the rcsb url as file input url if rcsb_response.status_code == 200: @@ -277,10 +294,13 @@ def get(self, model): snps = [x.upper() for x in snps] formatted_snps = [] for each in snps: - if re.match('^[a-zA-Z][1-9][0-9]*[a-zA-Z]$', each) is None: + if re.match("^[a-zA-Z][1-9][0-9]*[a-zA-Z]$", each) is None: return BARUtils.error_exit("Invalid SNP string format"), 400 elif each[-1] not in protein_letters or each[0] not in protein_letters: - return BARUtils.error_exit("Invalid SNP string for protein letters"), 400 + return ( + BARUtils.error_exit("Invalid SNP string for protein letters"), + 400, + ) else: formatted_snps.append(each) @@ -290,8 +310,13 @@ def get(self, model): conflict_snps_loc = list(set([x for x in loci if loci.count(x) > 1])) list_len = len(conflict_snps_loc) if list_len > 0: - return BARUtils.error_exit("Conflict SNPs input at loci: %s" - % [int(x) for x in conflict_snps_loc]), 400 + return ( + BARUtils.error_exit( + "Conflict SNPs input at loci: %s" + % [int(x) for x in conflict_snps_loc] + ), + 400, + ) # Sort snps in location in order and generate pdb filename no_duplicated_snps.sort(key=lambda x: int(x[1:-1])) @@ -300,44 +325,64 @@ def get(self, model): snps_string += "-" + each # new: filename with chain name for multimers0 - if chain != 'NONE': - filename = model.upper() + '-' + chain + snps_string + ".pdb" + if chain != "NONE": + filename = model.upper() + "-" + chain + snps_string + ".pdb" else: filename = model.upper() + snps_string + ".pdb" # pymol_path = "/var/www/html" + pymol_path the wd for all later pymol tasks. Should be root (/var/www/html) during PROD # new: separate the loading url from rcsb and from bar - if 'rcsb' in gene_pdb_path: + if "rcsb" in gene_pdb_path: loading_url = gene_pdb_path else: # bar.utoronto.ca server files - loading_url = str(gene_pdb_path).replace("/var/www/html/", "//bar.utoronto.ca/") + loading_url = str(gene_pdb_path).replace( + "/var/www/html/", "//bar.utoronto.ca/" + ) # 1. chain validation # new: checking pdb file instead of running pymol_script.py try: - file = requests.get('https:'+loading_url, allow_redirects=True) - content = re.sub('\n', '', file.content.decode("utf-8")) - first_atom_row = re.search('\nATOM(.*)\n', file.content.decode("utf-8")).group(1) + file = requests.get("https:" + loading_url, allow_redirects=True) + content = re.sub("\n", "", file.content.decode("utf-8")) + first_atom_row = re.search( + "\nATOM(.*)\n", file.content.decode("utf-8") + ).group(1) except AttributeError: return BARUtils.error_exit("Invalid entity id"), 400 - alphabet = re.findall('[A-Z]+', first_atom_row.strip()) # Vincent Fix + alphabet = re.findall("[A-Z]+", first_atom_row.strip()) # Vincent Fix if len(alphabet) == 3: # monomer - if chain != 'NONE': # but chain input is not none - return BARUtils.error_exit("Invalid chain input, the model is monomer"), 400 + if chain != "NONE": # but chain input is not none + return ( + BARUtils.error_exit("Invalid chain input, the model is monomer"), + 400, + ) else: - chain_string_index = re.search(r'CHAIN:[\s\S]*?;', content).span() # Looking for a CHAIN header e.g. "COMPND 3 CHAIN: A, B;" - sliced_chains = content[chain_string_index[0]+6:chain_string_index[1]-1].split(",") # e.g. ['A', 'B', 'C'] + chain_string_index = re.search( + r"CHAIN:[\s\S]*?;", content + ).span() # Looking for a CHAIN header e.g. "COMPND 3 CHAIN: A, B;" + sliced_chains = content[ + chain_string_index[0] + 6 : chain_string_index[1] - 1 + ].split( + "," + ) # e.g. ['A', 'B', 'C'] chains = [] for each in sliced_chains: chains.append(each[-1]) if chain not in chains: - return BARUtils.error_exit("Invalid chain input, chains in the model are %s" % chains), 400 + return ( + BARUtils.error_exit( + "Invalid chain input, chains in the model are %s" % chains + ), + 400, + ) # 2. original AAs match the model: - print(snps_string, 'snps string', file=sys.stderr) - validate_aas = PymolCmds.residue_validation(loading_url, chain, snps_string.split('-')[1:]) + print(snps_string, "snps string", file=sys.stderr) + validate_aas = PymolCmds.residue_validation( + loading_url, chain, snps_string.split("-")[1:] + ) if validate_aas["status"] is False: return BARUtils.error_exit(validate_aas["msg"]), 400 @@ -346,7 +391,9 @@ def get(self, model): if response.status_code != 200: # Execute mutate_snps, saving at wd_path: /var/www/html/pymol/ - PymolCmds.compute_mutation(loading_url, pymol_path + filename, chain, snps_string.split('-')[1:]) + PymolCmds.compute_mutation( + loading_url, pymol_path + filename, chain, snps_string.split("-")[1:] + ) # currently return url of local folder: wd_path/var/www/html/pymol # return BARUtils.success_exit(wd_path + pymol_path + filename) diff --git a/api/resources/summarization_gene_expression.py b/api/resources/summarization_gene_expression.py index 3cdde7d4..12c91e15 100644 --- a/api/resources/summarization_gene_expression.py +++ b/api/resources/summarization_gene_expression.py @@ -522,7 +522,11 @@ def get(self, gene): tbl = SummarizationGeneExpressionUtils.get_table_object(api_key) signals = [] try: - rows = con.execute(db.select([tbl.c.data_signal]).where(tbl.c.data_probeset_id == gene)) + rows = con.execute( + db.select([tbl.c.data_signal]).where( + tbl.c.data_probeset_id == gene + ) + ) except SQLAlchemyError: return BARUtils.error_exit("Internal server error"), 500 # Get values for this gene @@ -531,9 +535,15 @@ def get(self, gene): signals.sort() # Get middle value(s) if len(signals) % 2 == 0: - median = float(signals[int(len(signals)/2)-1] + signals[int(len(signals)/2)]) / 2 + median = ( + float( + signals[int(len(signals) / 2) - 1] + + signals[int(len(signals) / 2)] + ) + / 2 + ) else: - median = signals[floor(len(signals)/2)] + median = signals[floor(len(signals) / 2)] # Insert as CTRL_Median return BARUtils.success_exit(median) return BARUtils.error_exit("Internal server error"), 500 diff --git a/api/utils/pymol_script.py b/api/utils/pymol_script.py index 6ba6fcf3..d0e2da18 100644 --- a/api/utils/pymol_script.py +++ b/api/utils/pymol_script.py @@ -3,11 +3,28 @@ except ImportError: pass -protein_letters = {'A': 'ALA', 'C': 'CYS', 'D': 'ASP', 'E': 'GLU', - 'F': 'PHE', 'G': 'GLY', 'H': 'HIS', 'I': 'ILE', - 'K': 'LYS', 'L': 'LEU', 'M': 'MET', 'N': 'ASN', - 'P': 'PRO', 'Q': 'GLN', 'R': 'ARG', 'S': 'SER', - 'T': 'THR', 'V': 'VAL', 'W': 'TRP', 'Y': 'TYR'} +protein_letters = { + "A": "ALA", + "C": "CYS", + "D": "ASP", + "E": "GLU", + "F": "PHE", + "G": "GLY", + "H": "HIS", + "I": "ILE", + "K": "LYS", + "L": "LEU", + "M": "MET", + "N": "ASN", + "P": "PRO", + "Q": "GLN", + "R": "ARG", + "S": "SER", + "T": "THR", + "V": "VAL", + "W": "TRP", + "Y": "TYR", +} def findRange(chain): @@ -15,62 +32,100 @@ def findRange(chain): helper function return the message of input range residue position and name """ - if chain == 'NONE': - chain_str = '' # monomer, so chain arg is empty - chain = '' + if chain == "NONE": + chain_str = "" # monomer, so chain arg is empty + chain = "" else: chain_str = "and c. " + chain - sequence = cmd.get_fastastr("/target//"+chain) + sequence = cmd.get_fastastr("/target//" + chain) startResidue = sequence.split("\n")[1][0] # get residue name of the first AA endResidue = sequence.strip()[-1] # get residue name of the last AA - cmd.select("start", "(first resn {start} {chain})".format(start=protein_letters[startResidue], chain=chain_str)) # select the first - cmd.select("end", "(last resn {end} {chain})".format(end=protein_letters[endResidue], chain=chain_str)) # select the last + cmd.select( + "start", + "(first resn {start} {chain})".format( + start=protein_letters[startResidue], chain=chain_str + ), + ) # select the first + cmd.select( + "end", + "(last resn {end} {chain})".format( + end=protein_letters[endResidue], chain=chain_str + ), + ) # select the last stored.residues = [] # place holder array - cmd.iterate("start", 'stored.residues.append(resv)') # append the first residue postion int to place holder - cmd.iterate("end", 'stored.residues.append(resv)') # append the last residue position int - if chain == '': - return "residues range start from {start}({startRes}) to {end}({endRes})".format( - start=stored.residues[0], - startRes=startResidue, end=stored.residues[1], endRes=endResidue + cmd.iterate( + "start", "stored.residues.append(resv)" + ) # append the first residue postion int to place holder + cmd.iterate( + "end", "stored.residues.append(resv)" + ) # append the last residue position int + if chain == "": + return ( + "residues range start from {start}({startRes}) to {end}({endRes})".format( + start=stored.residues[0], + startRes=startResidue, + end=stored.residues[1], + endRes=endResidue, + ) ) else: return "residues range in chain {c} start from {start}({startRes}) to {end}({endRes})".format( - c=chain, start=stored.residues[0], - startRes=startResidue, end=stored.residues[1], endRes=endResidue + c=chain, + start=stored.residues[0], + startRes=startResidue, + end=stored.residues[1], + endRes=endResidue, ) class PymolCmds: @staticmethod def residue_validation(model, chain, snps): - """ Check if AA submitted to pymol are valid in PDB model + """Check if AA submitted to pymol are valid in PDB model :param model: Gene model URI e.g. //bar.utoronto.ca/eplant_poplar/pdb/Potri.016G107900.1.pdb :param chain: Chain if available, e.g. 'NONE' :param snps: List of SNPs, e.g. ['V25L', 'E26A'] """ - cmd.load("https:"+model, "target") + cmd.load("https:" + model, "target") query_string = "i. " - if chain.upper() != 'NONE': # multimer, introduce the c. for chain selection + if chain.upper() != "NONE": # multimer, introduce the c. for chain selection query_string = "c. " + chain.upper() + " & " + query_string for each in snps: - print('each', each) + print("each", each) try: - locus_selected = cmd.select(query_string + each[1:-1]) # select by residue postion + locus_selected = cmd.select( + query_string + each[1:-1] + ) # select by residue postion resn_selected = cmd.select( - query_string + each[1:-1] + " & resn " + protein_letters[each[0]]) # select by residue position + name + query_string + each[1:-1] + " & resn " + protein_letters[each[0]] + ) # select by residue position + name except CmdException: return {"status": False, "msg": "CmdException error for select"} else: - if (locus_selected == 0): # empty select by residue position, wrong locus - rangeInfo = findRange(chain) # get sequence start and end information + if locus_selected == 0: # empty select by residue position, wrong locus + rangeInfo = findRange( + chain + ) # get sequence start and end information # print('out of range;' + each[1:-1] + ';' + rangeInfo) - return {"status": False, "msg": f'Invalid SNP input range, see locus {each[1:-1]}; {rangeInfo}'} - if (resn_selected == 0): # empty select by residue position + name, unmatch original AA - cmd.select("curr", query_string + each[1:-1]) # select the residue at the postion, named "curr" - ori = cmd.get_fastastr('curr').strip()[-1] # get the residue name for "curr" + return { + "status": False, + "msg": f"Invalid SNP input range, see locus {each[1:-1]}; {rangeInfo}", + } + if ( + resn_selected == 0 + ): # empty select by residue position + name, unmatch original AA + cmd.select( + "curr", query_string + each[1:-1] + ) # select the residue at the postion, named "curr" + ori = cmd.get_fastastr("curr").strip()[ + -1 + ] # get the residue name for "curr" # print('invalid:' + each[1:-1] + " ori:%s" % ori) # print the correct original AA - return {"status": False, "msg": f'Invalid SNP residue, residue {each[1:-1]} of the model is {ori}'} + return { + "status": False, + "msg": f"Invalid SNP residue, residue {each[1:-1]} of the model is {ori}", + } return {"status": True} @staticmethod @@ -81,7 +136,7 @@ def compute_mutation(model, filename, chain, snps): :chain: Chain if available, e.g. 'NONE' :param snps: List of SNPs, e.g. ['V25L', 'E26A'] """ - cmd.load("https:"+model, "target") + cmd.load("https:" + model, "target") # init mutagenesis cmd.wizard("mutagenesis") diff --git a/tests/resources/test_gene_annotation.py b/tests/resources/test_gene_annotation.py index 569421da..ab340bb9 100644 --- a/tests/resources/test_gene_annotation.py +++ b/tests/resources/test_gene_annotation.py @@ -54,13 +54,13 @@ def test_post_anntns(self): "data": [ { "gene": "LOC_Os01g01010", - "annotation": "protein TBC domain containing protein, expressed" + "annotation": "protein TBC domain containing protein, expressed", }, { "gene": "LOC_Os01g01050", - "annotation": "protein R3H domain containing protein, expressed" - } - ] + "annotation": "protein R3H domain containing protein, expressed", + }, + ], } self.assertEqual(data, expected) diff --git a/tests/resources/test_gene_information.py b/tests/resources/test_gene_information.py index 4946f93c..5781480d 100644 --- a/tests/resources/test_gene_information.py +++ b/tests/resources/test_gene_information.py @@ -180,7 +180,10 @@ def test_post_arabidopsis_gene_isoform(self): response = self.app_client.post("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/gene_information/gene_isoforms/", json=data) expected = { "wasSuccessful": True, - "data": {"Glyma.01G000100": ["Glyma.01G000100"], "Glyma.01G000200": ["Glyma.01G000200"]}, + "data": { + "Glyma.01G000100": ["Glyma.01G000100"], + "Glyma.01G000200": ["Glyma.01G000200"], + }, } self.assertEqual(response.json, expected) diff --git a/tests/resources/test_snps.py b/tests/resources/test_snps.py index 39c4fb44..fb5da36c 100644 --- a/tests/resources/test_snps.py +++ b/tests/resources/test_snps.py @@ -120,39 +120,63 @@ def test_pymol_snps(self): """ # test_1: valid input + successful response - response = self.app_client.get("/snps/pymol/Potri.016G107900.1?snps=V25L&snps=E26A&chain=None") - expected = {"wasSuccessful": True, - "data": "//bar.utoronto.ca/pymol-mutated-pdbs/POTRI.016G107900.1-V25L-E26A.pdb"} + response = self.app_client.get( + "/snps/pymol/Potri.016G107900.1?snps=V25L&snps=E26A&chain=None" + ) + expected = { + "wasSuccessful": True, + "data": "//bar.utoronto.ca/pymol-mutated-pdbs/POTRI.016G107900.1-V25L-E26A.pdb", + } self.assertEqual(response.json, expected) # test_2: valid input + ignore cases + repeated identical SNP string - response = self.app_client.get("/snps/pymol/Potri.016G107900.1?snps=V25L&snps=v25l&chain=None") - expected = {"wasSuccessful": True, - "data": "//bar.utoronto.ca/pymol-mutated-pdbs/POTRI.016G107900.1-V25L.pdb"} + response = self.app_client.get( + "/snps/pymol/Potri.016G107900.1?snps=V25L&snps=v25l&chain=None" + ) + expected = { + "wasSuccessful": True, + "data": "//bar.utoronto.ca/pymol-mutated-pdbs/POTRI.016G107900.1-V25L.pdb", + } self.assertEqual(response.json, expected) # test_3: invalid input for snps + incorrect locus - response = self.app_client.get("/snps/pymol/Potri.016G107900.1?snps=V1L&chain=None") - expected = {"wasSuccessful": False, - "error": "Invalid SNP input range, see locus 1; residues range start from 24(I) to 569(C)"} + response = self.app_client.get( + "/snps/pymol/Potri.016G107900.1?snps=V1L&chain=None" + ) + expected = { + "wasSuccessful": False, + "error": "Invalid SNP input range, see locus 1; residues range start from 24(I) to 569(C)", + } self.assertEqual(response.json, expected) # test_4: invalid input for snps + incorrect residue name - response = self.app_client.get("/snps/pymol/Potri.016G107900.1?snps=K25L&chain=None") - expected = {"wasSuccessful": False, - "error": "Invalid SNP residue, residue 25 of the model is V"} + response = self.app_client.get( + "/snps/pymol/Potri.016G107900.1?snps=K25L&chain=None" + ) + expected = { + "wasSuccessful": False, + "error": "Invalid SNP residue, residue 25 of the model is V", + } self.assertEqual(response.json, expected) # test_5: invalid input for snps + conflict strings - response = self.app_client.get("/snps/pymol/Potri.016G107900.1?snps=V25L&snps=V25A&chain=None") - expected = {"wasSuccessful": False, - "error": "Conflict SNPs input at loci: [25]"} + response = self.app_client.get( + "/snps/pymol/Potri.016G107900.1?snps=V25L&snps=V25A&chain=None" + ) + expected = { + "wasSuccessful": False, + "error": "Conflict SNPs input at loci: [25]", + } self.assertEqual(response.json, expected) # test_6: invalid input for chain - response = self.app_client.get("/snps/pymol/Potri.016G107900.1?snps=V25L&chain=A") - expected = {"wasSuccessful": False, - "error": "Invalid chain input, the model is monomer"} + response = self.app_client.get( + "/snps/pymol/Potri.016G107900.1?snps=V25L&chain=A" + ) + expected = { + "wasSuccessful": False, + "error": "Invalid chain input, the model is monomer", + } self.assertEqual(response.json, expected) def test_pymol_snps_pymol_unneeded(self): @@ -164,18 +188,22 @@ def test_pymol_snps_pymol_unneeded(self): """ # test_1: invalid input for gene name response = self.app_client.get("/snps/pymol/aaa?snps=V25L&chain=None") - expected = {"wasSuccessful": False, - "error": "Invalid gene id"} + expected = {"wasSuccessful": False, "error": "Invalid gene id"} self.assertEqual(response.json, expected) # test_2: invalid input for snps + incorrect protein letter code - response = self.app_client.get("/snps/pymol/Potri.016G107900.1?snps=B25A&chain=None") - expected = {"wasSuccessful": False, - "error": "Invalid SNP string for protein letters"} + response = self.app_client.get( + "/snps/pymol/Potri.016G107900.1?snps=B25A&chain=None" + ) + expected = { + "wasSuccessful": False, + "error": "Invalid SNP string for protein letters", + } self.assertEqual(response.json, expected) # test_3: invalid input for snps + incorrect format - response = self.app_client.get("/snps/pymol/Potri.016G107900.1?snps=25l&chain=None") - expected = {"wasSuccessful": False, - "error": "Invalid SNP string format"} + response = self.app_client.get( + "/snps/pymol/Potri.016G107900.1?snps=25l&chain=None" + ) + expected = {"wasSuccessful": False, "error": "Invalid SNP string format"} self.assertEqual(response.json, expected) From 4cf7d85d259ebfbf4572030f1773a7f563d729c3 Mon Sep 17 00:00:00 2001 From: Bruno Pereira Date: Thu, 2 Jun 2022 17:22:29 -0400 Subject: [PATCH 15/16] Update keys in config for BAR emails --- config/BAR_API.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/BAR_API.cfg b/config/BAR_API.cfg index 7a298475..6e551e33 100644 --- a/config/BAR_API.cfg +++ b/config/BAR_API.cfg @@ -30,7 +30,7 @@ ADMIN_EMAIL = '/home/bpereira/data/adminemail' ADMIN_PASSWORD_FILE = '/home/bpereira/dev/pw-script/managerkey.bin' ADMIN_ENCRYPT_KEY = 'fabUGnTJ3UQ4qeDJbnSMrb-tDdmt9kxLkuq3GHKdGTo=' -EMAIL_PASS_KEY = '1_2SkFWmTeFnLWtO2oUIRi8pmSd2bMDAtlWq9khpzNc=' +EMAIL_PASS_KEY = 'DMKQpa7ndkE69rvuKiXk8P0NtyPID9puYbPRtyliHiY=' EMAIL_PASS_FILE = '/home/bpereira/dev/pw-script/emailkey.bin' TEST_ADMIN_PASSWORD_FILE = './tests/data/test_key.bin' From 5a47e4d2ad7302f0b88e6f3cd77ab745e06bce27 Mon Sep 17 00:00:00 2001 From: asherpasha Date: Mon, 6 Jun 2022 21:22:49 -0400 Subject: [PATCH 16/16] Fixed ATTED service. --- requirements.txt | 44 ++++++++++++++++++---------------- tests/data/get_atted_api4.json | 32 ++++++++++++------------- 2 files changed, 40 insertions(+), 36 deletions(-) diff --git a/requirements.txt b/requirements.txt index 29c8c104..c8b14aed 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,20 +3,22 @@ appdirs==1.4.4 async-timeout==4.0.2 attrs==21.4.0 black==22.3.0 -certifi==2021.10.8 +cachelib==0.7.0 +certifi==2022.5.18.1 cffi==1.15.0 chardet==4.0.0 charset-normalizer==2.0.12 -click==8.1.2 -coverage==6.3.2 -cryptography==36.0.2 +click==8.1.3 +commonmark==0.9.1 +coverage==6.4.1 +cryptography==37.0.2 Deprecated==1.2.13 docopt==0.6.2 flake8==4.0.1 -Flask==2.1.1 -Flask-Caching==1.10.1 +Flask==2.1.2 +Flask-Caching==1.11.1 Flask-Cors==3.0.10 -Flask-Limiter==2.3.2 +Flask-Limiter==2.4.5.1 flask-marshmallow==0.14.0 flask-restx==0.5.1 Flask-SQLAlchemy==2.5.1 @@ -25,13 +27,13 @@ idna==3.3 importlib-metadata==4.2.0 iniconfig==1.1.1 itsdangerous==2.1.2 -Jinja2==3.1.1 -jsonschema==4.4.0 -limits==2.5.2 +Jinja2==3.1.2 +jsonschema==4.6.0 +limits==2.6.3 MarkupSafe==2.1.1 -marshmallow==3.15.0 +marshmallow==3.16.0 mccabe==0.6.1 -more-itertools==8.12.0 +more-itertools==8.13.0 mypy-extensions==0.4.3 mysqlclient==2.1.0 numpy==1.21.5 @@ -44,23 +46,25 @@ py==1.11.0 pycodestyle==2.8.0 pycparser==2.21 pyflakes==2.4.0 -pyparsing==3.0.8 +Pygments==2.12.0 +pyparsing==3.0.9 pyrsistent==0.18.1 -pytest==7.1.1 +pytest==7.1.2 python-dateutil==2.8.2 pytz==2022.1 -redis==4.2.2 -regex==2022.3.15 +redis==4.3.3 +regex==2022.6.2 requests==2.27.1 +rich==12.4.4 scour==0.38.2 six==1.16.0 -SQLAlchemy==1.4.35 +SQLAlchemy==1.4.37 toml==0.10.2 tomli==2.0.1 -typed-ast==1.5.3 +typed-ast==1.5.4 typing_extensions==4.2.0 urllib3==1.26.9 wcwidth==0.2.5 -Werkzeug==2.1.1 -wrapt==1.14.0 +Werkzeug==2.1.2 +wrapt==1.14.1 zipp==3.8.0 diff --git a/tests/data/get_atted_api4.json b/tests/data/get_atted_api4.json index c8abf7bc..8cb96d00 100644 --- a/tests/data/get_atted_api4.json +++ b/tests/data/get_atted_api4.json @@ -8,8 +8,8 @@ "type": null, "value": null, "topN": 5, - "database": "Ath-u.c2-0", - "database_version": "c2-0" + "database": "Ath-u.c3-0", + "database_version": "c3-0" }, "result_set": [ { @@ -18,32 +18,32 @@ { "gene": 831644, "other_id": "AT5G17760", - "mutual_rank": 482.65, - "logit_score": 5.2584 + "mutual_rank": 509.98, + "logit_score": 5.1768 }, { "gene": 842367, "other_id": "AT1G60730", - "mutual_rank": 767.19, - "logit_score": 4.5674 + "mutual_rank": 807.94, + "logit_score": 4.4895 }, { "gene": 837321, "other_id": "AT1G08050", - "mutual_rank": 1076.41, - "logit_score": 4.0541 + "mutual_rank": 1125.71, + "logit_score": 3.9855 }, { - "gene": 814770, - "other_id": "AT2G02390", - "mutual_rank": 1148.08, - "logit_score": 3.9553 + "gene": 844034, + "other_id": "AT1G76980", + "mutual_rank": 1134.11, + "logit_score": 3.9741 }, { - "gene": 832064, - "other_id": "AT5G19440", - "mutual_rank": 1148.08, - "logit_score": 3.9553 + "gene": 814770, + "other_id": "AT2G02390", + "mutual_rank": 1186.04, + "logit_score": 3.9053 } ], "other_id": "AT1G01010"