Working on #1 - FDA import process works. 3 hours for first import, 1.25 hours on subsequent updates.
This commit is contained in:
9
Makefile
9
Makefile
@@ -1,13 +1,8 @@
|
|||||||
MAJOR:=$(shell bash -c 'source version.sh ; echo $$MAJOR')
|
MAJOR:=$(shell bash -c 'source version.sh ; echo $$MAJOR')
|
||||||
BUILD:=$(shell bash -c 'source version.sh ; echo $$BUILD')
|
BUILD:=$(shell bash -c 'source version.sh ; echo $$BUILD')
|
||||||
OS_NAME:=$(shell bash -c 'source version.sh ; echo $$OS_NAME')
|
OS_NAME:=$(shell bash -c 'source version.sh ; echo $$OS_NAME')
|
||||||
ifeq "$(OS_NAME)" "win"
|
PIP=$(shell pwd)/virtualenv/bin/pip
|
||||||
PIP=$(shell pwd)/virtualenv/Scripts/pip
|
VIRTUALENV_PKGS_DIR=$(shell pwd)/virtualenv/lib/python2.7/site-packages
|
||||||
VIRTUALENV_PKGS_DIR=$(shell pwd)/virtualenv/Lib/site-packages
|
|
||||||
else
|
|
||||||
PIP=$(shell pwd)/virtualenv/bin/pip
|
|
||||||
VIRTUALENV_PKGS_DIR=$(shell pwd)/virtualenv/lib/site-packages
|
|
||||||
endif
|
|
||||||
VIRTUALENV=$(shell which virtualenv)
|
VIRTUALENV=$(shell which virtualenv)
|
||||||
PYTHON=$(shell which python)
|
PYTHON=$(shell which python)
|
||||||
|
|
||||||
|
|||||||
@@ -1,4 +1,22 @@
|
|||||||
import flask
|
import flask
|
||||||
|
import mercy.config
|
||||||
|
from flask.ext.sqlalchemy import SQLAlchemy
|
||||||
|
|
||||||
class MercyApplication(flask.Flask):
|
class MercyApplication(flask.Flask):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
app = None
|
||||||
|
db = None
|
||||||
|
|
||||||
|
def get_db():
|
||||||
|
global db
|
||||||
|
if not db:
|
||||||
|
db = SQLAlchemy(get_app())
|
||||||
|
return db
|
||||||
|
|
||||||
|
def get_app():
|
||||||
|
global app
|
||||||
|
if not app:
|
||||||
|
app = MercyApplication("mercy")
|
||||||
|
app.config['SQLALCHEMY_DATABASE_URI'] = mercy.config.SQLALCHEMY_URI
|
||||||
|
return app
|
||||||
|
|||||||
@@ -1,8 +1,107 @@
|
|||||||
import mercy.db
|
import mercy.MercyApplication
|
||||||
|
import xml.etree.cElementTree as ET
|
||||||
|
from mercy.models.fda import Product
|
||||||
|
from mercy.models.fda import ProductSubstance
|
||||||
|
from mercy.models.fda import ProductSubstanceMap
|
||||||
|
from mercy.models.fda import PharmaceuticalClass
|
||||||
|
from mercy.models.fda import PharmaceuticalClassMap
|
||||||
|
import sqlalchemy.exc
|
||||||
|
import csv
|
||||||
|
|
||||||
class FDAImporter:
|
class FDAImporter:
|
||||||
def __init__(self, *args, **kwargs):
|
def __init__(self, *args, **kwargs):
|
||||||
self.__database = mercy.db.Database()
|
self.__db = mercy.MercyApplication.get_db()
|
||||||
|
|
||||||
def read(self, fname):
|
def read(self, fname, startIdx=0):
|
||||||
raise Exception("FDAImporter.read doesn't do anything yet")
|
with open(fname, "r") as ifile:
|
||||||
|
reader = csv.DictReader(ifile, delimiter="\t")
|
||||||
|
idx = 0
|
||||||
|
for row in reader:
|
||||||
|
if idx < startIdx:
|
||||||
|
print "Skipping from {} to {}".format(idx, startIdx)
|
||||||
|
idx += 1
|
||||||
|
continue
|
||||||
|
retries = 0
|
||||||
|
print "{} : {}".format(idx, row)
|
||||||
|
while retries < 3 :
|
||||||
|
try:
|
||||||
|
self._convert_row(row)
|
||||||
|
retries = 3
|
||||||
|
except sqlalchemy.exc.DatabaseError, e:
|
||||||
|
retries += 1
|
||||||
|
if retries == 3:
|
||||||
|
raise e
|
||||||
|
else:
|
||||||
|
continue
|
||||||
|
idx += 1
|
||||||
|
|
||||||
|
|
||||||
|
def _saveobj(self, obj):
|
||||||
|
self.__db.session.add(obj)
|
||||||
|
self.__db.session.commit()
|
||||||
|
|
||||||
|
def _convert_row(self, row):
|
||||||
|
# The FDA CSV is HIGHLY irregular. This function is needlessly large because of that/
|
||||||
|
product = Product.query.filter_by(productid=row['PRODUCTID']).first()
|
||||||
|
if not product:
|
||||||
|
product = Product()
|
||||||
|
product.productid = row['PRODUCTID']
|
||||||
|
product.ndc = row['PRODUCTNDC']
|
||||||
|
product.type = row['PRODUCTTYPENAME']
|
||||||
|
product.proprietaryName = row['PROPRIETARYNAME']
|
||||||
|
product.proprietaryNameSuffix = row['PROPRIETARYNAMESUFFIX']
|
||||||
|
product.genericName = row['NONPROPRIETARYNAME']
|
||||||
|
product.marketingCategoryName = row['MARKETINGCATEGORYNAME']
|
||||||
|
product.labelerName = row['LABELERNAME']
|
||||||
|
product.deaSchedule = (row['DEASCHEDULE'] if row['DEASCHEDULE'] else '')
|
||||||
|
self._saveobj(product)
|
||||||
|
|
||||||
|
# Strip the substances off of the product and make objects for them
|
||||||
|
substanceNames = [x.strip().lstrip() for x in row['SUBSTANCENAME'].split(';')]
|
||||||
|
tmpQtys = [x.strip().lstrip() for x in row['ACTIVE_NUMERATOR_STRENGTH'].split(';')]
|
||||||
|
# The list addition here is to make sure we have an equal number of values in
|
||||||
|
# quantities as we do in substance names, because the FDA CSV does not ensure
|
||||||
|
# that all substance names have a quantity or unit listed. So unknown quantities
|
||||||
|
# get entered as '0', unknown units of measure become '?'.
|
||||||
|
substanceQtys = [(float(x) if x else 0.0) for x in tmpQtys] + ([0] * (len(substanceNames) - len(tmpQtys)))
|
||||||
|
substanceUnits = [x.strip().lstrip() for x in row['ACTIVE_INGRED_UNIT'].split(';')]
|
||||||
|
substanceUnits += (['?'] * (len(substanceNames) - len(substanceUnits)))
|
||||||
|
|
||||||
|
for idx in range(0, len(substanceNames)):
|
||||||
|
substance = None
|
||||||
|
substanceName = substanceNames[idx]
|
||||||
|
|
||||||
|
substance = ProductSubstance.query.filter_by(name=substanceName).first()
|
||||||
|
if not substance:
|
||||||
|
substance = ProductSubstance()
|
||||||
|
substance.name = substanceName
|
||||||
|
self._saveobj(substance)
|
||||||
|
substanceMap = ProductSubstanceMap.query.filter_by(product_id=product.id,
|
||||||
|
substance_id=substance.id,
|
||||||
|
quantity=substanceQtys[idx],
|
||||||
|
units=substanceUnits[idx]).first()
|
||||||
|
if not substanceMap:
|
||||||
|
substanceMap = ProductSubstanceMap()
|
||||||
|
substanceMap.product_id = product.id
|
||||||
|
substanceMap.substance_id = substance.id
|
||||||
|
substanceMap.quantity = substanceQtys[idx]
|
||||||
|
substanceMap.units = substanceUnits[idx]
|
||||||
|
self._saveobj(substanceMap)
|
||||||
|
|
||||||
|
pharmaClassList = row.get('PHARM_CLASSES')
|
||||||
|
pharmaClasses = [x.strip().lstrip() for x in (pharmaClassList if pharmaClassList else '').split(',')]
|
||||||
|
for pharmaClass in pharmaClasses:
|
||||||
|
pharmaObj = PharmaceuticalClass.query.filter_by(name=pharmaClass).first()
|
||||||
|
if not pharmaObj:
|
||||||
|
pharmaObj = PharmaceuticalClass()
|
||||||
|
pharmaObj.name = pharmaClass
|
||||||
|
self._saveobj(pharmaObj)
|
||||||
|
mapObj = PharmaceuticalClassMap.query.filter_by(
|
||||||
|
product_id=product.id,
|
||||||
|
pharma_id=pharmaObj.id).first()
|
||||||
|
if not mapObj:
|
||||||
|
mapObj = PharmaceuticalClassMap()
|
||||||
|
mapObj.product_id = product.id
|
||||||
|
mapObj.pharma_id = pharmaObj.id
|
||||||
|
self._saveobj(mapObj)
|
||||||
|
return
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
from mercy.models.simplemodel import SimpleModel
|
from mercy.models.simplemodel import SimpleModel
|
||||||
import mercy.MercyApplication
|
import mercy.MercyApplication
|
||||||
|
from mercy.models.fda import Product
|
||||||
import sqlalchemy.dialects.postgresql as pgdialect
|
import sqlalchemy.dialects.postgresql as pgdialect
|
||||||
|
|
||||||
db = mercy.MercyApplication.get_db()
|
db = mercy.MercyApplication.get_db()
|
||||||
@@ -8,10 +9,11 @@ db = mercy.MercyApplication.get_db()
|
|||||||
class Drug(SimpleModel, db.Model):
|
class Drug(SimpleModel, db.Model):
|
||||||
__tablename__ = "drugbank_drugs"
|
__tablename__ = "drugbank_drugs"
|
||||||
|
|
||||||
id = sa.Column(sa.String, primary_key=True, unique=True)
|
id = sa.Column(sa.Integer, primary_key=True, autoincrement=True, nullable=False)
|
||||||
|
dbid = sa.Column(sa.String, index=True, unique=True)
|
||||||
name = sa.Column(sa.String, nullable=False, index=True)
|
name = sa.Column(sa.String, nullable=False, index=True)
|
||||||
indication = sa.Column(sa.String, nullable=False)
|
indication = sa.Column(sa.String, nullable=False)
|
||||||
ndc_id = sa.Column(sa.String, sa.ForeignKey('fda_products.id'), nullable=True)
|
fda_product_id = sa.Column(sa.String, sa.ForeignKey(Product.productid), nullable=True)
|
||||||
wikipedia = sa.Column(sa.String, nullable=True)
|
wikipedia = sa.Column(sa.String, nullable=True)
|
||||||
|
|
||||||
__repr_keys__ = { 'id': basestring,
|
__repr_keys__ = { 'id': basestring,
|
||||||
@@ -21,7 +23,8 @@ class Drug(SimpleModel, db.Model):
|
|||||||
|
|
||||||
class Price(SimpleModel, db.Model):
|
class Price(SimpleModel, db.Model):
|
||||||
__tablename__ = "drugbank_prices"
|
__tablename__ = "drugbank_prices"
|
||||||
drug_id = sa.Column(sa.String, sa.ForeignKey(Drug.id), primary_key=True, nullable=False)
|
id = sa.Column(sa.Integer, primary_key=True, autoincrement=True, nullable=False)
|
||||||
|
drug_id = sa.Column(sa.Integer, sa.ForeignKey(Drug.id), nullable=False)
|
||||||
description = sa.Column(sa.String, nullable=False)
|
description = sa.Column(sa.String, nullable=False)
|
||||||
currency = sa.Column(sa.String, nullable=False)
|
currency = sa.Column(sa.String, nullable=False)
|
||||||
cost = sa.Column(sa.Float, nullable=False, index=True)
|
cost = sa.Column(sa.Float, nullable=False, index=True)
|
||||||
@@ -30,40 +33,45 @@ class Price(SimpleModel, db.Model):
|
|||||||
class CategoryName(SimpleModel, db.Model):
|
class CategoryName(SimpleModel, db.Model):
|
||||||
__tablename__ = "drugbank_categories"
|
__tablename__ = "drugbank_categories"
|
||||||
id = sa.Column(sa.Integer, primary_key=True, autoincrement=True, nullable=False)
|
id = sa.Column(sa.Integer, primary_key=True, autoincrement=True, nullable=False)
|
||||||
name = sa.Column(sa.String, nullable=False)
|
name = sa.Column(sa.String, nullable=False, unique=True)
|
||||||
|
|
||||||
class CategoryMap(SimpleModel, db.Model):
|
class CategoryMap(SimpleModel, db.Model):
|
||||||
__tablename__ = "drugbank_category_maps"
|
__tablename__ = "drugbank_category_maps"
|
||||||
drug_id = sa.Column(sa.String, sa.ForeignKey(Drug.id), primary_key=True, nullable=False)
|
id = sa.Column(sa.Integer, primary_key=True, autoincrement=True, nullable=False)
|
||||||
|
drug_id = sa.Column(sa.Integer, sa.ForeignKey(Drug.id), nullable=False)
|
||||||
category_id = sa.Column(sa.Integer, sa.ForeignKey(CategoryName.id), nullable=False)
|
category_id = sa.Column(sa.Integer, sa.ForeignKey(CategoryName.id), nullable=False)
|
||||||
|
|
||||||
class Packager(SimpleModel, db.Model):
|
class Packager(SimpleModel, db.Model):
|
||||||
__tablename__ = "drugbank_packagers"
|
__tablename__ = "drugbank_packagers"
|
||||||
id = sa.Column(sa.Integer, primary_key=True, autoincrement=True, nullable=False)
|
id = sa.Column(sa.Integer, primary_key=True, autoincrement=True, nullable=False)
|
||||||
name = sa.Column(sa.String, nullable=False)
|
name = sa.Column(sa.String, nullable=False, unique=True)
|
||||||
url = sa.Column(sa.String, nullable=True)
|
url = sa.Column(sa.String, nullable=True)
|
||||||
|
|
||||||
class PackagerMap(SimpleModel, db.Model):
|
class PackagerMap(SimpleModel, db.Model):
|
||||||
__tablename__ = "drugbank_packager_maps"
|
__tablename__ = "drugbank_packager_maps"
|
||||||
drug_id = sa.Column(sa.String, sa.ForeignKey(Drug.id), primary_key=True, nullable=False)
|
id = sa.Column(sa.Integer, primary_key=True, autoincrement=True, nullable=False)
|
||||||
|
drug_id = sa.Column(sa.String, sa.ForeignKey(Drug.id), nullable=False)
|
||||||
packager_id = sa.Column(sa.Integer, sa.ForeignKey(Packager.id), nullable=False)
|
packager_id = sa.Column(sa.Integer, sa.ForeignKey(Packager.id), nullable=False)
|
||||||
|
|
||||||
class Manufacturer(SimpleModel, db.Model):
|
class Manufacturer(SimpleModel, db.Model):
|
||||||
__tablename__ = "drugbank_manufacturers"
|
__tablename__ = "drugbank_manufacturers"
|
||||||
id = sa.Column(sa.Integer, primary_key=True, autoincrement=True, nullable=False)
|
id = sa.Column(sa.Integer, primary_key=True, autoincrement=True, nullable=False)
|
||||||
name = sa.Column(sa.String, nullable=False)
|
name = sa.Column(sa.String, nullable=False, unique=True)
|
||||||
|
|
||||||
class ManufacturerMap(SimpleModel, db.Model):
|
class ManufacturerMap(SimpleModel, db.Model):
|
||||||
__tablename__ = "drugbank_manufacturer_maps"
|
__tablename__ = "drugbank_manufacturer_maps"
|
||||||
drug_id = sa.Column(sa.String, sa.ForeignKey(Drug.id), primary_key=True, nullable=False)
|
id = sa.Column(sa.Integer, primary_key=True, autoincrement=True, nullable=False)
|
||||||
|
drug_id = sa.Column(sa.Integer, sa.ForeignKey(Drug.id), nullable=False)
|
||||||
manufacturer_id = sa.Column(sa.Integer, sa.ForeignKey(Manufacturer.id), nullable=False)
|
manufacturer_id = sa.Column(sa.Integer, sa.ForeignKey(Manufacturer.id), nullable=False)
|
||||||
|
|
||||||
class GenericName(SimpleModel, db.Model):
|
class GenericName(SimpleModel, db.Model):
|
||||||
__tablename__ = "drugbank_genericnames"
|
__tablename__ = "drugbank_genericnames"
|
||||||
drug_id = sa.Column(sa.String, sa.ForeignKey(Drug.id), primary_key=True, nullable=False)
|
id = sa.Column(sa.Integer, primary_key=True, autoincrement=True, nullable=False)
|
||||||
|
drug_id = sa.Column(sa.Integer, sa.ForeignKey(Drug.id), nullable=False)
|
||||||
name = sa.Column(sa.String, nullable=False)
|
name = sa.Column(sa.String, nullable=False)
|
||||||
|
|
||||||
class Synonym(SimpleModel, db.Model):
|
class Synonym(SimpleModel, db.Model):
|
||||||
__tablename__ = "drugbank_synonyms"
|
__tablename__ = "drugbank_synonyms"
|
||||||
drug_id = sa.Column(sa.String, sa.ForeignKey(Drug.id), primary_key=True, nullable=False)
|
id = sa.Column(sa.Integer, primary_key=True, autoincrement=True, nullable=False)
|
||||||
|
drug_id = sa.Column(sa.Integer, sa.ForeignKey(Drug.id), nullable=False)
|
||||||
name = sa.Column(sa.String, nullable=False)
|
name = sa.Column(sa.String, nullable=False)
|
||||||
|
|||||||
@@ -8,8 +8,9 @@ db = mercy.MercyApplication.get_db()
|
|||||||
class Product(SimpleModel, db.Model):
|
class Product(SimpleModel, db.Model):
|
||||||
__tablename__ = 'fda_products'
|
__tablename__ = 'fda_products'
|
||||||
|
|
||||||
id = sa.Column(sa.String, primary_key=True)
|
id = sa.Column(sa.Integer, primary_key=True, autoincrement=True, nullable=False)
|
||||||
ndc = sa.Column(sa.String, nullable=False)
|
productid = sa.Column(sa.String, index=True, unique=True, nullable=False)
|
||||||
|
ndc = sa.Column(sa.String, index=True, nullable=False)
|
||||||
type = sa.Column(sa.String, nullable=False)
|
type = sa.Column(sa.String, nullable=False)
|
||||||
proprietaryName = sa.Column(sa.String, nullable=False, index=True)
|
proprietaryName = sa.Column(sa.String, nullable=False, index=True)
|
||||||
proprietaryNameSuffix = sa.Column(sa.String)
|
proprietaryNameSuffix = sa.Column(sa.String)
|
||||||
@@ -27,11 +28,36 @@ class Product(SimpleModel, db.Model):
|
|||||||
class ProductSubstance(SimpleModel, db.Model):
|
class ProductSubstance(SimpleModel, db.Model):
|
||||||
__tablename__ = 'fda_product_substances'
|
__tablename__ = 'fda_product_substances'
|
||||||
|
|
||||||
fda_product_id = sa.Column(sa.String,
|
id = sa.Column(sa.Integer, primary_key=True, autoincrement=True, nullable=False)
|
||||||
|
name = sa.Column(sa.String, nullable=False)
|
||||||
|
|
||||||
|
class ProductSubstanceMap(SimpleModel, db.Model):
|
||||||
|
__tablename__ = 'fda_product_substance_map'
|
||||||
|
|
||||||
|
id = sa.Column(sa.Integer, primary_key=True, autoincrement=True, nullable=False)
|
||||||
|
product_id = sa.Column(sa.Integer,
|
||||||
sa.ForeignKey(Product.id),
|
sa.ForeignKey(Product.id),
|
||||||
|
nullable=False)
|
||||||
|
substance_id = sa.Column(sa.Integer,
|
||||||
|
sa.ForeignKey(ProductSubstance.id),
|
||||||
|
nullable=False,
|
||||||
|
index=True)
|
||||||
|
quantity = sa.Column(sa.Float, nullable=False)
|
||||||
|
units = sa.Column(sa.String, nullable=False)
|
||||||
|
|
||||||
|
|
||||||
|
class PharmaceuticalClass(SimpleModel, db.Model):
|
||||||
|
__tablename__ = "fda_pharma_classes"
|
||||||
|
id = sa.Column(sa.Integer, primary_key=True, autoincrement=True, nullable=False)
|
||||||
|
name = sa.Column(sa.String, nullable=False, unique=True)
|
||||||
|
|
||||||
|
class PharmaceuticalClassMap(SimpleModel, db.Model):
|
||||||
|
__tablename__ = "fda_pharma_class_maps"
|
||||||
|
id = sa.Column(sa.Integer, primary_key=True, autoincrement=True, nullable=False)
|
||||||
|
product_id = sa.Column(sa.Integer,
|
||||||
|
sa.ForeignKey(Product.id),
|
||||||
|
nullable=False)
|
||||||
|
pharma_id = sa.Column(sa.Integer,
|
||||||
|
sa.ForeignKey(PharmaceuticalClass.id),
|
||||||
primary_key=True,
|
primary_key=True,
|
||||||
nullable=False)
|
nullable=False)
|
||||||
substanceName = sa.Column(sa.String, nullable=False)
|
|
||||||
strengthNumber = sa.Column(sa.Float, nullable=False)
|
|
||||||
strengthUnit = sa.Column(sa.String, nullable=False)
|
|
||||||
pharmaClasses = sa.Column(pgdialect.ARRAY(sa.String), nullable=False)
|
|
||||||
|
|||||||
@@ -1 +1 @@
|
|||||||
VERSION="0.0-0"
|
VERSION="0.0-1"
|
||||||
|
|||||||
@@ -1,14 +1,6 @@
|
|||||||
from mercy.MercyApplication import MercyApplication
|
import mercy.MercyApplication
|
||||||
|
|
||||||
class ScriptNameStripper(object):
|
app = get_application()
|
||||||
def __init__(self, app):
|
|
||||||
self.app = app
|
|
||||||
|
|
||||||
def __call_(self, environ, start_response):
|
|
||||||
environ['SCRIPT_NAME'] = ''
|
|
||||||
return self.app(environ, start_response)
|
|
||||||
|
|
||||||
app = ScriptNameStripper(MercyApplication())
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
app.run()
|
app.run()
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
#!/usr/bin/env python
|
#!/usr/bin/env python
|
||||||
|
|
||||||
from mercy.MercyApplication import MercyApplication
|
from mercy.MercyApplication import app
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
app = MercyApplication("mercy")
|
|
||||||
app.run()
|
app.run()
|
||||||
|
|||||||
11
setup.py
11
setup.py
@@ -8,7 +8,7 @@ if __name__ == "__main__":
|
|||||||
name="mercy",
|
name="mercy",
|
||||||
url="https://www.github.com/akesterson/mercy",
|
url="https://www.github.com/akesterson/mercy",
|
||||||
version=mercy.version.VERSION,
|
version=mercy.version.VERSION,
|
||||||
description="A flask application that facilitates paying for prescriptions",
|
description="A web application that facilitates charitable prescription payments",
|
||||||
long_description="",
|
long_description="",
|
||||||
author=("Andrew Kesterson"),
|
author=("Andrew Kesterson"),
|
||||||
author_email="andrew@aklabs.net",
|
author_email="andrew@aklabs.net",
|
||||||
@@ -16,9 +16,12 @@ if __name__ == "__main__":
|
|||||||
install_requires=["flask",
|
install_requires=["flask",
|
||||||
"sqlalchemy",
|
"sqlalchemy",
|
||||||
"alembic",
|
"alembic",
|
||||||
"psycopg2"],
|
"psycopg2",
|
||||||
scripts=[],
|
"flask-sqlalchemy"],
|
||||||
packages=["mercy"],
|
scripts=["scripts/mercy.wsgi"],
|
||||||
|
packages=["mercy",
|
||||||
|
"mercy/models",
|
||||||
|
"mercy/importers"],
|
||||||
data_files=[],
|
data_files=[],
|
||||||
classifiers=[
|
classifiers=[
|
||||||
'Development Status :: 1 - Planning',
|
'Development Status :: 1 - Planning',
|
||||||
|
|||||||
@@ -5,52 +5,116 @@ import mercy.models
|
|||||||
import mercy.importers.fda
|
import mercy.importers.fda
|
||||||
import mercy.exceptions
|
import mercy.exceptions
|
||||||
|
|
||||||
VALID_ROWS=[]
|
COMPARISON_KEYS={
|
||||||
|
'productid': 'PRODUCTID',
|
||||||
|
'ndc': 'PRODUCTNDC',
|
||||||
|
'type': 'PRODUCTTYPENAME',
|
||||||
|
'proprietaryName': 'PROPRIETARYNAME',
|
||||||
|
'proprietaryNameSuffix': 'PROPRIETARYNAMESUFFIX',
|
||||||
|
'genericName': 'NONPROPRIETARYNAME',
|
||||||
|
'marketingCategoryName': 'MARKETINGCATEGORYNAME',
|
||||||
|
'labelerName': 'LABELERNAME',
|
||||||
|
'deaSchedule': 'DEASCHEDULE'
|
||||||
|
}
|
||||||
|
|
||||||
|
CSV_KEYS=[
|
||||||
|
'PRODUCTID',
|
||||||
|
'PRODUCTNDC',
|
||||||
|
'PRODUCTTYPENAME',
|
||||||
|
'PROPRIETARYNAME',
|
||||||
|
'PROPRIETARYNAMESUFFIX',
|
||||||
|
'NONPROPRIETARYNAME',
|
||||||
|
'DOSAGEFORMNAME',
|
||||||
|
'ROUTENAME',
|
||||||
|
'STARTMARKETINGDATE',
|
||||||
|
'ENDMARKETINGDATE',
|
||||||
|
'MARKETINGCATEGORYNAME',
|
||||||
|
'APPLICATIONNUMBER',
|
||||||
|
'LABELERNAME',
|
||||||
|
'SUBSTANCENAME',
|
||||||
|
'ACTIVE_NUMERATOR_STRENGTH',
|
||||||
|
'ACTIVE_INGRED_UNIT',
|
||||||
|
'PHARM_CLASSES',
|
||||||
|
'DEASCHEDULE'
|
||||||
|
]
|
||||||
|
|
||||||
|
CANNED_ROWS=[
|
||||||
|
{'PRODUCTID': '0002-3230_b1642902-4a44-495a-8790-39598b168276',
|
||||||
|
'PRODUCTNDC': '0002-3230',
|
||||||
|
'PRODUCTTYPENAME': 'HUMAN PRESCRIPTION DRUG',
|
||||||
|
'PROPRIETARYNAME': 'Symbyax',
|
||||||
|
'PROPRIETARYNAMESUFFIX': '',
|
||||||
|
'NONPROPRIETARYNAME': 'Olanzapine and Fluoxetine hydrochloride',
|
||||||
|
'DOSAGEFORMNAME': 'CAPSULE',
|
||||||
|
'ROUTENAME': 'ORAL',
|
||||||
|
'STARTMARKETINGDATE': '20070409',
|
||||||
|
'ENDMARKETINGDATE': '',
|
||||||
|
'MARKETINGCATEGORYNAME': 'NDA',
|
||||||
|
'APPLICATIONNUMBER': 'NDA021520',
|
||||||
|
'LABELERNAME': 'Eli Lilly and Company',
|
||||||
|
'SUBSTANCENAME': 'FLUOXETINE HYDROCHLORIDE; OLANZAPINE',
|
||||||
|
'ACTIVE_NUMERATOR_STRENGTH': '25; 3',
|
||||||
|
'ACTIVE_INGRED_UNIT': 'mg/1; mg/1',
|
||||||
|
'PHARM_CLASSES': 'Atypical Antipsychotic [EPC],Serotonin Reuptake Inhibitor [EPC],Serotonin Uptake Inhibitors [MoA]',
|
||||||
|
'DEASCHEDULE': ''
|
||||||
|
},
|
||||||
|
{'PRODUCTID': '0002-3231_b1642902-4a44-495a-8790-39598b168276',
|
||||||
|
'PRODUCTNDC': '0002-3231',
|
||||||
|
'PRODUCTTYPENAME': 'HUMAN PRESCRIPTION DRUG',
|
||||||
|
'PROPRIETARYNAME': 'Symbyax',
|
||||||
|
'PROPRIETARYNAMESUFFIX': '',
|
||||||
|
'NONPROPRIETARYNAME': 'Olanzapine and Fluoxetine hydrochloride',
|
||||||
|
'DOSAGEFORMNAME': 'CAPSULE',
|
||||||
|
'ROUTENAME': 'ORAL',
|
||||||
|
'STARTMARKETINGDATE': '20070409',
|
||||||
|
'ENDMARKETINGDATE': '',
|
||||||
|
'MARKETINGCATEGORYNAME': 'NDA',
|
||||||
|
'APPLICATIONNUMBER': 'NDA021521',
|
||||||
|
'LABELERNAME': 'Eli Lilly and Company',
|
||||||
|
'SUBSTANCENAME': 'FLUOXETINE HYDROCHLORIDE; OLANZAPINE',
|
||||||
|
'ACTIVE_NUMERATOR_STRENGTH': '25; 6',
|
||||||
|
'ACTIVE_INGRED_UNIT': 'mg/1; mg/1',
|
||||||
|
'PHARM_CLASSES': 'Atypical Antipsychotic [EPC],Serotonin Reuptake Inhibitor [EPC],Serotonin Uptake Inhibitors [MoA]',
|
||||||
|
'DEASCHEDULE': ''
|
||||||
|
}
|
||||||
|
]
|
||||||
|
|
||||||
FIXTUREFILE=os.path.abspath(
|
FIXTUREFILE=os.path.abspath(
|
||||||
os.path.join(
|
os.path.join(
|
||||||
__file__,
|
os.path.dirname(__file__),
|
||||||
"..",
|
"..",
|
||||||
"fixtures",
|
"fixtures",
|
||||||
"fda_database.tar.gz"
|
"fda_database.txt"
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
FIXTUREFILE_BAD=os.path.abspath(
|
FIXTUREFILE_BAD=os.path.abspath(
|
||||||
os.path.join(
|
os.path.join(
|
||||||
__file__,
|
os.path.dirname(__file__),
|
||||||
"..",
|
"..",
|
||||||
"fixtures",
|
"fixtures",
|
||||||
"fda_database_bad.tar.gz"
|
"fda_database_bad.txt"
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
FIXTUREFILE_CORRUPT=os.path.abspath(
|
|
||||||
os.path.join(
|
|
||||||
__file__,
|
|
||||||
"..",
|
|
||||||
"fixtures",
|
|
||||||
"fda_database_corrupt.tar.gz"
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
@raises(mercy.exceptions.CorruptTarError)
|
|
||||||
def test_fda_import_fails_on_corrupt_tar():
|
|
||||||
importer = mercy.importers.fda.FDAImporter()
|
|
||||||
impoter.read(FIXTUREFILE_CORRUPT)
|
|
||||||
|
|
||||||
def test_fda_import_populates_table():
|
def test_fda_import_populates_table():
|
||||||
importer = FDAImporter().read(FIXTUREFILE)
|
with open(FIXTUREFILE, 'w') as ofile:
|
||||||
rows = mercy.models.fda.Product.query.all()
|
ofile.write("{}\n".format('\t'.join(CSV_KEYS)))
|
||||||
for i in range(0, len(rows)):
|
for row in CANNED_ROWS:
|
||||||
row = rows[i]
|
values = []
|
||||||
canned_row = CANNED_ROWS[i]
|
for key in CSV_KEYS:
|
||||||
assert(len(row) == len(canned_row))
|
values.append(row[key])
|
||||||
for j in canned_row.keys():
|
ofile.write("{}\n".format('\t'.join(values)))
|
||||||
assert(row[j] == canned_row[j])
|
|
||||||
|
|
||||||
@raises(AttributeError, KeyError, ValueError)
|
|
||||||
def test_fda_import_rejects_bad_records:
|
|
||||||
importer = mercy.importers.fda.FDAImporter()
|
importer = mercy.importers.fda.FDAImporter()
|
||||||
importer.read(FIXTUREFILE_BAD)
|
importer.read(FIXTUREFILE)
|
||||||
|
for row in CANNED_ROWS:
|
||||||
|
product = mercy.models.fda.Product.query.filter_by(productid = row['PRODUCTID']).first()
|
||||||
|
assert(product)
|
||||||
|
for (k, v) in COMPARISON_KEYS.iteritems():
|
||||||
|
assert(getattr(product, k) == row[v])
|
||||||
|
mapquery = mercy.models.fda.ProductSubstanceMap.query
|
||||||
|
substanceMaps = [x for x in mapquery.filter_by(product_id=product.id)]
|
||||||
|
assert(len(substanceMaps) == len(row['ACTIVE_NUMERATOR_STRENGTH'].split(';')))
|
||||||
|
# TO DO : This test doesn't look at the contents of the
|
||||||
|
# substances or substance maps, only that the right
|
||||||
|
# number of substance maps come out
|
||||||
|
|||||||
99
version.sh
99
version.sh
@@ -1,15 +1,98 @@
|
|||||||
TAG="build,0.0,0"
|
TAG="build,0.0,0"
|
||||||
BRANCH="master"
|
BRANCH="master"
|
||||||
MAJOR="0.0"
|
MAJOR="0.0"
|
||||||
BUILD="0"
|
BUILD="1"
|
||||||
SHA1="9b83e26ec3d0e3d1526a4226a8ac0c8580a77bcf"
|
SHA1="e2bc6022649f3af82800d75ba876411f1ed83fa9"
|
||||||
OS_NAME="${OS_NAME:-win}"
|
OS_NAME="${OS_NAME:-win}"
|
||||||
OS_VERSION="${OS_VERSION:-}"
|
OS_VERSION="${OS_VERSION:-}"
|
||||||
ARCH="${ARCH:-i686}"
|
ARCH="${ARCH:-x86_64}"
|
||||||
VERSION="0.0-0"
|
VERSION="0.0-1"
|
||||||
BUILDHOST="akesterson-pc"
|
BUILDHOST="akesterson-pc"
|
||||||
BUILDUSER="akesterson-pc\akesterson"
|
BUILDUSER="akesterson"
|
||||||
BUILDDIR="/c/Users/akesterson/source/upstream/git/akesterson/mercy"
|
BUILDDIR="/cygdrive/c/Users/akesterson/source/upstream/git/akesterson/mercy"
|
||||||
SOURCE="git@github.com:akesterson/mercy.git"
|
SOURCE="git@github.com:akesterson/mercy.git"
|
||||||
REBUILDING=0
|
REBUILDING=1
|
||||||
CHANGELOG=""
|
CHANGELOG="2013-10-19 23:35:39 -0400 Andrew Kesterson <andrew@aklabs.net>
|
||||||
|
|
||||||
|
#6 : Make fda_products.genericName an index
|
||||||
|
[e2bc602] (HEAD, master)
|
||||||
|
|
||||||
|
2013-10-19 23:24:38 -0400 Andrew Kesterson <andrew@aklabs.net>
|
||||||
|
|
||||||
|
Merge branch 'master' of github.com:akesterson/mercy
|
||||||
|
[ecef739] (origin/master, origin/HEAD)
|
||||||
|
|
||||||
|
2013-10-19 23:23:53 -0400 Andrew Kesterson <andrew@aklabs.net>
|
||||||
|
|
||||||
|
Added downgrade to initial DB script
|
||||||
|
[803d120]
|
||||||
|
|
||||||
|
2013-10-19 23:08:23 -0400 Andrew Kesterson <andrew@aklabs.net>
|
||||||
|
|
||||||
|
Removed more junk
|
||||||
|
[1753230]
|
||||||
|
|
||||||
|
2013-10-19 23:06:23 -0400 Andrew Kesterson <andrew@aklabs.net>
|
||||||
|
|
||||||
|
Merge branch 'master' of www.aklabs.net:~/mercy/
|
||||||
|
[6b49f61]
|
||||||
|
|
||||||
|
2013-10-19 23:05:52 -0400 Andrew Kesterson <andrew@aklabs.net>
|
||||||
|
|
||||||
|
Removed abunch of junk
|
||||||
|
[8caeb4d]
|
||||||
|
|
||||||
|
2013-10-19 22:58:57 -0400 Andrew Kesterson <andrew@aklabs.net>
|
||||||
|
|
||||||
|
Working on #6. I think I should break out the puppet module stuff into a separate issue, since that risks really putting this all off target.
|
||||||
|
[dbf64e8]
|
||||||
|
|
||||||
|
2013-10-19 22:45:53 -0400 Andrew Kesterson <andrew@aklabs.net>
|
||||||
|
|
||||||
|
Midstream
|
||||||
|
[64fec8d]
|
||||||
|
|
||||||
|
2013-10-19 22:45:35 -0400 Andrew Kesterson <andrew@aklabs.net>
|
||||||
|
|
||||||
|
Midstream
|
||||||
|
[646df51]
|
||||||
|
|
||||||
|
2013-10-19 22:44:59 -0400 Andrew Kesterson <andrew@aklabs.net>
|
||||||
|
|
||||||
|
Midstream
|
||||||
|
[dcec82b]
|
||||||
|
|
||||||
|
2013-10-19 22:43:59 -0400 Andrew Kesterson <andrew@aklabs.net>
|
||||||
|
|
||||||
|
Midstream
|
||||||
|
[4332d08]
|
||||||
|
|
||||||
|
2013-10-19 22:30:49 -0400 Andrew Kesterson <andrew@aklabs.net>
|
||||||
|
|
||||||
|
Midstream
|
||||||
|
[c4f610a]
|
||||||
|
|
||||||
|
2013-10-19 22:28:46 -0400 Andrew Kesterson <andrew@aklabs.net>
|
||||||
|
|
||||||
|
Midstream
|
||||||
|
[56d46c5]
|
||||||
|
|
||||||
|
2013-10-19 17:59:56 -0400 Andrew Kesterson <andrew@aklabs.net>
|
||||||
|
|
||||||
|
Updated some design docs
|
||||||
|
[86e8db6]
|
||||||
|
|
||||||
|
2013-10-19 10:38:45 -0400 Andrew Kesterson <andrew@aklabs.net>
|
||||||
|
|
||||||
|
Added nagios directory
|
||||||
|
[45cfa0b]
|
||||||
|
|
||||||
|
2013-10-19 10:37:30 -0400 Andrew Kesterson <andrew@aklabs.net>
|
||||||
|
|
||||||
|
Added puppet module skeleton (also puppet module tool is lame)
|
||||||
|
[363d5ef]
|
||||||
|
|
||||||
|
2013-10-19 09:53:37 -0400 Andrew Kesterson <andrew@aklabs.net>
|
||||||
|
|
||||||
|
Initial commit
|
||||||
|
[9895e8e]"
|
||||||
|
|||||||
Reference in New Issue
Block a user