andrew/mercy
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Working on #1 - FDA import process works. 3 hours for first import, 1.25 hours on subsequent updates.

Browse Source
This commit is contained in:
Andrew Kesterson
2013-10-27 18:35:35 -04:00
parent e0232aa02d
commit 0a7bf8f174
21 changed files with 759 additions and 472 deletions
Download Patch File Download Diff File Expand all files Collapse all files

173
Makefile
View File

@@ -1,90 +1,85 @@
MAJOR:=$(shell bash -c 'source version.sh ; echo $$MAJOR')
BUILD:=$(shell bash -c 'source version.sh ; echo $$BUILD')
OS_NAME:=$(shell bash -c 'source version.sh ; echo $$OS_NAME')
ifeq "$(OS_NAME)" "win"
PIP=$(shell pwd)/virtualenv/Scripts/pip
VIRTUALENV_PKGS_DIR=$(shell pwd)/virtualenv/Lib/site-packages
else
PIP=$(shell pwd)/virtualenv/bin/pip
VIRTUALENV_PKGS_DIR=$(shell pwd)/virtualenv/lib/site-packages
endif
VIRTUALENV=$(shell which virtualenv)
PYTHON=$(shell which python)
PYTHON_FILES=setup.py mercy/version.py $(shell find mercy -iname "*py")
PYTHON_SDIST=./dist/mercy-$(MAJOR)-$(BUILD).tar.gz
.PHONY: clean
clean:
rm -fr dist/*
find mercy -iname "*pyc" -exec rm -vf \{\} \;
cd puppet && make clean
############## Targets for puppet module
puppet/version.sh: version.sh
cp version.sh $@
.PHONY: puppet
puppet: puppet_dist
.PHONY: puppet_dist
puppet_dist: puppet/version.sh
cd puppet && make dist
.PHONY: puppet_install
puppet_install: puppet/version.sh
cd puppet && make install
.PHONY: puppet_uninstall
puppet_uninstall: puppet/version.sh
cd puppet && make uninstall
################ /puppet module
############### Targets for python app
dist: $(PYTHON_SDIST) puppet
sdist: $(PYTHON_SDIST)
mercy/version.py: version.sh
bash -c 'source version.sh && echo "VERSION=\"$${MAJOR}-$${BUILD}\"" > $@'
$(PYTHON_SDIST): $(PYTHON_FILES)
$(PYTHON) setup.py sdist --formats=gztar
uninstall:
rm -fr $(VIRTUALENV_PKGS_DIR)/mercy-* || echo 'not installed'
install: $(PYTHON_SDIST) virtualenv
$(PIP) install $(PYTHON_SDIST) --upgrade
.PHONY: virtualenv
virtualenv:
if [ ! -e $(PIP) ]; then \
$(VIRTUALENV) --no-site-packages --distribute virtualenv ; \
fi
################## /python app
################## Targets for supporting development work
databases: databases/fda_ndc.zip databases/drugbank.xml.zip
.PHONY: databases/fda_ndc.zip
databases/fda_ndc.zip:
rm -fr databases/fda_ndc*
mkdir -p databases/fda_ndc
rm -f fda_ndc.zip
DBPAGE=$$(wget -O - http://www.fda.gov/Drugs/InformationOnDrugs/default.htm --quiet | grep -Eo 'National Drug Code Directory Search(</strong>)*(</a>)*(<br />)*<a href="[a-zA-Z0-9\:/\.]+">More information about the database</a>' | cut -d \" -f 2); \
LINK=$$(wget -O - http://www.fda.gov/$${DBPAGE} --quiet | grep ">NDC Database File" | cut -d \" -f 2) ; \
wget -O $@ http://www.fda.gov/$${LINK}
cd databases/fda_ndc && unzip -e ../fda_ndc.zip
.PHONY: databases/drugbank.xml.zip
databases/drugbank.xml.zip:
rm -fr databases/drugbank*
mkdir -p databases/drugbank
wget -O $@ http://www.drugbank.ca/system/downloads/current/drugbank.xml.zip
MAJOR:=$(shell bash -c 'source version.sh ; echo $$MAJOR')
BUILD:=$(shell bash -c 'source version.sh ; echo $$BUILD')
OS_NAME:=$(shell bash -c 'source version.sh ; echo $$OS_NAME')
PIP=$(shell pwd)/virtualenv/bin/pip
VIRTUALENV_PKGS_DIR=$(shell pwd)/virtualenv/lib/python2.7/site-packages
VIRTUALENV=$(shell which virtualenv)
PYTHON=$(shell which python)
PYTHON_FILES=setup.py mercy/version.py $(shell find mercy -iname "*py")
PYTHON_SDIST=./dist/mercy-$(MAJOR)-$(BUILD).tar.gz
.PHONY: clean
clean:
rm -fr dist/*
find mercy -iname "*pyc" -exec rm -vf \{\} \;
cd puppet && make clean
############## Targets for puppet module
puppet/version.sh: version.sh
cp version.sh $@
.PHONY: puppet
puppet: puppet_dist
.PHONY: puppet_dist
puppet_dist: puppet/version.sh
cd puppet && make dist
.PHONY: puppet_install
puppet_install: puppet/version.sh
cd puppet && make install
.PHONY: puppet_uninstall
puppet_uninstall: puppet/version.sh
cd puppet && make uninstall
################ /puppet module
############### Targets for python app
dist: $(PYTHON_SDIST) puppet
sdist: $(PYTHON_SDIST)
mercy/version.py: version.sh
bash -c 'source version.sh && echo "VERSION=\"$${MAJOR}-$${BUILD}\"" > $@'
$(PYTHON_SDIST): $(PYTHON_FILES)
$(PYTHON) setup.py sdist --formats=gztar
uninstall:
rm -fr $(VIRTUALENV_PKGS_DIR)/mercy-* || echo 'not installed'
install: $(PYTHON_SDIST) virtualenv
$(PIP) install $(PYTHON_SDIST) --upgrade
.PHONY: virtualenv
virtualenv:
if [ ! -e $(PIP) ]; then \
$(VIRTUALENV) --no-site-packages --distribute virtualenv ; \
fi
################## /python app
################## Targets for supporting development work
databases: databases/fda_ndc.zip databases/drugbank.xml.zip
.PHONY: databases/fda_ndc.zip
databases/fda_ndc.zip:
rm -fr databases/fda_ndc*
mkdir -p databases/fda_ndc
rm -f fda_ndc.zip
DBPAGE=$$(wget -O - http://www.fda.gov/Drugs/InformationOnDrugs/default.htm --quiet | grep -Eo 'National Drug Code Directory Search(</strong>)*(</a>)*(<br />)*<a href="[a-zA-Z0-9\:/\.]+">More information about the database</a>' | cut -d \" -f 2); \
LINK=$$(wget -O - http://www.fda.gov/$${DBPAGE} --quiet | grep ">NDC Database File" | cut -d \" -f 2) ; \
wget -O $@ http://www.fda.gov/$${LINK}
cd databases/fda_ndc && unzip -e ../fda_ndc.zip
.PHONY: databases/drugbank.xml.zip
databases/drugbank.xml.zip:
rm -fr databases/drugbank*
mkdir -p databases/drugbank
wget -O $@ http://www.drugbank.ca/system/downloads/current/drugbank.xml.zip
cd databases/drugbank && unzip -e ../drugbank.xml.zip

56
README.md
View File

@@ -1,28 +1,28 @@
mercy
=====
Mercy is a web application designed to facilitate payments between people who need prescriptions but can't afford them, and charitable people who can afford them
The Idea
========
* Patient: "Here's my prescription, Mr Pharmacist"
* Pharmacist: "Sure thing. That'll be $150 USD."
* Patient: "I can't afford that! Can you submit it to Mercy for me?"
* Pharmacist: "Sure thing. We'll call you when it gets filled."
... Meanwhile, back at stately Charitable Person Manor ...
* Charitable Person's Phone: *DING*
* Charitable Person: "Huh? Oh! Mercy is telling me there's a new prescription I can pay for. I love helping people!"
* Charitable person clicks some buttons
* Charitable Person: "*CLICK* Paid! Enjoy some good health, stranger. Now to post it on my facebook wall and collect the new achievement for buying my first Leukemia drug..."
... Back at the pharmacy ...
* Pharmacist : "Hello, Patient? I just wanted to let you know that your prescription is filled and has been paid in full."
* Patient: "That's so great! I'll come pick it up right away. Now I won't die because my medication is too *!#&$# expensive!"
Then everyone rides dinosaurs into the sunset, happy and healthy. It's pretty awesome.
(That's the idea anyway. WE'll see if reality follows.)
mercy
=====
Mercy is a web application designed to facilitate payments between people who need prescriptions but can't afford them, and charitable people who can afford them
The Idea
========
* Patient: "Here's my prescription, Mr Pharmacist"
* Pharmacist: "Sure thing. That'll be $150 USD."
* Patient: "I can't afford that! Can you submit it to Mercy for me?"
* Pharmacist: "Sure thing. We'll call you when it gets filled."
... Meanwhile, back at stately Charitable Person Manor ...
* Charitable Person's Phone: *DING*
* Charitable Person: "Huh? Oh! Mercy is telling me there's a new prescription I can pay for. I love helping people!"
* Charitable person clicks some buttons
* Charitable Person: "*CLICK* Paid! Enjoy some good health, stranger. Now to post it on my facebook wall and collect the new achievement for buying my first Leukemia drug..."
... Back at the pharmacy ...
* Pharmacist : "Hello, Patient? I just wanted to let you know that your prescription is filled and has been paid in full."
* Patient: "That's so great! I'll come pick it up right away. Now I won't die because my medication is too *!#&$# expensive!"
Then everyone rides dinosaurs into the sunset, happy and healthy. It's pretty awesome.
(That's the idea anyway. WE'll see if reality follows.)

98
alembic.ini
View File

@@ -1,49 +1,49 @@
# A generic, single database configuration.
[alembic]
# path to migration scripts
script_location = alembic
# template used to generate migration files
# file_template = %%(rev)s_%%(slug)s
# set to 'true' to run the environment during
# the 'revision' command, regardless of autogenerate
# revision_environment = false
sqlalchemy.url = postgresql://mercy:mercy@postgresql.aklabs.net/mercy
# Logging configuration
[loggers]
keys = root,sqlalchemy,alembic
[handlers]
keys = console
[formatters]
keys = generic
[logger_root]
level = WARN
handlers = console
qualname =
[logger_sqlalchemy]
level = WARN
handlers =
qualname = sqlalchemy.engine
[logger_alembic]
level = INFO
handlers =
qualname = alembic
[handler_console]
class = StreamHandler
args = (sys.stderr,)
level = NOTSET
formatter = generic
[formatter_generic]
format = %(levelname)-5.5s [%(name)s] %(message)s
datefmt = %H:%M:%S
# A generic, single database configuration.
[alembic]
# path to migration scripts
script_location = alembic
# template used to generate migration files
# file_template = %%(rev)s_%%(slug)s
# set to 'true' to run the environment during
# the 'revision' command, regardless of autogenerate
# revision_environment = false
sqlalchemy.url = postgresql://mercy:mercy@postgresql.aklabs.net/mercy
# Logging configuration
[loggers]
keys = root,sqlalchemy,alembic
[handlers]
keys = console
[formatters]
keys = generic
[logger_root]
level = WARN
handlers = console
qualname =
[logger_sqlalchemy]
level = WARN
handlers =
qualname = sqlalchemy.engine
[logger_alembic]
level = INFO
handlers =
qualname = alembic
[handler_console]
class = StreamHandler
args = (sys.stderr,)
level = NOTSET
formatter = generic
[formatter_generic]
format = %(levelname)-5.5s [%(name)s] %(message)s
datefmt = %H:%M:%S

26
mercy/MercyApplication.py
View File

@@ -1,4 +1,22 @@
import flask
class MercyApplication(flask.Flask):
pass
import flask
import mercy.config
from flask.ext.sqlalchemy import SQLAlchemy
class MercyApplication(flask.Flask):
pass
app = None
db = None
def get_db():
global db
if not db:
db = SQLAlchemy(get_app())
return db
def get_app():
global app
if not app:
app = MercyApplication("mercy")
app.config['SQLALCHEMY_DATABASE_URI'] = mercy.config.SQLALCHEMY_URI
return app

107
mercy/importers/fda.py
View File

@@ -1,8 +1,107 @@
import mercy.db
import mercy.MercyApplication
import xml.etree.cElementTree as ET
from mercy.models.fda import Product
from mercy.models.fda import ProductSubstance
from mercy.models.fda import ProductSubstanceMap
from mercy.models.fda import PharmaceuticalClass
from mercy.models.fda import PharmaceuticalClassMap
import sqlalchemy.exc
import csv
class FDAImporter:
def __init__(self, *args, **kwargs):
self.__database = mercy.db.Database()
self.__db = mercy.MercyApplication.get_db()
def read(self, fname):
raise Exception("FDAImporter.read doesn't do anything yet")
def read(self, fname, startIdx=0):
with open(fname, "r") as ifile:
reader = csv.DictReader(ifile, delimiter="\t")
idx = 0
for row in reader:
if idx < startIdx:
print "Skipping from {} to {}".format(idx, startIdx)
idx += 1
continue
retries = 0
print "{} : {}".format(idx, row)
while retries < 3 :
try:
self._convert_row(row)
retries = 3
except sqlalchemy.exc.DatabaseError, e:
retries += 1
if retries == 3:
raise e
else:
continue
idx += 1
def _saveobj(self, obj):
self.__db.session.add(obj)
self.__db.session.commit()
def _convert_row(self, row):
# The FDA CSV is HIGHLY irregular. This function is needlessly large because of that/
product = Product.query.filter_by(productid=row['PRODUCTID']).first()
if not product:
product = Product()
product.productid = row['PRODUCTID']
product.ndc = row['PRODUCTNDC']
product.type = row['PRODUCTTYPENAME']
product.proprietaryName = row['PROPRIETARYNAME']
product.proprietaryNameSuffix = row['PROPRIETARYNAMESUFFIX']
product.genericName = row['NONPROPRIETARYNAME']
product.marketingCategoryName = row['MARKETINGCATEGORYNAME']
product.labelerName = row['LABELERNAME']
product.deaSchedule = (row['DEASCHEDULE'] if row['DEASCHEDULE'] else '')
self._saveobj(product)
# Strip the substances off of the product and make objects for them
substanceNames = [x.strip().lstrip() for x in row['SUBSTANCENAME'].split(';')]
tmpQtys = [x.strip().lstrip() for x in row['ACTIVE_NUMERATOR_STRENGTH'].split(';')]
# The list addition here is to make sure we have an equal number of values in
# quantities as we do in substance names, because the FDA CSV does not ensure
# that all substance names have a quantity or unit listed. So unknown quantities
# get entered as '0', unknown units of measure become '?'.
substanceQtys = [(float(x) if x else 0.0) for x in tmpQtys] + ([0] * (len(substanceNames) - len(tmpQtys)))
substanceUnits = [x.strip().lstrip() for x in row['ACTIVE_INGRED_UNIT'].split(';')]
substanceUnits += (['?'] * (len(substanceNames) - len(substanceUnits)))
for idx in range(0, len(substanceNames)):
substance = None
substanceName = substanceNames[idx]
substance = ProductSubstance.query.filter_by(name=substanceName).first()
if not substance:
substance = ProductSubstance()
substance.name = substanceName
self._saveobj(substance)
substanceMap = ProductSubstanceMap.query.filter_by(product_id=product.id,
substance_id=substance.id,
quantity=substanceQtys[idx],
units=substanceUnits[idx]).first()
if not substanceMap:
substanceMap = ProductSubstanceMap()
substanceMap.product_id = product.id
substanceMap.substance_id = substance.id
substanceMap.quantity = substanceQtys[idx]
substanceMap.units = substanceUnits[idx]
self._saveobj(substanceMap)
pharmaClassList = row.get('PHARM_CLASSES')
pharmaClasses = [x.strip().lstrip() for x in (pharmaClassList if pharmaClassList else '').split(',')]
for pharmaClass in pharmaClasses:
pharmaObj = PharmaceuticalClass.query.filter_by(name=pharmaClass).first()
if not pharmaObj:
pharmaObj = PharmaceuticalClass()
pharmaObj.name = pharmaClass
self._saveobj(pharmaObj)
mapObj = PharmaceuticalClassMap.query.filter_by(
product_id=product.id,
pharma_id=pharmaObj.id).first()
if not mapObj:
mapObj = PharmaceuticalClassMap()
mapObj.product_id = product.id
mapObj.pharma_id = pharmaObj.id
self._saveobj(mapObj)
return

30
mercy/models/drugbank.py
View File

@@ -1,6 +1,7 @@
import sqlalchemy as sa
from mercy.models.simplemodel import SimpleModel
import mercy.MercyApplication
from mercy.models.fda import Product
import sqlalchemy.dialects.postgresql as pgdialect
db = mercy.MercyApplication.get_db()
@@ -8,10 +9,11 @@ db = mercy.MercyApplication.get_db()
class Drug(SimpleModel, db.Model):
__tablename__ = "drugbank_drugs"
id = sa.Column(sa.String, primary_key=True, unique=True)
id = sa.Column(sa.Integer, primary_key=True, autoincrement=True, nullable=False)
dbid = sa.Column(sa.String, index=True, unique=True)
name = sa.Column(sa.String, nullable=False, index=True)
indication = sa.Column(sa.String, nullable=False)
ndc_id = sa.Column(sa.String, sa.ForeignKey('fda_products.id'), nullable=True)
fda_product_id = sa.Column(sa.String, sa.ForeignKey(Product.productid), nullable=True)
wikipedia = sa.Column(sa.String, nullable=True)
__repr_keys__ = { 'id': basestring,
@@ -21,7 +23,8 @@ class Drug(SimpleModel, db.Model):
class Price(SimpleModel, db.Model):
__tablename__ = "drugbank_prices"
drug_id = sa.Column(sa.String, sa.ForeignKey(Drug.id), primary_key=True, nullable=False)
id = sa.Column(sa.Integer, primary_key=True, autoincrement=True, nullable=False)
drug_id = sa.Column(sa.Integer, sa.ForeignKey(Drug.id), nullable=False)
description = sa.Column(sa.String, nullable=False)
currency = sa.Column(sa.String, nullable=False)
cost = sa.Column(sa.Float, nullable=False, index=True)
@@ -30,40 +33,45 @@ class Price(SimpleModel, db.Model):
class CategoryName(SimpleModel, db.Model):
__tablename__ = "drugbank_categories"
id = sa.Column(sa.Integer, primary_key=True, autoincrement=True, nullable=False)
name = sa.Column(sa.String, nullable=False)
name = sa.Column(sa.String, nullable=False, unique=True)
class CategoryMap(SimpleModel, db.Model):
__tablename__ = "drugbank_category_maps"
drug_id = sa.Column(sa.String, sa.ForeignKey(Drug.id), primary_key=True, nullable=False)
id = sa.Column(sa.Integer, primary_key=True, autoincrement=True, nullable=False)
drug_id = sa.Column(sa.Integer, sa.ForeignKey(Drug.id), nullable=False)
category_id = sa.Column(sa.Integer, sa.ForeignKey(CategoryName.id), nullable=False)
class Packager(SimpleModel, db.Model):
__tablename__ = "drugbank_packagers"
id = sa.Column(sa.Integer, primary_key=True, autoincrement=True, nullable=False)
name = sa.Column(sa.String, nullable=False)
name = sa.Column(sa.String, nullable=False, unique=True)
url = sa.Column(sa.String, nullable=True)
class PackagerMap(SimpleModel, db.Model):
__tablename__ = "drugbank_packager_maps"
drug_id = sa.Column(sa.String, sa.ForeignKey(Drug.id), primary_key=True, nullable=False)
id = sa.Column(sa.Integer, primary_key=True, autoincrement=True, nullable=False)
drug_id = sa.Column(sa.String, sa.ForeignKey(Drug.id), nullable=False)
packager_id = sa.Column(sa.Integer, sa.ForeignKey(Packager.id), nullable=False)
class Manufacturer(SimpleModel, db.Model):
__tablename__ = "drugbank_manufacturers"
id = sa.Column(sa.Integer, primary_key=True, autoincrement=True, nullable=False)
name = sa.Column(sa.String, nullable=False)
name = sa.Column(sa.String, nullable=False, unique=True)
class ManufacturerMap(SimpleModel, db.Model):
__tablename__ = "drugbank_manufacturer_maps"
drug_id = sa.Column(sa.String, sa.ForeignKey(Drug.id), primary_key=True, nullable=False)
id = sa.Column(sa.Integer, primary_key=True, autoincrement=True, nullable=False)
drug_id = sa.Column(sa.Integer, sa.ForeignKey(Drug.id), nullable=False)
manufacturer_id = sa.Column(sa.Integer, sa.ForeignKey(Manufacturer.id), nullable=False)
class GenericName(SimpleModel, db.Model):
__tablename__ = "drugbank_genericnames"
drug_id = sa.Column(sa.String, sa.ForeignKey(Drug.id), primary_key=True, nullable=False)
id = sa.Column(sa.Integer, primary_key=True, autoincrement=True, nullable=False)
drug_id = sa.Column(sa.Integer, sa.ForeignKey(Drug.id), nullable=False)
name = sa.Column(sa.String, nullable=False)
class Synonym(SimpleModel, db.Model):
__tablename__ = "drugbank_synonyms"
drug_id = sa.Column(sa.String, sa.ForeignKey(Drug.id), primary_key=True, nullable=False)
id = sa.Column(sa.Integer, primary_key=True, autoincrement=True, nullable=False)
drug_id = sa.Column(sa.Integer, sa.ForeignKey(Drug.id), nullable=False)
name = sa.Column(sa.String, nullable=False)

46
mercy/models/fda.py
View File

@@ -8,8 +8,9 @@ db = mercy.MercyApplication.get_db()
class Product(SimpleModel, db.Model):
__tablename__ = 'fda_products'
id = sa.Column(sa.String, primary_key=True)
ndc = sa.Column(sa.String, nullable=False)
id = sa.Column(sa.Integer, primary_key=True, autoincrement=True, nullable=False)
productid = sa.Column(sa.String, index=True, unique=True, nullable=False)
ndc = sa.Column(sa.String, index=True, nullable=False)
type = sa.Column(sa.String, nullable=False)
proprietaryName = sa.Column(sa.String, nullable=False, index=True)
proprietaryNameSuffix = sa.Column(sa.String)
@@ -27,11 +28,36 @@ class Product(SimpleModel, db.Model):
class ProductSubstance(SimpleModel, db.Model):
__tablename__ = 'fda_product_substances'
fda_product_id = sa.Column(sa.String,
sa.ForeignKey(Product.id),
primary_key=True,
nullable=False)
substanceName = sa.Column(sa.String, nullable=False)
strengthNumber = sa.Column(sa.Float, nullable=False)
strengthUnit = sa.Column(sa.String, nullable=False)
pharmaClasses = sa.Column(pgdialect.ARRAY(sa.String), nullable=False)
id = sa.Column(sa.Integer, primary_key=True, autoincrement=True, nullable=False)
name = sa.Column(sa.String, nullable=False)
class ProductSubstanceMap(SimpleModel, db.Model):
__tablename__ = 'fda_product_substance_map'
id = sa.Column(sa.Integer, primary_key=True, autoincrement=True, nullable=False)
product_id = sa.Column(sa.Integer,
sa.ForeignKey(Product.id),
nullable=False)
substance_id = sa.Column(sa.Integer,
sa.ForeignKey(ProductSubstance.id),
nullable=False,
index=True)
quantity = sa.Column(sa.Float, nullable=False)
units = sa.Column(sa.String, nullable=False)
class PharmaceuticalClass(SimpleModel, db.Model):
__tablename__ = "fda_pharma_classes"
id = sa.Column(sa.Integer, primary_key=True, autoincrement=True, nullable=False)
name = sa.Column(sa.String, nullable=False, unique=True)
class PharmaceuticalClassMap(SimpleModel, db.Model):
__tablename__ = "fda_pharma_class_maps"
id = sa.Column(sa.Integer, primary_key=True, autoincrement=True, nullable=False)
product_id = sa.Column(sa.Integer,
sa.ForeignKey(Product.id),
nullable=False)
pharma_id = sa.Column(sa.Integer,
sa.ForeignKey(PharmaceuticalClass.id),
primary_key=True,
nullable=False)

2
mercy/version.py
View File

@@ -1 +1 @@
VERSION="0.0-0"
VERSION="0.0-1"

6
nagios/README.md
View File

@@ -1,4 +1,4 @@
mercy/nagios
============
mercy/nagios
============
This is a directory full of nagios plugins meant to monitor the Mercy application's health.

62
puppet/Makefile
View File

@@ -1,32 +1,32 @@
MAJOR:=$(shell bash -c 'source version.sh ; echo $$MAJOR')
BUILD:=$(shell bash -c 'source version.sh ; echo $$BUILD')
MODULEPATH:=$(shell puppet config print modulepath)
ifeq ($(MODULEPATH),)
$(error "I can't figure out your puppet modulepath, this will cause all kinds of bad things to happen with 'make install' and 'make uninstall'")
endif
PUPPET_DEPS=$(find mercy -type f)
PUPPET_DIST=mercy/pkg/akesterson-mercy-$(MAJOR).$(BUILD).tar.gz
.PHONY: clean
clean:
rm pkg/*
rm Modulefile
mercy/Modulefile: Modulefile.template.sh version.sh
source version.sh && bash Modulefile.template.sh > Modulefile
$(PUPPET_DIST): $(PUPPET_DEPS) mercy/Modulefile
rm -f $(PUPPET_DIST)
mkdir -p $$(dirname $(PUPPET_DIST))
echo 'We would use puppet module tool here but we dont because it doesnt work on windows buildhosts and cant upload from the command line anyway'
tar --exclude=mercy/Modulefile --exclude=mercy/pkg -czvf $(PUPPET_DIST) mercy/
dist: $(PUPPET_DIST)
all: $(PUPPET_DIST)
install: $(PUPPET_DIST)
tar -zxvf $(PUPPET_DIST) -C $(MODULEPATH)
uninstall:
MAJOR:=$(shell bash -c 'source version.sh ; echo $$MAJOR')
BUILD:=$(shell bash -c 'source version.sh ; echo $$BUILD')
MODULEPATH:=$(shell puppet config print modulepath)
ifeq ($(MODULEPATH),)
$(error "I can't figure out your puppet modulepath, this will cause all kinds of bad things to happen with 'make install' and 'make uninstall'")
endif
PUPPET_DEPS=$(find mercy -type f)
PUPPET_DIST=mercy/pkg/akesterson-mercy-$(MAJOR).$(BUILD).tar.gz
.PHONY: clean
clean:
rm pkg/*
rm Modulefile
mercy/Modulefile: Modulefile.template.sh version.sh
source version.sh && bash Modulefile.template.sh > Modulefile
$(PUPPET_DIST): $(PUPPET_DEPS) mercy/Modulefile
rm -f $(PUPPET_DIST)
mkdir -p $$(dirname $(PUPPET_DIST))
echo 'We would use puppet module tool here but we dont because it doesnt work on windows buildhosts and cant upload from the command line anyway'
tar --exclude=mercy/Modulefile --exclude=mercy/pkg -czvf $(PUPPET_DIST) mercy/
dist: $(PUPPET_DIST)
all: $(PUPPET_DIST)
install: $(PUPPET_DIST)
tar -zxvf $(PUPPET_DIST) -C $(MODULEPATH)
uninstall:
rm -fr $(MODULEPATH)/mercy

24
puppet/Modulefile.template.sh
View File

@@ -1,12 +1,12 @@
#!/bin/bash
cat <<EOF
name 'akesterson-mercy'
version '${MAJOR}-${BUILD}'
dependency 'apache/apache'
summary 'Puppet module for deploying the mercy web application'
description 'Puppet module for deploying the mercy web application'
project_page http://github.com/akesterson/mercy/tree/master/puppet
license 'MIT'
author 'Andrew Kesterson <andrew@aklabs.net>'
EOF
#!/bin/bash
cat <<EOF
name 'akesterson-mercy'
version '${MAJOR}-${BUILD}'
dependency 'apache/apache'
summary 'Puppet module for deploying the mercy web application'
description 'Puppet module for deploying the mercy web application'
project_page http://github.com/akesterson/mercy/tree/master/puppet
license 'MIT'
author 'Andrew Kesterson <andrew@aklabs.net>'
EOF

98
puppet/README.md
View File

@@ -1,49 +1,49 @@
mercy
=====
A puppet module for installing the Mercy Flask application
Requirements
============
This puppet module assumes that you have:
* A RabbitMQ broker that we can use for Celery tasks
* Working apache on localhost, with mod_wsgi enabled and with a puppet service we can notify
* Working python 2.7 and pip on localhost
* Working postgres with a username, password, and database for mercy to use
Aside from that, the module has no dependencies.
Usage
=========
class { 'mercy':
# ---- These are required, they have no defaults
environment => 'dev|production',
version => 'absent|latest|MAJOR-MINOR',
ensure => 'running|stopped'.
rabbitmq_uri => 'RABBITMQ_BROKER_URI',
# ---- Everything below is optional
process_user => 'mercy',
process_group => 'mercy',
process_threads => 5,
servername => $::fqdn,
rabbitmq_user => 'mercy',
rabbitmq_pw => 'mercy',
rabbitmq_vhost => 'mercy',
vhost_dir => '/etc/apache/httpd/conf.d',
apache_service => 'httpd',
port => 443,
postgres_uri => 'localhost',
postgres_user => 'mercy',
postgres_pw => 'mercy',
postgres_db => 'mercy'
}
If 'environment' is dev, then the mercy application will be installed via a tarball located in ./mercy/files/mercy-${ensure}.tar.gz. If 'environment' is production, then mercy will be installed via pip and ensured at the given version.
Events
======
This puppet module will notify your apache service whenever the vhost is modified, or the mercy application version is updated, since an apache graceful restart will be required in either case.
mercy
=====
A puppet module for installing the Mercy Flask application
Requirements
============
This puppet module assumes that you have:
* A RabbitMQ broker that we can use for Celery tasks
* Working apache on localhost, with mod_wsgi enabled and with a puppet service we can notify
* Working python 2.7 and pip on localhost
* Working postgres with a username, password, and database for mercy to use
Aside from that, the module has no dependencies.
Usage
=========
class { 'mercy':
# ---- These are required, they have no defaults
environment => 'dev|production',
version => 'absent|latest|MAJOR-MINOR',
ensure => 'running|stopped'.
rabbitmq_uri => 'RABBITMQ_BROKER_URI',
# ---- Everything below is optional
process_user => 'mercy',
process_group => 'mercy',
process_threads => 5,
servername => $::fqdn,
rabbitmq_user => 'mercy',
rabbitmq_pw => 'mercy',
rabbitmq_vhost => 'mercy',
vhost_dir => '/etc/apache/httpd/conf.d',
apache_service => 'httpd',
port => 443,
postgres_uri => 'localhost',
postgres_user => 'mercy',
postgres_pw => 'mercy',
postgres_db => 'mercy'
}
If 'environment' is dev, then the mercy application will be installed via a tarball located in ./mercy/files/mercy-${ensure}.tar.gz. If 'environment' is production, then mercy will be installed via pip and ensured at the given version.
Events
======
This puppet module will notify your apache service whenever the vhost is modified, or the mercy application version is updated, since an apache graceful restart will be required in either case.

44
puppet/mercy/manifests/init.pp
View File

@@ -1,22 +1,22 @@
class mercy(
$environment,
$version,
$ensure,
$rabbitmq_uri,
$process_user => 'mercy',
$process_group => 'mercy',
$process_threads => 5,
$servername => $::fqdn,
$rabbitmq_user => 'mercy',
$rabbitmq_pw => 'mercy',
$rabbitmq_vhost => 'mercy',
$vhost_dir => '/etc/apache/httpd/conf.d',
$apache_service => 'httpd',
$port => 443,
$postgres_uri => 'localhost',
$postgres_user => 'mercy',
$postgres_pw => 'mercy',
$postgres_db => 'mercy')
{
include 'mercy::params'
}
class mercy(
$environment,
$version,
$ensure,
$rabbitmq_uri,
$process_user => 'mercy',
$process_group => 'mercy',
$process_threads => 5,
$servername => $::fqdn,
$rabbitmq_user => 'mercy',
$rabbitmq_pw => 'mercy',
$rabbitmq_vhost => 'mercy',
$vhost_dir => '/etc/apache/httpd/conf.d',
$apache_service => 'httpd',
$port => 443,
$postgres_uri => 'localhost',
$postgres_user => 'mercy',
$postgres_pw => 'mercy',
$postgres_db => 'mercy')
{
include 'mercy::params'
}

8
puppet/mercy/manifests/params.pp
View File

@@ -1,4 +1,4 @@
class mercy::params
{
}
class mercy::params
{
}

22
puppet/mercy/templates/apache/vhost.erb
View File

@@ -1,12 +1,12 @@
<VirtualHost * :<%= scope.lookupvar('::mercy::port') %>>
ServerName <%= scope.lookupvar('::fqdn') %>
WSGIDaemonProcess mercy user=<%= scope.lookupvar('::mercy::process_user') %> group=<%= scope.lookupvar('::mercy::process_group') %> threads=<%= scope.lookupvar('::mercy::process_threads') %>
WSGIScriptAlias / /opt/mercy/scripts/mercy.wsgi
<Directory /opt/mercy>
WSGIProcessGroup mercy
WSGIApplicationGroup %{GLOBAL}
Order deny, allow
Allow from all
</Directory>
<VirtualHost * :<%= scope.lookupvar('::mercy::port') %>>
ServerName <%= scope.lookupvar('::fqdn') %>
WSGIDaemonProcess mercy user=<%= scope.lookupvar('::mercy::process_user') %> group=<%= scope.lookupvar('::mercy::process_group') %> threads=<%= scope.lookupvar('::mercy::process_threads') %>
WSGIScriptAlias / /opt/mercy/scripts/mercy.wsgi
<Directory /opt/mercy>
WSGIProcessGroup mercy
WSGIApplicationGroup %{GLOBAL}
Order deny, allow
Allow from all
</Directory>
</VirtualHost>

98
puppet/mercy/templates/python/alembic.ini.erb
View File

@@ -1,49 +1,49 @@
# A generic, single database configuration.
[alembic]
# path to migration scripts
script_location = /opt/mercy/alembic
# template used to generate migration files
# file_template = %%(rev)s_%%(slug)s
# set to 'true' to run the environment during
# the 'revision' command, regardless of autogenerate
# revision_environment = false
sqlalchemy.url = postgresql://<%= scope.lookupvar('::mercy::postgres_user') %>:<%+ scope.lookupvar('::mercy::postgres_pw') %>@<%= scope.lookupvar('::mercy::postgres_uri') %>/<%= scope.lookupvar('::mercy::postgres_db') %>
# Logging configuration
[loggers]
keys = root,sqlalchemy,alembic
[handlers]
keys = console
[formatters]
keys = generic
[logger_root]
level = WARN
handlers = console
qualname =
[logger_sqlalchemy]
level = WARN
handlers =
qualname = sqlalchemy.engine
[logger_alembic]
level = INFO
handlers =
qualname = alembic
[handler_console]
class = StreamHandler
args = (sys.stderr,)
level = NOTSET
formatter = generic
[formatter_generic]
format = %(levelname)-5.5s [%(name)s] %(message)s
datefmt = %H:%M:%S
# A generic, single database configuration.
[alembic]
# path to migration scripts
script_location = /opt/mercy/alembic
# template used to generate migration files
# file_template = %%(rev)s_%%(slug)s
# set to 'true' to run the environment during
# the 'revision' command, regardless of autogenerate
# revision_environment = false
sqlalchemy.url = postgresql://<%= scope.lookupvar('::mercy::postgres_user') %>:<%+ scope.lookupvar('::mercy::postgres_pw') %>@<%= scope.lookupvar('::mercy::postgres_uri') %>/<%= scope.lookupvar('::mercy::postgres_db') %>
# Logging configuration
[loggers]
keys = root,sqlalchemy,alembic
[handlers]
keys = console
[formatters]
keys = generic
[logger_root]
level = WARN
handlers = console
qualname =
[logger_sqlalchemy]
level = WARN
handlers =
qualname = sqlalchemy.engine
[logger_alembic]
level = INFO
handlers =
qualname = alembic
[handler_console]
class = StreamHandler
args = (sys.stderr,)
level = NOTSET
formatter = generic
[formatter_generic]
format = %(levelname)-5.5s [%(name)s] %(message)s
datefmt = %H:%M:%S

20
scripts/mercy.wsgi
View File

@@ -1,14 +1,6 @@
from mercy.MercyApplication import MercyApplication
class ScriptNameStripper(object):
def __init__(self, app):
self.app = app
def __call_(self, environ, start_response):
environ['SCRIPT_NAME'] = ''
return self.app(environ, start_response)
app = ScriptNameStripper(MercyApplication())
if __name__ == "__main__":
app.run()
import mercy.MercyApplication
app = get_application()
if __name__ == "__main__":
app.run()

13
scripts/mercy_app.py
View File

@@ -1,7 +1,6 @@
#!/usr/bin/env python
from mercy.MercyApplication import MercyApplication
if __name__ == "__main__":
app = MercyApplication("mercy")
app.run()
#!/usr/bin/env python
from mercy.MercyApplication import app
if __name__ == "__main__":
app.run()

71
setup.py
View File

@@ -1,34 +1,37 @@
from distutils.core import setup
import mercy.version
import os
import sys
if __name__ == "__main__":
setup(
name="mercy",
url="https://www.github.com/akesterson/mercy",
version=mercy.version.VERSION,
description="A flask application that facilitates paying for prescriptions",
long_description="",
author=("Andrew Kesterson"),
author_email="andrew@aklabs.net",
license="MIT",
install_requires=["flask",
"sqlalchemy",
"alembic",
"psycopg2"],
scripts=[],
packages=["mercy"],
data_files=[],
classifiers=[
'Development Status :: 1 - Planning',
'Environment :: Web Environment',
'Framework :: Flask',
'Intended Audience :: Healthcare Industry',
'License :: OSI Approved :: MIT License',
'Natural Language :: English',
'Programming Language :: Python :: 2.7',
'Topic :: Other/Nonlisted Topic',
],
)
from distutils.core import setup
import mercy.version
import os
import sys
if __name__ == "__main__":
setup(
name="mercy",
url="https://www.github.com/akesterson/mercy",
version=mercy.version.VERSION,
description="A web application that facilitates charitable prescription payments",
long_description="",
author=("Andrew Kesterson"),
author_email="andrew@aklabs.net",
license="MIT",
install_requires=["flask",
"sqlalchemy",
"alembic",
"psycopg2",
"flask-sqlalchemy"],
scripts=["scripts/mercy.wsgi"],
packages=["mercy",
"mercy/models",
"mercy/importers"],
data_files=[],
classifiers=[
'Development Status :: 1 - Planning',
'Environment :: Web Environment',
'Framework :: Flask',
'Intended Audience :: Healthcare Industry',
'License :: OSI Approved :: MIT License',
'Natural Language :: English',
'Programming Language :: Python :: 2.7',
'Topic :: Other/Nonlisted Topic',
],
)

128
tests/importers/test_fda_importer.py
View File

@@ -5,52 +5,116 @@ import mercy.models
import mercy.importers.fda
import mercy.exceptions
VALID_ROWS=[]
COMPARISON_KEYS={
'productid': 'PRODUCTID',
'ndc': 'PRODUCTNDC',
'type': 'PRODUCTTYPENAME',
'proprietaryName': 'PROPRIETARYNAME',
'proprietaryNameSuffix': 'PROPRIETARYNAMESUFFIX',
'genericName': 'NONPROPRIETARYNAME',
'marketingCategoryName': 'MARKETINGCATEGORYNAME',
'labelerName': 'LABELERNAME',
'deaSchedule': 'DEASCHEDULE'
}
CSV_KEYS=[
'PRODUCTID',
'PRODUCTNDC',
'PRODUCTTYPENAME',
'PROPRIETARYNAME',
'PROPRIETARYNAMESUFFIX',
'NONPROPRIETARYNAME',
'DOSAGEFORMNAME',
'ROUTENAME',
'STARTMARKETINGDATE',
'ENDMARKETINGDATE',
'MARKETINGCATEGORYNAME',
'APPLICATIONNUMBER',
'LABELERNAME',
'SUBSTANCENAME',
'ACTIVE_NUMERATOR_STRENGTH',
'ACTIVE_INGRED_UNIT',
'PHARM_CLASSES',
'DEASCHEDULE'
]
CANNED_ROWS=[
{'PRODUCTID': '0002-3230_b1642902-4a44-495a-8790-39598b168276',
'PRODUCTNDC': '0002-3230',
'PRODUCTTYPENAME': 'HUMAN PRESCRIPTION DRUG',
'PROPRIETARYNAME': 'Symbyax',
'PROPRIETARYNAMESUFFIX': '',
'NONPROPRIETARYNAME': 'Olanzapine and Fluoxetine hydrochloride',
'DOSAGEFORMNAME': 'CAPSULE',
'ROUTENAME': 'ORAL',
'STARTMARKETINGDATE': '20070409',
'ENDMARKETINGDATE': '',
'MARKETINGCATEGORYNAME': 'NDA',
'APPLICATIONNUMBER': 'NDA021520',
'LABELERNAME': 'Eli Lilly and Company',
'SUBSTANCENAME': 'FLUOXETINE HYDROCHLORIDE; OLANZAPINE',
'ACTIVE_NUMERATOR_STRENGTH': '25; 3',
'ACTIVE_INGRED_UNIT': 'mg/1; mg/1',
'PHARM_CLASSES': 'Atypical Antipsychotic [EPC],Serotonin Reuptake Inhibitor [EPC],Serotonin Uptake Inhibitors [MoA]',
'DEASCHEDULE': ''
},
{'PRODUCTID': '0002-3231_b1642902-4a44-495a-8790-39598b168276',
'PRODUCTNDC': '0002-3231',
'PRODUCTTYPENAME': 'HUMAN PRESCRIPTION DRUG',
'PROPRIETARYNAME': 'Symbyax',
'PROPRIETARYNAMESUFFIX': '',
'NONPROPRIETARYNAME': 'Olanzapine and Fluoxetine hydrochloride',
'DOSAGEFORMNAME': 'CAPSULE',
'ROUTENAME': 'ORAL',
'STARTMARKETINGDATE': '20070409',
'ENDMARKETINGDATE': '',
'MARKETINGCATEGORYNAME': 'NDA',
'APPLICATIONNUMBER': 'NDA021521',
'LABELERNAME': 'Eli Lilly and Company',
'SUBSTANCENAME': 'FLUOXETINE HYDROCHLORIDE; OLANZAPINE',
'ACTIVE_NUMERATOR_STRENGTH': '25; 6',
'ACTIVE_INGRED_UNIT': 'mg/1; mg/1',
'PHARM_CLASSES': 'Atypical Antipsychotic [EPC],Serotonin Reuptake Inhibitor [EPC],Serotonin Uptake Inhibitors [MoA]',
'DEASCHEDULE': ''
}
]
FIXTUREFILE=os.path.abspath(
os.path.join(
__file__,
os.path.dirname(__file__),
"..",
"fixtures",
"fda_database.tar.gz"
"fda_database.txt"
)
)
FIXTUREFILE_BAD=os.path.abspath(
os.path.join(
__file__,
os.path.dirname(__file__),
"..",
"fixtures",
"fda_database_bad.tar.gz"
"fda_database_bad.txt"
)
)
FIXTUREFILE_CORRUPT=os.path.abspath(
os.path.join(
__file__,
"..",
"fixtures",
"fda_database_corrupt.tar.gz"
)
)
@raises(mercy.exceptions.CorruptTarError)
def test_fda_import_fails_on_corrupt_tar():
importer = mercy.importers.fda.FDAImporter()
impoter.read(FIXTUREFILE_CORRUPT)
def test_fda_import_populates_table():
importer = FDAImporter().read(FIXTUREFILE)
rows = mercy.models.fda.Product.query.all()
for i in range(0, len(rows)):
row = rows[i]
canned_row = CANNED_ROWS[i]
assert(len(row) == len(canned_row))
for j in canned_row.keys():
assert(row[j] == canned_row[j])
@raises(AttributeError, KeyError, ValueError)
def test_fda_import_rejects_bad_records:
with open(FIXTUREFILE, 'w') as ofile:
ofile.write("{}\n".format('\t'.join(CSV_KEYS)))
for row in CANNED_ROWS:
values = []
for key in CSV_KEYS:
values.append(row[key])
ofile.write("{}\n".format('\t'.join(values)))
importer = mercy.importers.fda.FDAImporter()
importer.read(FIXTUREFILE_BAD)
importer.read(FIXTUREFILE)
for row in CANNED_ROWS:
product = mercy.models.fda.Product.query.filter_by(productid = row['PRODUCTID']).first()
assert(product)
for (k, v) in COMPARISON_KEYS.iteritems():
assert(getattr(product, k) == row[v])
mapquery = mercy.models.fda.ProductSubstanceMap.query
substanceMaps = [x for x in mapquery.filter_by(product_id=product.id)]
assert(len(substanceMaps) == len(row['ACTIVE_NUMERATOR_STRENGTH'].split(';')))
# TO DO : This test doesn't look at the contents of the
# substances or substance maps, only that the right
# number of substance maps come out

99
version.sh
View File

@@ -1,15 +1,98 @@
TAG="build,0.0,0"
BRANCH="master"
MAJOR="0.0"
BUILD="0"
SHA1="9b83e26ec3d0e3d1526a4226a8ac0c8580a77bcf"
BUILD="1"
SHA1="e2bc6022649f3af82800d75ba876411f1ed83fa9"
OS_NAME="${OS_NAME:-win}"
OS_VERSION="${OS_VERSION:-}"
ARCH="${ARCH:-i686}"
VERSION="0.0-0"
ARCH="${ARCH:-x86_64}"
VERSION="0.0-1"
BUILDHOST="akesterson-pc"
BUILDUSER="akesterson-pc\akesterson"
BUILDDIR="/c/Users/akesterson/source/upstream/git/akesterson/mercy"
BUILDUSER="akesterson"
BUILDDIR="/cygdrive/c/Users/akesterson/source/upstream/git/akesterson/mercy"
SOURCE="git@github.com:akesterson/mercy.git"
REBUILDING=0
CHANGELOG=""
REBUILDING=1
CHANGELOG="2013-10-19 23:35:39 -0400 Andrew Kesterson <andrew@aklabs.net>
#6 : Make fda_products.genericName an index
[e2bc602] (HEAD, master)
2013-10-19 23:24:38 -0400 Andrew Kesterson <andrew@aklabs.net>
Merge branch 'master' of github.com:akesterson/mercy
[ecef739] (origin/master, origin/HEAD)
2013-10-19 23:23:53 -0400 Andrew Kesterson <andrew@aklabs.net>
Added downgrade to initial DB script
[803d120]
2013-10-19 23:08:23 -0400 Andrew Kesterson <andrew@aklabs.net>
Removed more junk
[1753230]
2013-10-19 23:06:23 -0400 Andrew Kesterson <andrew@aklabs.net>
Merge branch 'master' of www.aklabs.net:~/mercy/
[6b49f61]
2013-10-19 23:05:52 -0400 Andrew Kesterson <andrew@aklabs.net>
Removed abunch of junk
[8caeb4d]
2013-10-19 22:58:57 -0400 Andrew Kesterson <andrew@aklabs.net>
Working on #6. I think I should break out the puppet module stuff into a separate issue, since that risks really putting this all off target.
[dbf64e8]
2013-10-19 22:45:53 -0400 Andrew Kesterson <andrew@aklabs.net>
Midstream
[64fec8d]
2013-10-19 22:45:35 -0400 Andrew Kesterson <andrew@aklabs.net>
Midstream
[646df51]
2013-10-19 22:44:59 -0400 Andrew Kesterson <andrew@aklabs.net>
Midstream
[dcec82b]
2013-10-19 22:43:59 -0400 Andrew Kesterson <andrew@aklabs.net>
Midstream
[4332d08]
2013-10-19 22:30:49 -0400 Andrew Kesterson <andrew@aklabs.net>
Midstream
[c4f610a]
2013-10-19 22:28:46 -0400 Andrew Kesterson <andrew@aklabs.net>
Midstream
[56d46c5]
2013-10-19 17:59:56 -0400 Andrew Kesterson <andrew@aklabs.net>
Updated some design docs
[86e8db6]
2013-10-19 10:38:45 -0400 Andrew Kesterson <andrew@aklabs.net>
Added nagios directory
[45cfa0b]
2013-10-19 10:37:30 -0400 Andrew Kesterson <andrew@aklabs.net>
Added puppet module skeleton (also puppet module tool is lame)
[363d5ef]
2013-10-19 09:53:37 -0400 Andrew Kesterson <andrew@aklabs.net>
Initial commit
[9895e8e]"