Working on #1 - FDA import process works. 3 hours for first import, 1.25 hours on subsequent updates.

2013-10-27 18:33:30 -04:00
parent e2bc602264
commit e0232aa02d
13 changed files with 575 additions and 71 deletions
--- a/alembic/env.py
+++ b/alembic/env.py
@@ -1,71 +1,79 @@
-from __future__ import with_statement
-from alembic import context
-from sqlalchemy import engine_from_config, pool
-from logging.config import fileConfig
-
-# this is the Alembic Config object, which provides
-# access to the values within the .ini file in use.
-config = context.config
-
-# Interpret the config file for Python logging.
-# This line sets up loggers basically.
-fileConfig(config.config_file_name)
-
-# add your model's MetaData object here
-# for 'autogenerate' support
-# from myapp import mymodel
-# target_metadata = mymodel.Base.metadata
-target_metadata = None
-
-# other values from the config, defined by the needs of env.py,
-# can be acquired:
-# my_important_option = config.get_main_option("my_important_option")
-# ... etc.
-
-def run_migrations_offline():
-    """Run migrations in 'offline' mode.
-
-    This configures the context with just a URL
-    and not an Engine, though an Engine is acceptable
-    here as well.  By skipping the Engine creation
-    we don't even need a DBAPI to be available.
-
-    Calls to context.execute() here emit the given string to the
-    script output.
-
-    """
-    url = config.get_main_option("sqlalchemy.url")
-    context.configure(url=url)
-
-    with context.begin_transaction():
-        context.run_migrations()
-
-def run_migrations_online():
-    """Run migrations in 'online' mode.
-
-    In this scenario we need to create an Engine
-    and associate a connection with the context.
-
-    """
-    engine = engine_from_config(
-                config.get_section(config.config_ini_section),
-                prefix='sqlalchemy.',
-                poolclass=pool.NullPool)
-
-    connection = engine.connect()
-    context.configure(
-                connection=connection,
-                target_metadata=target_metadata
-                )
-
-    try:
-        with context.begin_transaction():
-            context.run_migrations()
-    finally:
-        connection.close()
-
-if context.is_offline_mode():
-    run_migrations_offline()
-else:
-    run_migrations_online()
-
+from __future__ import with_statement
+from alembic import context
+from sqlalchemy import engine_from_config, pool
+from logging.config import fileConfig
+import mercy.MercyApplication
+import mercy.models
+import mercy.config
+
+db = mercy.MercyApplication.get_db()
+# this is the Alembic Config object, which provides
+# access to the values within the .ini file in use.
+config = context.config
+
+# Interpret the config file for Python logging.
+# This line sets up loggers basically.
+fileConfig(config.config_file_name)
+
+# add your model's MetaData object here
+# for 'autogenerate' support
+# from myapp import mymodel
+# target_metadata = mymodel.Base.metadata
+target_metadata = db.Model.metadata
+
+# other values from the config, defined by the needs of env.py,
+# can be acquired:
+# my_important_option = config.get_main_option("my_important_option")
+# ... etc.
+
+def run_migrations_offline():
+    """Run migrations in 'offline' mode.
+
+    This configures the context with just a URL
+    and not an Engine, though an Engine is acceptable
+    here as well.  By skipping the Engine creation
+    we don't even need a DBAPI to be available.
+
+    Calls to context.execute() here emit the given string to the
+    script output.
+
+    """
+    url = config.get_main_option("sqlalchemy.url")
+    context.configure(url=url)
+
+    with context.begin_transaction():
+        context.run_migrations()
+
+def run_migrations_online():
+    """Run migrations in 'online' mode.
+
+    In this scenario we need to create an Engine
+    and associate a connection with the context.
+
+    """
+
+    alembic_config = config.get_section(config.config_ini_section)
+    alembic_config['sqlalchemy.url'] = mercy.config.SQLALCHEMY_URI
+
+    engine = engine_from_config(
+                config.get_section(config.config_ini_section),
+                prefix='sqlalchemy.',
+                poolclass=pool.NullPool)
+
+    connection = engine.connect()
+    context.configure(
+                connection=connection,
+                target_metadata=target_metadata
+                )
+
+    try:
+        with context.begin_transaction():
+            context.run_migrations()
+    finally:
+        connection.close()
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()
+
--- a/alembic/versions/2b64ad923738_initial_schema.py
+++ b/alembic/versions/2b64ad923738_initial_schema.py
@@ -0,0 +1,158 @@
+"""Initial schema
+
+Revision ID: 2b64ad923738
+Revises: None
+Create Date: 2013-10-27 11:46:11.475707
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '2b64ad923738'
+down_revision = None
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+    ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('fda_products',
+    sa.Column('id', sa.Integer(), primary_key=True, autoincrement=True, nullable=False),
+    sa.Column('productid', sa.String(), index=True, unique=True, nullable=False),
+    sa.Column('ndc', sa.String(), index=True, nullable=False),
+    sa.Column('type', sa.String(), nullable=False),
+    sa.Column('proprietaryName', sa.String(), nullable=False),
+    sa.Column('proprietaryNameSuffix', sa.String(), nullable=True),
+    sa.Column('genericName', sa.String(), nullable=False),
+    sa.Column('marketingCategoryName', sa.String(), nullable=False),
+    sa.Column('labelerName', sa.String(), nullable=False),
+    sa.Column('deaSchedule', sa.String(), nullable=False),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_table('drugbank_packagers',
+    sa.Column('id', sa.Integer(), primary_key=True, autoincrement=True, nullable=False),
+    sa.Column('name', sa.String(), nullable=False),
+    sa.Column('url', sa.String(), nullable=True),
+    sa.PrimaryKeyConstraint('id'),
+    sa.UniqueConstraint('name')
+    )
+    op.create_table('drugbank_manufacturers',
+    sa.Column('id', sa.Integer(), primary_key=True, autoincrement=True, nullable=False),
+    sa.Column('name', sa.String(), nullable=False),
+    sa.PrimaryKeyConstraint('id'),
+    sa.UniqueConstraint('name')
+    )
+    op.create_table('fda_pharma_classes',
+    sa.Column('id', sa.Integer(), primary_key=True, autoincrement=True, nullable=False),
+    sa.Column('name', sa.String(), nullable=False),
+    sa.PrimaryKeyConstraint('id'),
+    sa.UniqueConstraint('name')
+    )
+    op.create_table('fda_product_substances',
+    sa.Column('id', sa.Integer(), primary_key=True, autoincrement=True, nullable=False),
+    sa.Column('name', sa.String(), nullable=False),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_table('drugbank_categories',
+    sa.Column('id', sa.Integer(), primary_key=True, autoincrement=True, nullable=False),
+    sa.Column('name', sa.String(), nullable=False),
+    sa.PrimaryKeyConstraint('id'),
+    sa.UniqueConstraint('name')
+    )
+    op.create_table('fda_pharma_class_maps',
+    sa.Column('id', sa.Integer(), primary_key=True, autoincrement=True, nullable=False),
+    sa.Column('product_id', sa.Integer(), nullable=False),
+    sa.Column('pharma_id', sa.Integer(), nullable=False),
+    sa.ForeignKeyConstraint(['pharma_id'], ['fda_pharma_classes.id'], ),
+    sa.ForeignKeyConstraint(['product_id'], ['fda_products.id'], ),
+    sa.PrimaryKeyConstraint('id', 'pharma_id')
+    )
+    op.create_table('drugbank_drugs',
+    sa.Column('id', sa.Integer(), primary_key=True, autoincrement=True, nullable=False),
+    sa.Column('dbid', sa.String(), unique=True, nullable=True),
+    sa.Column('name', sa.String(), nullable=False),
+    sa.Column('indication', sa.String(), nullable=False),
+    sa.Column('fda_product_id', sa.String(), nullable=True),
+    sa.Column('wikipedia', sa.String(), nullable=True),
+    sa.ForeignKeyConstraint(['fda_product_id'], ['fda_products.productid'], ),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_table('fda_product_substance_map',
+    sa.Column('id', sa.Integer(), primary_key=True, autoincrement=True, nullable=False),
+    sa.Column('product_id', sa.Integer(), nullable=False),
+    sa.Column('substance_id', sa.Integer(), nullable=False),
+    sa.Column('quantity', sa.Float(), nullable=False),
+    sa.Column('units', sa.String(), nullable=False),
+    sa.ForeignKeyConstraint(['product_id'], ['fda_products.id'], ),
+    sa.ForeignKeyConstraint(['substance_id'], ['fda_product_substances.id'], ),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_table('drugbank_synonyms',
+    sa.Column('id', sa.Integer(), primary_key=True, autoincrement=True, nullable=False),
+    sa.Column('drug_id', sa.Integer(), nullable=False),
+    sa.Column('name', sa.String(), nullable=False),
+    sa.ForeignKeyConstraint(['drug_id'], ['drugbank_drugs.id'], ),
+    sa.PrimaryKeyConstraint('drug_id')
+    )
+    op.create_table('drugbank_packager_maps',
+    sa.Column('id', sa.Integer(), primary_key=True, autoincrement=True, nullable=False),
+    sa.Column('drug_id', sa.Integer(), nullable=False),
+    sa.Column('packager_id', sa.Integer(), nullable=False),
+    sa.ForeignKeyConstraint(['drug_id'], ['drugbank_drugs.id'], ),
+    sa.ForeignKeyConstraint(['packager_id'], ['drugbank_packagers.id'], ),
+    sa.PrimaryKeyConstraint('drug_id')
+    )
+    op.create_table('drugbank_genericnames',
+    sa.Column('id', sa.Integer(), primary_key=True, autoincrement=True, nullable=False),
+    sa.Column('drug_id', sa.Integer(), nullable=False),
+    sa.Column('name', sa.String(), nullable=False),
+    sa.ForeignKeyConstraint(['drug_id'], ['drugbank_drugs.id'], ),
+    sa.PrimaryKeyConstraint('drug_id')
+    )
+    op.create_table('drugbank_prices',
+    sa.Column('id', sa.Integer(), primary_key=True, autoincrement=True, nullable=False),
+    sa.Column('drug_id', sa.Integer(), nullable=False),
+    sa.Column('description', sa.String(), nullable=False),
+    sa.Column('currency', sa.String(), nullable=False),
+    sa.Column('cost', sa.Float(), nullable=False),
+    sa.Column('unit', sa.String(), nullable=False),
+    sa.ForeignKeyConstraint(['drug_id'], ['drugbank_drugs.id'], ),
+    sa.PrimaryKeyConstraint('drug_id')
+    )
+    op.create_table('drugbank_manufacturer_maps',
+    sa.Column('id', sa.Integer(), primary_key=True, autoincrement=True, nullable=False),
+    sa.Column('drug_id', sa.Integer(), nullable=False),
+    sa.Column('manufacturer_id', sa.Integer(), nullable=False),
+    sa.ForeignKeyConstraint(['drug_id'], ['drugbank_drugs.id'], ),
+    sa.ForeignKeyConstraint(['manufacturer_id'], ['drugbank_manufacturers.id'], ),
+    sa.PrimaryKeyConstraint('drug_id')
+    )
+    op.create_table('drugbank_category_maps',
+    sa.Column('id', sa.Integer(), primary_key=True, autoincrement=True, nullable=False),
+    sa.Column('drug_id', sa.Integer(), nullable=False),
+    sa.Column('category_id', sa.Integer(), nullable=False),
+    sa.ForeignKeyConstraint(['category_id'], ['drugbank_categories.id'], ),
+    sa.ForeignKeyConstraint(['drug_id'], ['drugbank_drugs.id'], ),
+    sa.PrimaryKeyConstraint('drug_id')
+    )
+    ### end Alembic commands ###
+
+
+def downgrade():
+    ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('drugbank_category_maps')
+    op.drop_table('drugbank_manufacturer_maps')
+    op.drop_table('drugbank_prices')
+    op.drop_table('drugbank_genericnames')
+    op.drop_table('drugbank_packager_maps')
+    op.drop_table('drugbank_synonyms')
+    op.drop_table('fda_product_substance_map')
+    op.drop_table('drugbank_drugs')
+    op.drop_table('fda_pharma_class_maps')
+    op.drop_table('drugbank_categories')
+    op.drop_table('fda_product_substances')
+    op.drop_table('fda_pharma_classes')
+    op.drop_table('fda_products')
+    op.drop_table('drugbank_manufacturers')
+    op.drop_table('drugbank_packagers')
+    ### end Alembic commands ###
--- a/alembic/versions/5ac93692b0ab_initial_revision.py
+++ b/alembic/versions/5ac93692b0ab_initial_revision.py
@@ -0,0 +1,125 @@
+"""Initial revision
+
+Revision ID: 5ac93692b0ab
+Revises: None
+Create Date: 2013-10-26 13:25:27.853595
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '5ac93692b0ab'
+down_revision = None
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+def upgrade():
+    ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('drugbank_manufacturers',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('name', sa.String(), nullable=False),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_table('drugbank_packagers',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('name', sa.String(), nullable=False),
+    sa.Column('url', sa.String(), nullable=True),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_table('drugbank_categories',
+    sa.Column('id', sa.Integer(), nullable=False),
+    sa.Column('name', sa.String(), nullable=False),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_table('fda_products',
+    sa.Column('id', sa.String(), nullable=False),
+    sa.Column('ndc', sa.String(), nullable=False),
+    sa.Column('type', sa.String(), nullable=False),
+    sa.Column('proprietaryName', sa.String(), nullable=False),
+    sa.Column('proprietaryNameSuffix', sa.String(), nullable=True),
+    sa.Column('genericName', sa.String(), nullable=False),
+    sa.Column('marketingCategoryName', sa.String(), nullable=False),
+    sa.Column('labelerName', sa.String(), nullable=False),
+    sa.Column('deaSchedule', sa.String(), nullable=False),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_table('drugbank_drugs',
+    sa.Column('id', sa.String(), nullable=False),
+    sa.Column('name', sa.String(), nullable=False),
+    sa.Column('indication', sa.String(), nullable=False),
+    sa.Column('ndc_id', sa.String(), nullable=True),
+    sa.Column('wikipedia', sa.String(), nullable=True),
+    sa.ForeignKeyConstraint(['ndc_id'], ['fda_products.id'], ),
+    sa.PrimaryKeyConstraint('id'),
+    sa.UniqueConstraint('id')
+    )
+    op.create_table('fda_product_substances',
+    sa.Column('fda_product_id', sa.String(), nullable=False),
+    sa.Column('substanceName', sa.String(), nullable=False),
+    sa.Column('strengthNumber', sa.Float(), nullable=False),
+    sa.Column('strengthUnit', sa.String(), nullable=False),
+    sa.Column('pharmaClasses', postgresql.ARRAY(sa.String()), nullable=False),
+    sa.ForeignKeyConstraint(['fda_product_id'], ['fda_products.id'], ),
+    sa.PrimaryKeyConstraint('fda_product_id')
+    )
+    op.create_table('drugbank_synonyms',
+    sa.Column('drug_id', sa.String(), nullable=False),
+    sa.Column('name', sa.String(), nullable=False),
+    sa.ForeignKeyConstraint(['drug_id'], ['drugbank_drugs.id'], ),
+    sa.PrimaryKeyConstraint('drug_id')
+    )
+    op.create_table('drugbank_packager_maps',
+    sa.Column('drug_id', sa.String(), nullable=False),
+    sa.Column('packager_id', sa.Integer(), nullable=False),
+    sa.ForeignKeyConstraint(['drug_id'], ['drugbank_drugs.id'], ),
+    sa.ForeignKeyConstraint(['packager_id'], ['drugbank_packagers.id'], ),
+    sa.PrimaryKeyConstraint('drug_id')
+    )
+    op.create_table('drugbank_genericnames',
+    sa.Column('drug_id', sa.String(), nullable=False),
+    sa.Column('name', sa.String(), nullable=False),
+    sa.ForeignKeyConstraint(['drug_id'], ['drugbank_drugs.id'], ),
+    sa.PrimaryKeyConstraint('drug_id')
+    )
+    op.create_table('drugbank_prices',
+    sa.Column('drug_id', sa.String(), nullable=False),
+    sa.Column('description', sa.String(), nullable=False),
+    sa.Column('currency', sa.String(), nullable=False),
+    sa.Column('cost', sa.Float(), nullable=False),
+    sa.Column('unit', sa.String(), nullable=False),
+    sa.ForeignKeyConstraint(['drug_id'], ['drugbank_drugs.id'], ),
+    sa.PrimaryKeyConstraint('drug_id')
+    )
+    op.create_table('drugbank_manufacturer_maps',
+    sa.Column('drug_id', sa.String(), nullable=False),
+    sa.Column('manufacturer_id', sa.Integer(), nullable=False),
+    sa.ForeignKeyConstraint(['drug_id'], ['drugbank_drugs.id'], ),
+    sa.ForeignKeyConstraint(['manufacturer_id'], ['drugbank_manufacturers.id'], ),
+    sa.PrimaryKeyConstraint('drug_id')
+    )
+    op.create_table('drugbank_category_maps',
+    sa.Column('drug_id', sa.String(), nullable=False),
+    sa.Column('category_id', sa.Integer(), nullable=False),
+    sa.ForeignKeyConstraint(['category_id'], ['drugbank_categories.id'], ),
+    sa.ForeignKeyConstraint(['drug_id'], ['drugbank_drugs.id'], ),
+    sa.PrimaryKeyConstraint('drug_id')
+    )
+    ### end Alembic commands ###
+
+
+def downgrade():
+    ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('drugbank_category_maps')
+    op.drop_table('drugbank_manufacturer_maps')
+    op.drop_table('drugbank_prices')
+    op.drop_table('drugbank_genericnames')
+    op.drop_table('drugbank_packager_maps')
+    op.drop_table('drugbank_synonyms')
+    op.drop_table('fda_product_substances')
+    op.drop_table('drugbank_drugs')
+    op.drop_table('fda_products')
+    op.drop_table('drugbank_categories')
+    op.drop_table('drugbank_packagers')
+    op.drop_table('drugbank_manufacturers')
+    ### end Alembic commands ###