Skip to content

Commit

Permalink
Removed the schemas for tables data_models and datasets.
Browse files Browse the repository at this point in the history
  • Loading branch information
@KFilippopolitis
KFilippopolitis committed Jun 17, 2024
1 parent 1d402b0 commit fd6c317
Show file tree
Hide file tree
Showing 9 changed files with 207 additions and 228 deletions.
31 changes: 15 additions & 16 deletions mipdb/database.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@

from mipdb.exceptions import DataBaseError

METADATA_SCHEMA = "mipdb_metadata"
METADATA_TABLE = "variables_metadata"

CONFIG = "/home/config.toml"
Expand Down Expand Up @@ -322,15 +321,15 @@ def drop_schema(self, schema_name):

def get_data_model_status(self, data_model_id):
select = sql.text(
f"SELECT status FROM {METADATA_SCHEMA}.data_models "
f"SELECT status FROM data_models "
"WHERE data_model_id = :data_model_id "
)
(status, *_), *_ = self.execute(select, data_model_id=data_model_id)
return status

def update_data_model_status(self, status, data_model_id):
update = sql.text(
f"UPDATE {METADATA_SCHEMA}.data_models "
f"UPDATE data_models "
"SET status = :status "
"WHERE data_model_id = :data_model_id "
"AND status <> :status"
Expand All @@ -339,7 +338,7 @@ def update_data_model_status(self, status, data_model_id):

def get_dataset_status(self, dataset_id):
select = sql.text(
f"SELECT status FROM {METADATA_SCHEMA}.datasets "
f"SELECT status FROM datasets "
"WHERE dataset_id = :dataset_id "
)
(status, *_), *_ = self.execute(select, dataset_id=dataset_id)
Expand All @@ -355,7 +354,7 @@ def get_metadata(self, schema):

def update_dataset_status(self, status, dataset_id):
update = sql.text(
f"UPDATE {METADATA_SCHEMA}.datasets "
f"UPDATE datasets "
"SET status = :status "
"WHERE dataset_id = :dataset_id "
"AND status <> :status"
Expand All @@ -371,7 +370,7 @@ def get_data_model_id(self, code, version):
# implemented.
select = sql.text(
"SELECT data_model_id "
f"FROM {METADATA_SCHEMA}.data_models "
f"FROM data_models "
"WHERE code = :code "
"AND version = :version "
)
Expand All @@ -390,7 +389,7 @@ def get_data_model_id(self, code, version):
def get_dataset_id(self, code, data_model_id):
select = sql.text(
"SELECT dataset_id "
f"FROM {METADATA_SCHEMA}.datasets "
f"FROM datasets "
"WHERE code = :code "
"AND data_model_id = :data_model_id "
)
Expand All @@ -413,7 +412,7 @@ def get_dataset_count_by_data_model_id(self):
res = self.execute(
f"""
SELECT data_model_id, COUNT(data_model_id) as count
FROM {METADATA_SCHEMA}.datasets
FROM datasets
GROUP BY data_model_id
"""
)
Expand Down Expand Up @@ -446,7 +445,7 @@ def get_dataset(self, dataset_id, columns):
dataset = self.execute(
f"""
SELECT {columns_query}
FROM {METADATA_SCHEMA}.datasets
FROM datasets
WHERE dataset_id = {dataset_id}
LIMIT 1
"""
Expand All @@ -459,7 +458,7 @@ def get_data_model(self, data_model_id, columns):
data_model = self.execute(
f"""
SELECT {columns_query}
FROM {METADATA_SCHEMA}.data_models
FROM data_models
WHERE data_model_id = {data_model_id}
LIMIT 1
"""
Expand All @@ -471,7 +470,7 @@ def get_data_models(self, columns=None):
data_models = self.execute(
f"""
SELECT {columns_query}
FROM {METADATA_SCHEMA}.data_models
FROM data_models
"""
)

Expand All @@ -485,7 +484,7 @@ def get_values(self, data_model_id=None, columns=None):
datasets = self.execute(
f"""
SELECT {columns_query}
FROM {METADATA_SCHEMA}.datasets {data_model_id_clause}
FROM datasets {data_model_id_clause}
"""
)
return list(datasets)
Expand All @@ -507,25 +506,25 @@ def grant_select_access_rights(self, table, user):

def get_dataset_properties(self, dataset_id):
(properties, *_), *_ = self.execute(
f"SELECT properties FROM {METADATA_SCHEMA}.datasets WHERE dataset_id = {dataset_id}"
f"SELECT properties FROM datasets WHERE dataset_id = {dataset_id}"
)
return properties

def get_data_model_properties(self, data_model_id):
(properties, *_), *_ = self.execute(
f"SELECT properties FROM {METADATA_SCHEMA}.data_models WHERE data_model_id = {data_model_id}"
f"SELECT properties FROM data_models WHERE data_model_id = {data_model_id}"
)
return properties

def set_data_model_properties(self, properties, data_model_id):
properties_monetized = monetize.convert(properties)
query = f"""UPDATE {METADATA_SCHEMA}.data_models SET properties = {properties_monetized}
query = f"""UPDATE data_models SET properties = {properties_monetized}
WHERE data_model_id = {data_model_id}"""
self.execute(query)

def set_dataset_properties(self, properties, dataset_id):
properties_monetized = monetize.convert(properties)
query = f"""UPDATE {METADATA_SCHEMA}.datasets SET properties = {properties_monetized}
query = f"""UPDATE datasets SET properties = {properties_monetized}
WHERE dataset_id = {dataset_id}"""
self.execute(query)

Expand Down
34 changes: 18 additions & 16 deletions mipdb/tables.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,12 @@
from typing import Union, List

import sqlalchemy as sql
from sqlalchemy import ForeignKey, Integer, MetaData
from sqlalchemy import ForeignKey, MetaData
from sqlalchemy.ext.compiler import compiles

from mipdb.database import DataBase, Connection, credentials_from_config
from mipdb.database import credentials_from_config
from mipdb.data_frame import DATASET_COLUMN_NAME
from mipdb.database import DataBase, Connection
from mipdb.database import METADATA_SCHEMA
from mipdb.database import METADATA_TABLE
from mipdb.dataelements import CommonDataElement
from mipdb.exceptions import UserInputError
Expand Down Expand Up @@ -78,14 +77,16 @@ def drop(self, db: Union[DataBase, Connection]):
db.drop_table(self._table)


metadata = MetaData()

class DataModelTable(Table):
def __init__(self, schema):
def __init__(self):
self.data_model_id_seq = sql.Sequence(
"data_model_id_seq", metadata=schema.schema
"data_model_id_seq", metadata=metadata
)
self._table = sql.Table(
"data_models",
schema.schema,
metadata,
sql.Column(
"data_model_id",
SQLTYPES.INTEGER,
Expand All @@ -97,6 +98,7 @@ def __init__(self, schema):
sql.Column("label", SQLTYPES.STRING),
sql.Column("status", SQLTYPES.STRING, nullable=False),
sql.Column("properties", SQLTYPES.JSON),
extend_existing=True
)

def drop_sequence(self, db: Union[DataBase, Connection]):
Expand All @@ -107,15 +109,15 @@ def get_data_models(self, db, columns: list = None):
if columns and not set(columns).issubset(self.table.columns.keys()):
non_existing_columns = list(set(columns) - set(self.table.columns.keys()))
raise ValueError(
f"The columns: {non_existing_columns} do not exist in the data models schema"
f"The columns: {non_existing_columns} do not exist in the data models table"
)
return db.get_data_models(columns)

def get_data_model(self, data_model_id, db, columns: list = None):
if columns and not set(columns).issubset(self.table.columns.keys()):
non_existing_columns = list(set(columns) - set(self.table.columns.keys()))
raise ValueError(
f"The columns: {non_existing_columns} do not exist in the data model's schema"
f"The columns: {non_existing_columns} do not exist in the data model table"
)
return db.get_data_model(data_model_id, columns)

Expand All @@ -139,7 +141,7 @@ def set_data_model_status(self, status, data_model_id, db):

def delete_data_model(self, code, version, db):
delete = sql.text(
f"DELETE FROM {METADATA_SCHEMA}.data_models "
f"DELETE FROM data_models "
"WHERE code = :code "
"AND version = :version "
)
Expand All @@ -148,13 +150,12 @@ def delete_data_model(self, code, version, db):
def get_next_data_model_id(self, db):
return db.execute(self.data_model_id_seq)


class DatasetsTable(Table):
def __init__(self, schema):
self.dataset_id_seq = sql.Sequence("dataset_id_seq", metadata=schema.schema)
def __init__(self):
self.dataset_id_seq = sql.Sequence("dataset_id_seq", metadata=metadata)
self._table = sql.Table(
"datasets",
schema.schema,
metadata,
sql.Column(
"dataset_id",
SQLTYPES.INTEGER,
Expand All @@ -172,6 +173,7 @@ def __init__(self, schema):
sql.Column("status", SQLTYPES.STRING, nullable=False),
sql.Column("csv_path", SQLTYPES.STRING, nullable=False),
sql.Column("properties", SQLTYPES.JSON),
extend_existing=True
)

def drop_sequence(self, db: Union[DataBase, Connection]):
Expand All @@ -182,7 +184,7 @@ def get_values(self, db, data_model_id=None, columns=None):
if columns and not set(columns).issubset(self.table.columns.keys()):
non_existing_columns = list(set(columns) - set(self.table.columns.keys()))
raise ValueError(
f"The columns: {non_existing_columns} do not exist in the datasets schema"
f"The columns: {non_existing_columns} do not exist in the datasets table"
)
datasets = db.get_values(data_model_id, columns)
if columns and len(columns) == 1:
Expand All @@ -193,7 +195,7 @@ def get_dataset(self, db, dataset_id=None, columns=None):
if columns and not set(columns).issubset(self.table.columns.keys()):
non_existing_columns = list(set(columns) - set(self.table.columns.keys()))
raise ValueError(
f"The columns: {non_existing_columns} do not exist in the datasets schema"
f"The columns: {non_existing_columns} do not exist in the datasets table"
)
return db.get_dataset(dataset_id, columns)

Expand All @@ -208,7 +210,7 @@ def set_dataset_properties(self, properties, dataset_id, db):

def delete_dataset(self, dataset_id, data_model_id, db):
delete = sql.text(
f"DELETE FROM {METADATA_SCHEMA}.datasets "
f"DELETE FROM datasets "
"WHERE dataset_id = :dataset_id "
"AND data_model_id = :data_model_id "
)
Expand Down
Loading

0 comments on commit fd6c317

Please sign in to comment.