From b6b978da2047232f5ff678b455113f9aa75c52a5 Mon Sep 17 00:00:00 2001 From: Shantanu Singh Date: Thu, 14 Dec 2017 17:45:19 -0500 Subject: [PATCH 1/3] Add option to skip image header --- cytominer_database/commands/command_ingest.py | 15 +++++++++++++-- cytominer_database/ingest.py | 14 ++++++++++---- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/cytominer_database/commands/command_ingest.py b/cytominer_database/commands/command_ingest.py index 32513fd..93e670e 100644 --- a/cytominer_database/commands/command_ingest.py +++ b/cytominer_database/commands/command_ingest.py @@ -44,7 +44,17 @@ the CSV will be split into one CSV per compartment. """ ) -def command(source, target, config_file, munge): +@click.option( + "--skip-image-prefix/--no-skip-image-prefix", + default=False, + help="""\ +True if the prefix of image table name should be +excluded from the names of columns from per image +table e.g. use `Metadata_Plate` instead of +`Image_Metadata_Plate`. +""" +) +def command(source, target, config_file, munge, skip_image_prefix): config = configparser.ConfigParser() with open(config_file, "r") as config_fd: @@ -53,4 +63,5 @@ def command(source, target, config_file, munge): if munge: cytominer_database.munge.munge(config=config, source=source) - cytominer_database.ingest.seed(source, target, config) + cytominer_database.ingest.seed(source, target, config, + skip_image_prefix) diff --git a/cytominer_database/ingest.py b/cytominer_database/ingest.py index b778c14..7947a51 100644 --- a/cytominer_database/ingest.py +++ b/cytominer_database/ingest.py @@ -63,13 +63,15 @@ def __format__(name, header): return "{}_{}".format(name, header) -def into(input, output, name, identifier): +def into(input, output, name, identifier, skip_table_prefix=False): """Ingest a CSV file into a table in a database. :param input: Input CSV file. :param output: Connection string for the database. :param name: Table in database into which the CSV file will be ingested :param identifier: Unique identifier for ``input``. + :param skip_table_prefix: True if the prefix of the table name should be excluded + from the names of columns. """ with backports.tempfile.TemporaryDirectory() as directory: @@ -80,7 +82,8 @@ def into(input, output, name, identifier): writer = csv.writer(fout) headers = next(reader) - headers = [__format__(name, header) for header in headers] + if not skip_table_prefix: + headers = [__format__(name, header) for header in headers] headers = ["TableNumber"] + headers writer.writerow(headers) @@ -97,13 +100,15 @@ def into(input, output, name, identifier): odo.odo(source, "{}::{}".format(output, name), has_header=True, delimiter=",") -def seed(source, target, config): +def seed(source, target, config, skip_image_prefix): """ Read CSV files into a database backend. :param config: Configuration file. :param source: Directory containing subdirectories that contain CSV files. :param target: Connection string for the database. + :param skip_image_prefix: True if the prefix of image table name should be excluded + from the names of columns from per image table """ directories = sorted(list(cytominer_database.utils.find_directories(source))) @@ -122,7 +127,8 @@ def seed(source, target, config): name, _ = os.path.splitext(config["filenames"]["image"]) try: - into(input=image, output=target, name=name.capitalize(), identifier=identifier) + into(input=image, output=target, name=name.capitalize(), identifier=identifier, + skip_table_prefix = skip_image_prefix) except sqlalchemy.exc.DatabaseError as e: click.echo(e) From 4f55933070a8c12f773eea2f3fe0609e18989ee8 Mon Sep 17 00:00:00 2001 From: Shantanu Singh Date: Thu, 14 Dec 2017 18:10:00 -0500 Subject: [PATCH 2/3] Missing default for new param --- cytominer_database/ingest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cytominer_database/ingest.py b/cytominer_database/ingest.py index 7947a51..2ba9a4a 100644 --- a/cytominer_database/ingest.py +++ b/cytominer_database/ingest.py @@ -100,7 +100,7 @@ def into(input, output, name, identifier, skip_table_prefix=False): odo.odo(source, "{}::{}".format(output, name), has_header=True, delimiter=",") -def seed(source, target, config, skip_image_prefix): +def seed(source, target, config, skip_image_prefix=False): """ Read CSV files into a database backend. From 36c36e5fd5f9b326f1ee9ffc3c7eeb2b55239657 Mon Sep 17 00:00:00 2001 From: Shantanu Singh Date: Thu, 14 Dec 2017 20:38:02 -0500 Subject: [PATCH 3/3] skip image prefix should be true by default --- cytominer_database/commands/command_ingest.py | 2 +- cytominer_database/ingest.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cytominer_database/commands/command_ingest.py b/cytominer_database/commands/command_ingest.py index 93e670e..1442d3a 100644 --- a/cytominer_database/commands/command_ingest.py +++ b/cytominer_database/commands/command_ingest.py @@ -46,7 +46,7 @@ ) @click.option( "--skip-image-prefix/--no-skip-image-prefix", - default=False, + default=True, help="""\ True if the prefix of image table name should be excluded from the names of columns from per image diff --git a/cytominer_database/ingest.py b/cytominer_database/ingest.py index 2ba9a4a..c46e59a 100644 --- a/cytominer_database/ingest.py +++ b/cytominer_database/ingest.py @@ -100,7 +100,7 @@ def into(input, output, name, identifier, skip_table_prefix=False): odo.odo(source, "{}::{}".format(output, name), has_header=True, delimiter=",") -def seed(source, target, config, skip_image_prefix=False): +def seed(source, target, config, skip_image_prefix=True): """ Read CSV files into a database backend.