|
| 1 | +#!/bin/sh |
| 2 | + |
| 3 | +# Copyright (c) 2023 MIT Laboratory for Computational Physiology |
| 4 | +# Copyright (c) 2021 Thomas Ward <thomas@thomasward.com> |
| 5 | +# |
| 6 | +# Permission is hereby granted, free of charge, to any person obtaining a copy |
| 7 | +# of this software and associated documentation files (the "Software"), to deal |
| 8 | +# in the Software without restriction, including without limitation the rights |
| 9 | +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell |
| 10 | +# copies of the Software, and to permit persons to whom the Software is |
| 11 | +# furnished to do so, subject to the following conditions: |
| 12 | +# |
| 13 | +# The above copyright notice and this permission notice shall be included in all |
| 14 | +# copies or substantial portions of the Software. |
| 15 | +# |
| 16 | +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
| 17 | +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
| 18 | +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE |
| 19 | +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
| 20 | +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
| 21 | +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
| 22 | +# SOFTWARE. |
| 23 | + |
| 24 | +yell () { echo "$0: $*" >&2; } |
| 25 | +die () { yell "$*"; exit 111; } |
| 26 | +try () { "$@" || die "Exiting. Failed to run: \"$*\""; } |
| 27 | + |
| 28 | +usage () { |
| 29 | + die " |
| 30 | +USAGE: ./import_duckdb.sh mimic_data_dir [output_db] |
| 31 | +WHERE: |
| 32 | + mimic_data_dir directory that contains csv.tar.gz or csv files |
| 33 | + output_db: optional filename for duckdb file (default: mimic4_ed.db)\ |
| 34 | +" |
| 35 | +} |
| 36 | + |
| 37 | +# Print help if requested |
| 38 | +echo "$0 $* " | grep -Eq " -h | --help " && usage |
| 39 | + |
| 40 | +# rename CLI positional args to more friendly variable names |
| 41 | +MIMIC_DIR=$1 |
| 42 | +# allow optional specification of duckdb name, otherwise default to mimic4_ed.db |
| 43 | +OUTFILE=mimic4_ed.db |
| 44 | +if [ -n "$2" ]; then |
| 45 | + OUTFILE=$2 |
| 46 | +fi |
| 47 | + |
| 48 | + |
| 49 | +# basic error checking before running |
| 50 | +if [ -z "$MIMIC_DIR" ]; then |
| 51 | + yell "Please specify a mimic data directory" |
| 52 | + die "Usage: ./import_duckdb.sh mimic_data_dir [output_db]" |
| 53 | +elif [ ! -d "$MIMIC_DIR" ]; then |
| 54 | + yell "Specified directory \"$MIMIC_DIR\" does not exist." |
| 55 | + die "Usage: ./import_duckdb.sh mimic_data_dir [output_db]" |
| 56 | +elif [ -n "$3" ]; then |
| 57 | + yell "import.sh takes a maximum of two arguments." |
| 58 | + die "Usage: ./import_duckdb.sh mimic_data_dir [output_db]" |
| 59 | +elif [ -s "$OUTFILE" ]; then |
| 60 | + yell "File \"$OUTFILE\" already exists." |
| 61 | + read -p "Continue? (y/d/n) 'y' continues, 'd' deletes original file, 'n' stops: " yn |
| 62 | + case $yn in |
| 63 | + [Yy]* ) ;; # OK |
| 64 | + [Nn]* ) exit;; |
| 65 | + [Dd]* ) rm "$OUTFILE";; |
| 66 | + * ) die "Unrecognized input.";; |
| 67 | + esac |
| 68 | +fi |
| 69 | + |
| 70 | +# we will copy the postgresql create.sql file, and apply regex |
| 71 | +# to fix the following issues: |
| 72 | +# 1. Remove optional precision value from TIMESTAMP(NN) -> TIMESTAMP |
| 73 | +# duckdb does not support this. |
| 74 | +export REGEX_TIMESTAMP='s/TIMESTAMP\([0-9]+\)/TIMESTAMP/g' |
| 75 | +# 2. Remove NOT NULL constraint from mimiciv_hosp.microbiologyevents.spec_type_desc |
| 76 | +# as there is one (!) zero-length string which is treated as a NULL by the import. |
| 77 | +export REGEX_SPEC_TYPE='s/spec_type_desc(.+)NOT NULL/spec_type_desc\1/g' |
| 78 | +# 3. Remove NOT NULL constraint from mimiciv_hosp.prescriptions.drug |
| 79 | +# as there are zero-length strings which are treated as NULLs by the import. |
| 80 | +export REGEX_DRUG='s/drug +(VARCHAR.+)NOT NULL/drug \1/g' |
| 81 | + |
| 82 | +# use sed + above regex to create tables within db |
| 83 | +sed -r -e "${REGEX_TIMESTAMP}" ../postgres/create.sql | \ |
| 84 | + sed -r -e "${REGEX_SPEC_TYPE}" | \ |
| 85 | + sed -r -e "${REGEX_DRUG}" | \ |
| 86 | + duckdb "$OUTFILE" |
| 87 | + |
| 88 | +# goal: get path from find, e.g., ./1.0/icu/d_items |
| 89 | +# and return database table name for it, e.g., mimiciv_icu.d_items |
| 90 | +make_table_name () { |
| 91 | + # strip leading directories (e.g., ./icu/hello.csv.gz -> hello.csv.gz) |
| 92 | + BASENAME=${1##*/} |
| 93 | + # strip suffix (e.g., hello.csv.gz -> hello; hello.csv -> hello) |
| 94 | + TABLE_NAME=${BASENAME%%.*} |
| 95 | + # strip basename (e.g., ./icu/hello.csv.gz -> ./icu) |
| 96 | + PATHNAME=${1%/*} |
| 97 | + # strip leading directories from PATHNAME (e.g. ./icu -> icu) |
| 98 | + DIRNAME=${PATHNAME##*/} |
| 99 | + TABLE_NAME="mimiciv_$DIRNAME.$TABLE_NAME" |
| 100 | +} |
| 101 | + |
| 102 | + |
| 103 | +# load data into database |
| 104 | +find "$MIMIC_DIR" -type f -name '*.csv???' | sort | while IFS= read -r FILE; do |
| 105 | + make_table_name "$FILE" |
| 106 | + |
| 107 | + # skip directories which we do not expect in mimic-iv-ed |
| 108 | + # avoids syntax errors if mimic-iv in the same dir |
| 109 | + case $DIRNAME in |
| 110 | + (ed) ;; # OK |
| 111 | + (*) continue; |
| 112 | + esac |
| 113 | + echo "Loading $FILE .. \c" |
| 114 | + try duckdb "$OUTFILE" <<-EOSQL |
| 115 | + COPY $TABLE_NAME FROM '$FILE' (HEADER); |
| 116 | +EOSQL |
| 117 | + echo "done!" |
| 118 | +done && echo "Successfully finished loading data into $OUTFILE." |
0 commit comments