diff --git a/run_airline_ontime_data_mysql_to_ch_migration.sh b/run_airline_ontime_data_mysql_to_ch_migration.sh new file mode 100755 index 0000000..fb29296 --- /dev/null +++ b/run_airline_ontime_data_mysql_to_ch_migration.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# migrate data from MySQL to ClickHouse + +# put csv files in this dir +CSV_FILES_DIR="/var/lib/mysql-files" + +# dump CSV files into CSV_FILES_DIR +sudo mysqldump \ + -u root \ + --tz-utc \ + --quick \ + --fields-terminated-by=, \ + --fields-optionally-enclosed-by=\" \ + --fields-escaped-by=\\ \ + --tab="$CSV_FILES_DIR"/ \ + airline ontime + +# replay CSV files from CSV_FILES_DIR +sudo cat "$CSV_FILES_DIR"/ontime.txt | clickhouse-client --query="INSERT INTO airline.ontime FORMAT CSV" diff --git a/src/tablebuilder.py b/src/tablebuilder.py index 4db7d90..f32990a 100644 --- a/src/tablebuilder.py +++ b/src/tablebuilder.py @@ -264,19 +264,19 @@ def map_type(self, mysql_type, nullable=False): ch_type = 'String' elif mysql_type.startswith('TINYINT'): ch_type = 'UInt8' if mysql_type.endswith('UNSIGNED') else 'Int8' - elif mysql_type.startswith('BOOL') or mysql_type.startswith('BOOLEAN'): + elif mysql_type.startswith('BOOLEAN') or mysql_type.startswith('BOOL'): ch_type = 'UInt8' elif mysql_type.startswith('SMALLINT'): ch_type = 'UInt16' if mysql_type.endswith('UNSIGNED') else 'Int16' elif mysql_type.startswith('MEDIUMINT'): ch_type = 'UInt32' if mysql_type.endswith('UNSIGNED') else 'Int32' - elif mysql_type.startswith('INT') or mysql_type.startswith('INTEGER'): + elif mysql_type.startswith('INTEGER') or mysql_type.startswith('INT'): ch_type = 'UInt32' if mysql_type.endswith('UNSIGNED') else 'Int32' elif mysql_type.startswith('BIGINT'): ch_type = 'UInt64' if mysql_type.endswith('UNSIGNED') else 'Int64' elif mysql_type.startswith('SERIAL'): ch_type = 'UInt64' - elif mysql_type.startswith('DEC') or mysql_type.startswith('DECIMAL') or mysql_type.startswith('FIXED') or mysql_type.startswith('NUMERIC'): + elif mysql_type.startswith('DECIMAL') or mysql_type.startswith('DEC') or mysql_type.startswith('FIXED') or mysql_type.startswith('NUMERIC'): ch_type = 'String' elif mysql_type.startswith('FLOAT'): ch_type = 'Float32' @@ -284,10 +284,10 @@ def map_type(self, mysql_type, nullable=False): ch_type = 'Float64' # Date and Time Types - elif mysql_type.startswith('DATE'): - ch_type = 'Date' elif mysql_type.startswith('DATETIME'): ch_type = 'DateTime' + elif mysql_type.startswith('DATE'): + ch_type = 'Date' elif mysql_type.startswith('TIMESTAMP'): ch_type = 'DateTime' elif mysql_type.startswith('TIME'):