table builder CLI options

Author: sunsingerus

main.py

@@ -33,11 +33,17 @@ def __init__(self):
 #        mp.set_start_method('forkserver')
 
     def run(self):
-        pumper = Pumper(
-            reader=self.config.reader(),
-            writer=self.config.writer(),
-        )
-        pumper.run()
+        if self.config.is_table_templates():
+            templates = self.config.table_builder().templates()
+            for db in templates:
+                for table in templates[db]:
+                    print(db, ':', table, ':', templates[db][table])
+        else:
+            pumper = Pumper(
+                reader=self.config.reader(),
+                writer=self.config.writer(),
+            )
+            pumper.run()
 
     def start(self):
         if self.config.is_daemon():

src/cliopts.py

@@ -146,6 +146,11 @@ def config():
             action='store_true',
             help='Keep CSV pool files. Useful for debugging'
         )
+        argparser.add_argument(
+            '--table-templates',
+            action='store_true',
+            help='Prepare table templates.'
+        )
 
         argparser.add_argument(
             '--src-server-id',
@@ -262,12 +267,14 @@ def config():
 
         # build options
         return Config ({
+
             'app-config': {
                 'config-file': args.config_file,
                 'log-file': args.log_file,
                 'log-level': CLIOpts.log_level_from_string(args.log_level),
                 'dry': args.dry,
                 'daemon': args.daemon,
+                'table-templates': args.table_templates,
                 'pid_file': args.pid_file,
                 'mempool': args.mempool or args.csvpool, # csvpool assumes mempool to be enabled
                 'mempool-max-events-num': args.mempool_max_events_num,
@@ -284,6 +291,15 @@ def config():
                 },
             },
 
+            'tablebuilder-config': {
+                'host': args.src_host,
+                'port': args.src_port,
+                'user': args.src_user,
+                'password': args.src_password,
+                'dbs': [x for x in args.src_only_schemas.split(',') if x] if args.src_only_schemas else None,
+                'tables': [x for x in args.src_only_tables.split(',') if x] if args.src_only_tables else None,
+            },
+
             'reader-config': {
                 'mysql': {
                     'connection_settings': {

src/config.py

@@ -13,7 +13,7 @@
 
 from .converter.csvwriteconverter import CSVWriteConverter
 from .converter.chwriteconverter import CHWriteConverter
-
+from .tablebuilder import TableBuilder
 
 class Config(object):
 
@@ -46,6 +46,12 @@ def is_daemon(self):
     def is_pool(self):
         return self.config['app-config']['mempool']
 
+    def is_table_templates(self):
+        return self.config['app-config']['table-templates']
+
+    def table_builder(self):
+        return TableBuilder(**self.config['tablebuilder-config'])
+
     def reader(self):
         if self.config['reader-config']['file']['csv_file_path']:
             return CSVReader(**self.config['reader-config']['file'])

src/tablebuilder.py

@@ -10,50 +10,74 @@ class TableBuilder(object):
     connection = None
     cursor = None
 
-    def templates(self, host, user, password=None, db=None, tables=None):
+    host = None
+    port = None
+    user = None
+    password = None
+    dbs = None
+    tables = None
+
+    def __init__(self, host, port, user, password=None, dbs=None, tables=None):
+        self.host = host
+        self.port = port
+        self.user = user
+        self.password = password
+        self.dbs = dbs
+        self.tables = tables
+
+    def templates(self):
         """
         Create templates for specified MySQL tables. In case no tables specified all tables from specified db are templated
 
         :param host: string MySQL host
         :param user: string MySQL user
         :param password: string MySQL password
-        :param db: string MySQL datatabse/ May be omitted, in this case tables has to contain full table names, Ex.: db.table1
-        :param tables: string|list either comma-separated string or list of table names. May be short (in case db specified) or full (in the form db.table, in case no db specified)
+        :param dbs: list of string MySQL datatabse/ May be omitted, in this case tables has to contain full table names, Ex.: db.table1
+        :param tables: list of string list of table names. May be short (in case db specified) or full (in the form db.table, in case no db specified)
         :return: dict of CREATE TABLE () templates
         """
         res = {}
 
+        db = None
+
+        try:
+            db = self.dbs[0]
+        except:
+            pass
+
         # sanity check
-        if db is None and tables is None:
+        if db is None and self.tables is None:
             return res
 
         # MySQL connections
         self.connection = MySQLdb.connect(
-            host=host,
-            user=user,
-            passwd=password,
+            host=self.host,
+            user=self.user,
+            passwd=self.password,
             db=db,
         )
         self.cursor = self.connection.cursor()
 
         # in case to tables specified - list all tables of the DB specified
-        if db is not None and tables is None:
+        if db is not None and self.tables is None:
             self.cursor.execute("USE " + db)
-            tables = []
+            self.tables = []
             self.cursor.execute("SHOW TABLES")  # execute 'SHOW TABLES' (but data is not returned)
             for (table_name,) in self.cursor:
-                tables.append(table_name)
-
-        # tables can be something like 'db1, db2, db3'
-        # make [db1, db2, db3]
-        if isinstance(tables, str):
-            tables = [table.strip() for table in tables.split(',')]
+                self.tables.append(table_name)
 
         # create dict of table templates
-        for table in tables:
-            res[table] = self.create_table_template(table, db)
-
-        # {'table1': 'CREATE TABLE(...)...', 'table2': 'CREATE TABLE(...)...'}
+        for table in self.tables:
+            if not db in res:
+                res[db] = {}
+            res[db][table] = self.create_table_template(table, db)
+
+        # {
+        #   'db': {
+        #       'table1': 'CREATE TABLE(...)...',
+        #       'table2': 'CREATE TABLE(...)...',
+        #   }
+        # }
         return res
 
     def create_table_template(self, table_name, db=None):