refactor sqlthread and add create test cases

Author: cis-muzahid

setup.py

@@ -51,7 +51,7 @@ def run(self):
 if __name__ == "__main__":
     here = os.path.abspath(os.path.dirname(__file__))
     with open(os.path.join(here, 'README.md')) as f:
-        README = f.read()
+        README. = f.read()
 
     with open(os.path.join(here, 'requirements.txt'), 'r') as f:
         requirements = list(f.readlines())

src/class_sqlThread.py

@@ -0,0 +1 @@
+ALTER TABLE pubkeys ADD usedpersonally text DEFAULT 'no';

src/sql/config_setting_ver_2.sql

@@ -0,0 +1,5 @@
+ALTER TABLE inbox ADD encodingtype int DEFAULT '2';
+
+ALTER TABLE inbox ADD read bool DEFAULT '1';
+
+ALTER TABLE sent ADD encodingtype int DEFAULT '2';

src/sql/config_setting_ver_3.sql

@@ -0,0 +1,15 @@
+-- --
+-- --  Update the address colunm to unique in addressbook table
+-- --
+
+ALTER TABLE addressbook RENAME TO old_addressbook;
+
+CREATE TABLE `addressbook` (
+    `label` text ,
+    `address` text ,
+    UNIQUE(address) ON CONFLICT IGNORE
+) ;
+
+INSERT INTO addressbook SELECT label, address FROM old_addressbook;
+
+DROP TABLE old_addressbook;

src/sql/init_version_10.sql

@@ -0,0 +1,29 @@
+--
+-- Let's get rid of the first20bytesofencryptedmessage field in the inventory table.
+--
+
+CREATE TEMP TABLE `inventory_backup` (
+    `hash` blob ,
+    `objecttype` text ,
+    `streamnumber` int ,
+    `payload` blob ,
+    `receivedtime` int ,
+    UNIQUE(hash) ON CONFLICT REPLACE
+) ;
+
+INSERT INTO `inventory_backup` SELECT  hash, objecttype, streamnumber, payload, receivedtime FROM inventory;
+
+DROP TABLE inventory;
+
+CREATE TABLE `inventory` (
+    `hash` blob ,
+    `objecttype` text ,
+    `streamnumber` int ,
+    `payload` blob ,
+    `receivedtime` int ,
+    UNIQUE(hash) ON CONFLICT REPLACE
+) ;
+
+INSERT INTO inventory SELECT hash, objecttype, streamnumber, payload, receivedtime FROM inventory_backup;
+
+DROP TABLE inventory_backup;

src/sql/init_version_2.sql

@@ -0,0 +1,5 @@
+--
+-- Add a new column to the inventory table to store tags.
+--
+
+ALTER TABLE inventory ADD tag blob DEFAULT '';

src/sql/init_version_3.sql

@@ -0,0 +1,17 @@
+ --
+ -- Add a new column to the pubkeys table to store the address version.
+ -- We're going to trash all of our pubkeys and let them be redownloaded.
+ --
+
+DROP TABLE pubkeys;
+
+CREATE TABLE `pubkeys` (
+    `hash` blob ,
+    `addressversion` int ,
+    `transmitdata` blob ,
+    `time` int ,
+    `usedpersonally` text ,
+    UNIQUE(hash, addressversion) ON CONFLICT REPLACE
+) ;
+
+DELETE FROM inventory WHERE objecttype = 'pubkey';

src/sql/init_version_4.sql

@@ -0,0 +1,12 @@
+ --
+ -- Add a new table: objectprocessorqueue with which to hold objects
+ -- that have yet to be processed if the user shuts down Bitmessage.
+ --
+
+DROP TABLE knownnodes;
+
+CREATE TABLE `objectprocessorqueue` (
+    `objecttype` text,
+    `data` blob,
+UNIQUE(objecttype, data) ON CONFLICT REPLACE
+) ;

src/sql/init_version_5.sql

@@ -0,0 +1,25 @@
+--
+-- changes related to protocol v3
+-- In table inventory and objectprocessorqueue, objecttype is now
+-- an integer (it was a human-friendly string previously)
+--
+
+DROP TABLE inventory;
+
+CREATE TABLE `inventory` (
+    `hash` blob,
+    `objecttype` int,
+    `streamnumber` int,
+    `payload` blob,
+    `expirestime` integer,
+    `tag` blob,
+    UNIQUE(hash) ON CONFLICT REPLACE
+) ;
+
+DROP TABLE objectprocessorqueue;
+
+CREATE TABLE `objectprocessorqueue` (
+    `objecttype` int,
+    `data` blob,
+    UNIQUE(objecttype, data) ON CONFLICT REPLACE
+) ;

src/sql/init_version_6.sql

@@ -0,0 +1,11 @@
+--
+-- The format of data stored in the pubkeys table has changed. Let's
+-- clear it, and the pubkeys from inventory, so that they'll
+-- be re-downloaded.
+--
+
+DELETE FROM inventory WHERE objecttype = 1;
+
+DELETE FROM pubkeys;
+
+UPDATE sent SET status='msgqueued' WHERE status='doingmsgpow' or status='badkey';

src/sql/init_version_7.sql

@@ -0,0 +1,7 @@
+--
+-- Add a new column to the inbox table to store the hash of
+-- the message signature. We'll use this as temporary message UUID
+-- in order to detect duplicates.
+--
+
+ALTER TABLE inbox ADD sighash blob DEFAULT '';

src/sql/init_version_8.sql

@@ -0,0 +1,74 @@
+CREATE TEMPORARY TABLE `sent_backup` (
+    `msgid` blob,
+    `toaddress` text,
+    `toripe` blob,
+    `fromaddress` text,
+    `subject` text,
+    `message` text,
+    `ackdata` blob,
+    `lastactiontime` integer,
+    `status` text,
+    `retrynumber` integer,
+    `folder` text,
+    `encodingtype` int
+) ;
+
+INSERT INTO sent_backup SELECT msgid, toaddress, toripe, fromaddress, subject, message, ackdata, lastactiontime, status, 0, folder, encodingtype FROM sent;
+
+DROP TABLE sent;
+
+CREATE TABLE `sent` (
+    `msgid` blob,
+    `toaddress` text,
+    `toripe` blob,
+    `fromaddress` text,
+    `subject` text,
+    `message` text,
+    `ackdata` blob,
+    `senttime` integer,
+    `lastactiontime` integer,
+    `sleeptill` int,
+    `status` text,
+    `retrynumber` integer,
+    `folder` text,
+    `encodingtype` int,
+    `ttl` int
+) ;
+
+INSERT INTO sent SELECT msgid, toaddress, toripe, fromaddress, subject, message, ackdata, lastactiontime, lastactiontime, 0, status, 0, folder, encodingtype, 216000 FROM sent_backup;
+
+DROP TABLE sent_backup;
+
+ALTER TABLE pubkeys ADD address text DEFAULT '' ;
+
+--
+-- replica for loop to update hashed address
+--
+
+UPDATE pubkeys SET address=(enaddr(pubkeys.addressversion, 1, hash));
+
+CREATE TEMPORARY TABLE `pubkeys_backup` (
+    `address` text,
+    `addressversion` int,
+    `transmitdata` blob,
+    `time` int,
+    `usedpersonally` text,
+    UNIQUE(address) ON CONFLICT REPLACE
+) ;
+
+INSERT INTO pubkeys_backup SELECT address, addressversion, transmitdata, `time`, usedpersonally FROM pubkeys;
+
+DROP TABLE pubkeys;
+
+CREATE TABLE `pubkeys` (
+    `address` text,
+    `addressversion` int,
+    `transmitdata` blob,
+    `time` int,
+    `usedpersonally` text,
+    UNIQUE(address) ON CONFLICT REPLACE
+) ;
+
+INSERT INTO pubkeys SELECT address, addressversion, transmitdata, `time`, usedpersonally FROM pubkeys_backup;
+
+DROP TABLE pubkeys_backup;

src/sql/init_version_9.sql

@@ -0,0 +1,103 @@
+CREATE TABLE `inbox` (
+   `msgid` blob,
+   `toaddress` text,
+   `fromaddress` text,
+   `subject` text,
+   `received` text,
+   `message` text,
+   `folder` text,
+   `encodingtype` int,
+   `read` bool,
+   `sighash` blob,
+UNIQUE(msgid) ON CONFLICT REPLACE
+) ;
+
+CREATE TABLE `sent` (
+   `msgid` blob,
+   `toaddress` text,
+   `toripe` blob,
+   `fromaddress` text,
+   `subject` text,
+   `message` text,
+   `ackdata` blob,
+   `senttime` integer,
+   `lastactiontime` integer,
+   `sleeptill` integer,
+   `status` text,
+   `retrynumber` integer,
+   `folder` text,
+   `encodingtype` int,
+   `ttl` int
+) ;
+
+
+CREATE TABLE `subscriptions` (
+   `label` text,
+   `address` text,
+   `enabled` bool
+) ;
+
+
+CREATE TABLE `addressbook` (
+   `label` text,
+   `address` text,
+   UNIQUE(address) ON CONFLICT IGNORE
+) ;
+
+
+ CREATE TABLE `blacklist` (
+    `label` text,
+    `address` text,
+    `enabled` bool
+ ) ;
+
+
+ CREATE TABLE `whitelist` (
+    `label` text,
+    `address` text,
+    `enabled` bool
+ ) ;
+
+
+CREATE TABLE `pubkeys` (
+    `address` text,
+    `addressversion` int,
+    `transmitdata` blob,
+    `time` int,
+    `usedpersonally` text,
+    UNIQUE(address) ON CONFLICT REPLACE
+) ;
+
+
+CREATE TABLE `inventory` (
+    `hash` blob,
+    `objecttype` int,
+    `streamnumber` int,
+    `payload` blob,
+    `expirestime` integer,
+    `tag` blob,
+    UNIQUE(hash) ON CONFLICT REPLACE
+) ;
+
+
+INSERT INTO subscriptions VALUES ('Bitmessage new releases/announcements', 'BM-GtovgYdgs7qXPkoYaRgrLFuFKz1SFpsw', 1);
+
+
+ CREATE TABLE `settings` (
+    `key` blob,
+    `value` blob,
+    UNIQUE(key) ON CONFLICT REPLACE
+ ) ;
+
+INSERT INTO settings VALUES('version','11');
+
+INSERT INTO settings VALUES('lastvacuumtime', CAST(strftime('%s', 'now') AS STR) );
+
+CREATE TABLE `objectprocessorqueue` (
+    `objecttype` int,
+    `data` blob,
+    UNIQUE(objecttype, data) ON CONFLICT REPLACE
+) ;
+
+
+

src/sql/initialize_schema.sql

@@ -0,0 +1,30 @@
+CREATE TEMPORARY TABLE `pubkeys_backup` (
+    `hash` blob,
+    `transmitdata` blob,
+    `time` int,
+    `usedpersonally` text,
+    UNIQUE(hash) ON CONFLICT REPLACE
+) ;
+
+INSERT INTO `pubkeys_backup` SELECT hash, transmitdata, `time`, usedpersonally FROM `pubkeys`;
+
+DROP TABLE `pubkeys`
+
+CREATE TABLE `pubkeys` (
+    `hash` blob,
+    `transmitdata` blob,
+    `time` int,
+    `usedpersonally` text,
+    UNIQUE(hash) ON CONFLICT REPLACE
+) ;
+
+
+INSERT INTO `pubkeys` SELECT hash, transmitdata, `time`, usedpersonally FROM `pubkeys_backup`;
+
+DROP TABLE `pubkeys_backup`;
+
+DELETE FROM inventory WHERE objecttype = 'pubkey';
+
+DELETE FROM subscriptions WHERE address='BM-BbkPSZbzPwpVcYZpU4yHwf9ZPEapN5Zx'
+
+INSERT INTO subscriptions VALUES('Bitmessage new releases/announcements','BM-GtovgYdgs7qXPkoYaRgrLFuFKz1SFpsw',1)

src/sql/upg_sc_if_old_ver_1.sql

@@ -0,0 +1,7 @@
+UPDATE `sent` SET status='doingmsgpow' WHERE status='doingpow';
+
+UPDATE `sent` SET status='msgsent' WHERE status='sentmessage';
+
+UPDATE `sent` SET status='doingpubkeypow' WHERE status='findingpubkey';
+
+UPDATE `sent` SET status='broadcastqueued' WHERE status='broadcastpending';

src/sql/upg_sc_if_old_ver_2.sql

@@ -23,7 +23,7 @@
 
 from bmconfigparser import config
 from helper_msgcoding import MsgEncode, MsgDecode
-from helper_sql import sqlQuery
+from helper_sql import sqlQuery, sqlExecute
 from network import asyncore_pollchoose as asyncore, knownnodes
 from network.bmproto import BMProto
 from network.connectionpool import BMConnectionPool
@@ -409,6 +409,22 @@ def test_adding_two_same_case_sensitive_addresses(self):
         self.delete_address_from_addressbook(address1)
         self.delete_address_from_addressbook(address2)
 
+    def test_sqlscripts(self):
+        """ Test sql statements"""
+
+        sqlExecute('create table if not exists testtbl (id integer)')
+        tables = list(sqlQuery("select name from sqlite_master where type is 'table'"))
+        res = [item for item in tables if 'testtbl' in item]
+        self.assertEqual(res[0][0], 'testtbl')
+
+        queryreturn = sqlExecute("INSERT INTO testtbl VALUES(101);")
+        self.assertEqual(queryreturn, 1)
+
+        queryreturn = sqlQuery('''SELECT * FROM testtbl''')
+        self.assertEqual(queryreturn[0][0], 101)
+
+        sqlQuery("DROP TABLE testtbl")
+
 
 def run():
     """Starts all tests defined in this module"""

src/tests/core.py

@@ -0,0 +1 @@
+INSERT INTO `addressbook` VALUES ('test', "BM-2cWzMnxjJ7yRP3nLEWUV5LisTZyREWSxYz"), ('testone', "BM-2cWzMnxjJ7yRP3nLEWUV5LisTZyREWSxYz");