WIP: Performance optimized indexes

index.js

@@ -13,7 +13,6 @@ nodeVersionCheck(process.versions.node, packageData.engines.node);
 module.exports = require('./lib');
 module.exports.nodeVersionCheck = nodeVersionCheck;
 module.exports.Node = require('./lib/node');
-module.exports.Transaction = require('./lib/transaction');
 module.exports.Service = require('./lib/service');
 module.exports.errors = require('./lib/errors');
 

lib/services/address/constants.js

@@ -3,9 +3,10 @@
 var exports = {};
 
 exports.PREFIXES = {
-  OUTPUTS: new Buffer('02', 'hex'), // Query outputs by address and/or height
-  SPENTS: new Buffer('03', 'hex'), // Query inputs by address and/or height
-  SPENTSMAP: new Buffer('05', 'hex') // Get the input that spends an output
+  SUMMARY: new Buffer('02', 'hex'), // Query the balance of an address
+  UNSPENT: new Buffer('03', 'hex'), // Query unspent output positions
+  PREV: new Buffer('05', 'hex'), // Query the address and satoshis for an output by txid and output index
+  TXIDS: new Buffer('06', 'hex') // Query all of the txids for an address
 };
 
 exports.MEMPREFIXES = {
@@ -29,6 +30,13 @@ exports.HASH_TYPES_READABLE = {
   '02': 'scripthash'
 };
 
+exports.ADDRESS_KEY_SIZE = 21;
+exports.SPACER_SIZE = 1;
+exports.PREFIX_SIZE = 1;
+exports.HEIGHT_SIZE = 4;
+exports.TXID_SIZE = 32;
+exports.PAGE_SIZE = 10;
+
 exports.HASH_TYPES_MAP = {
   'pubkeyhash': exports.HASH_TYPES.PUBKEY,
   'scripthash': exports.HASH_TYPES.REDEEMSCRIPT
@@ -51,6 +59,8 @@ exports.MAX_HISTORY_QUERY_LENGTH = 100;
 exports.MAX_ADDRESSES_QUERY = 10000;
 // The maximum number of simultaneous requests
 exports.MAX_ADDRESSES_LIMIT = 5;
+// The maximum number of transactions that can be queried with multiple addresses
+exports.MAX_MULTI_HISTORY_COUNT = 1000000;
 
 module.exports = exports;
 

lib/services/address/encoding.js

@@ -9,32 +9,104 @@ var $ = bitcore.util.preconditions;
 
 var exports = {};
 
-exports.encodeSpentIndexSyncKey = function(txidBuffer, outputIndex) {
-  var outputIndexBuffer = new Buffer(4);
+exports.encodeAddressTxIdKey = function(hashBuffer, hashTypeBuffer) {
+  return Buffer.concat([
+    constants.PREFIXES.TXIDS,
+    hashBuffer,
+    hashTypeBuffer
+  ]);
+};
+
+exports.decodeAddressTxIdValue = function(buffer, page, range) {
+  var pos = 0;
+  if (page) {
+    pos = 36 * page * range;
+  }
+  var txids = [];
+  while (pos < buffer.length) {
+    var height = buffer.readUInt32BE(pos);
+    var txid = buffer.slice(pos, pos + 32);
+    txids.push({
+      height: height,
+      txid: txid
+    });
+    pos += 36;
+  }
+  return txids;
+};
+
+exports.encodePrevOutputKey = function(prevTxIdBuffer, outputIndex) {
+  var outputIndexBuffer = new Buffer(new Array(4));
   outputIndexBuffer.writeUInt32BE(outputIndex);
-  var key = Buffer.concat([
-    txidBuffer,
+  return Buffer.concat([
+    constants.PREFIXES.PREV,
+    prevTxIdBuffer,
     outputIndexBuffer
   ]);
-  return key.toString('binary');
 };
 
-exports.encodeMempoolAddressIndexKey = function(hashBuffer, hashTypeBuffer) {
-  var key = Buffer.concat([
+exports.encodePrevOutputValue = function(hashBuffer, hashTypeBuffer, satoshis) {
+  var satoshisBuffer = new Buffer(new Array(8));
+  satoshisBuffer.writeDoubleBE(satoshis);
+  return Buffer.concat([
     hashBuffer,
     hashTypeBuffer,
+    satoshisBuffer
   ]);
-  return key.toString('binary');
 };
 
+exports.encodeSummaryKey = function(hashBuffer, hashTypeBuffer) {
+  return Buffer.concat([
+    constants.PREFIXES.SUMMARY,
+    hashBuffer,
+    hashTypeBuffer
+  ]);
+};
 
-exports.encodeOutputKey = function(hashBuffer, hashTypeBuffer, height, txidBuffer, outputIndex) {
-  var heightBuffer = new Buffer(4);
-  heightBuffer.writeUInt32BE(height);
+/**
+ * This function is optimized to return address information about an output script
+ * without constructing a Bitcore Address instance.
+ * @param {Script} - An instance of a Bitcore Script
+ * @param {Network|String} - The network for the address
+ */
+exports.extractAddressInfoFromScript = function(script, network) {
+  $.checkArgument(network, 'Second argument is expected to be a network');
+  var hashBuffer;
+  var addressType;
+  var hashTypeBuffer;
+  if (script.isPublicKeyHashOut()) {
+    hashBuffer = script.chunks[2].buf;
+    hashTypeBuffer = constants.HASH_TYPES.PUBKEY;
+    addressType = Address.PayToPublicKeyHash;
+  } else if (script.isScriptHashOut()) {
+    hashBuffer = script.chunks[1].buf;
+    hashTypeBuffer = constants.HASH_TYPES.REDEEMSCRIPT;
+    addressType = Address.PayToScriptHash;
+  } else if (script.isPublicKeyOut()) {
+    var pubkey = script.chunks[0].buf;
+    var address = Address.fromPublicKey(new PublicKey(pubkey), network);
+    hashBuffer = address.hashBuffer;
+    hashTypeBuffer = constants.HASH_TYPES.PUBKEY;
+    // pay-to-publickey doesn't have an address, however for compatibility
+    // purposes, we can create an address
+    addressType = Address.PayToPublicKeyHash;
+  } else {
+    return false;
+  }
+  return {
+    hashBuffer: hashBuffer,
+    hashTypeBuffer: hashTypeBuffer,
+    addressType: addressType
+  };
+};
+
+exports.encodeUnspentKey = function(hashBuffer, hashTypeBuffer, height, txidBuffer, outputIndex) {
   var outputIndexBuffer = new Buffer(4);
   outputIndexBuffer.writeUInt32BE(outputIndex);
+  var heightBuffer = new Buffer(4);
+  heightBuffer.writeUInt32BE(height);
   var key = Buffer.concat([
-    constants.PREFIXES.OUTPUTS,
+    constants.PREFIXES.UNSPENT,
     hashBuffer,
     hashTypeBuffer,
     constants.SPACER_MIN,
@@ -53,21 +125,46 @@ exports.decodeOutputKey = function(buffer) {
   var spacer = reader.read(1);
   var height = reader.readUInt32BE();
   var txid = reader.read(32);
-  var outputIndex = reader.readUInt32BE();
   return {
     prefix: prefix,
     hashBuffer: hashBuffer,
     hashTypeBuffer: hashTypeBuffer,
     height: height,
-    txid: txid,
+    txid: txid
+  };
+};
+
+exports.encodeOutputValue = function(outputIndex, satoshis) {
+  var outputIndexBuffer = new Buffer(4);
+  outputIndexBuffer.writeUInt32BE(outputIndex);
+  return outputIndexBuffer;
+};
+
+exports.decodeOutputValue = function(buffer) {
+  var outputIndex = buffer.readUInt32BE(0);
+  return {
     outputIndex: outputIndex
   };
 };
 
-exports.encodeOutputValue = function(satoshis, scriptBuffer) {
-  var satoshisBuffer = new Buffer(8);
-  satoshisBuffer.writeDoubleBE(satoshis);
-  return Buffer.concat([satoshisBuffer, scriptBuffer]);
+////////////////////////////////////////////////////////////////////////////////////
+
+exports.encodeSpentIndexSyncKey = function(txidBuffer, outputIndex) {
+  var outputIndexBuffer = new Buffer(4);
+  outputIndexBuffer.writeUInt32BE(outputIndex);
+  var key = Buffer.concat([
+    txidBuffer,
+    outputIndexBuffer
+  ]);
+  return key.toString('binary');
+};
+
+exports.encodeMempoolAddressIndexKey = function(hashBuffer, hashTypeBuffer) {
+  var key = Buffer.concat([
+    hashBuffer,
+    hashTypeBuffer,
+  ]);
+  return key.toString('binary');
 };
 
 exports.encodeOutputMempoolValue = function(satoshis, timestampBuffer, scriptBuffer) {
@@ -76,15 +173,6 @@ exports.encodeOutputMempoolValue = function(satoshis, timestampBuffer, scriptBuf
   return Buffer.concat([satoshisBuffer, timestampBuffer, scriptBuffer]);
 };
 
-exports.decodeOutputValue = function(buffer) {
-  var satoshis = buffer.readDoubleBE(0);
-  var scriptBuffer = buffer.slice(8, buffer.length);
-  return {
-    satoshis: satoshis,
-    scriptBuffer: scriptBuffer
-  };
-};
-
 exports.decodeOutputMempoolValue = function(buffer) {
   var satoshis = buffer.readDoubleBE(0);
   var timestamp = buffer.readDoubleBE(8);
@@ -259,49 +347,12 @@ exports.decodeSummaryCacheValue = function(buffer) {
 exports.getAddressInfo = function(addressStr) {
   var addrObj = bitcore.Address(addressStr);
   var hashTypeBuffer = constants.HASH_TYPES_MAP[addrObj.type];
-  
+
   return {
     hashBuffer: addrObj.hashBuffer,
     hashTypeBuffer: hashTypeBuffer,
     hashTypeReadable: addrObj.type
   };
 };
 
-/**
- * This function is optimized to return address information about an output script
- * without constructing a Bitcore Address instance.
- * @param {Script} - An instance of a Bitcore Script
- * @param {Network|String} - The network for the address
- */
-exports.extractAddressInfoFromScript = function(script, network) {
-  $.checkArgument(network, 'Second argument is expected to be a network');
-  var hashBuffer;
-  var addressType;
-  var hashTypeBuffer;
-  if (script.isPublicKeyHashOut()) {
-    hashBuffer = script.chunks[2].buf;
-    hashTypeBuffer = constants.HASH_TYPES.PUBKEY;
-    addressType = Address.PayToPublicKeyHash;
-  } else if (script.isScriptHashOut()) {
-    hashBuffer = script.chunks[1].buf;
-    hashTypeBuffer = constants.HASH_TYPES.REDEEMSCRIPT;
-    addressType = Address.PayToScriptHash;
-  } else if (script.isPublicKeyOut()) {
-    var pubkey = script.chunks[0].buf;
-    var address = Address.fromPublicKey(new PublicKey(pubkey), network);
-    hashBuffer = address.hashBuffer;
-    hashTypeBuffer = constants.HASH_TYPES.PUBKEY;
-    // pay-to-publickey doesn't have an address, however for compatibility
-    // purposes, we can create an address
-    addressType = Address.PayToPublicKeyHash;
-  } else {
-    return false;
-  }
-  return {
-    hashBuffer: hashBuffer,
-    hashTypeBuffer: hashTypeBuffer,
-    addressType: addressType
-  };
-};
-
 module.exports = exports;