Sub client hangs on reconnect when Redis close subscribe connection

[TatianaSudakova]

Hi,

To reproduce this problem we need to run:

• one subscribe client which listens to 10000 channels
• twenty publish clients which publish to that 10000 channels quite a big messages

subscribe.js

let redis = require('redis');

let sub = redis.createClient(6379, 'localhost');

sub.on('error', function (err) {
    console.error('Sub client error: ', err);
});
sub.on('end', function () {
    console.log('Sub client ending');
});
sub.on('reconnecting', function () {
    console.log('Sub client reconnecting ');
});
sub.on('ready', function () {
    console.log('sub ready');

});
sub.on('message', function(channel, message) {
    try {
        let t = JSON.parse(message);
        console.log('Got message in channel: ' + channel, t.message_id);
    } catch(e) {
        console.log('Got message in channel: ' + channel);
    }
});

for (let i = 0; i < 10000; i++) {
    sub.subscribe('message_' + i, function(error) {
        if (error) {
            console.error('Unable to subscribe', error);
        }
    });
}

publish.js

let redis = require('redis');

let pub = redis.createClient(6379, 'localhost');
pub.on('error', function(err) {
    console.error('Pub client error: ', err);
});
pub.on('end', function () {
    console.error('Pub client ending');
});
pub.on('reconnecting', function () {
    console.log('Pub client reconnecting');
});
pub.on('ready', function () {
    console.log('pub ready');
});

for (let i = 0; i < 10000; i++) {
    pub.publish('message_' + i, JSON.stringify({
        message_id: process.pid + '_' + i,
        text: 'Very big, large, sizable, substantial, great, huge, immense, enormous, extensive, colossal message! ' +
        'Very big, large, sizable, substantial, great, huge, immense, enormous, extensive, colossal message! ' +
        'Very big, large, sizable, substantial, great, huge, immense, enormous, extensive, colossal message! ' +
        'Very big, large, sizable, substantial, great, huge, immense, enormous, extensive, colossal message! ' +
        'Very big, large, sizable, substantial, great, huge, immense, enormous, extensive, colossal message! ' +
        'Very big, large, sizable, substantial, great, huge, immense, enormous, extensive, colossal message! ' +
        'Very big, large, sizable, substantial, great, huge, immense, enormous, extensive, colossal message! ' +
        'Very big, large, sizable, substantial, great, huge, immense, enormous, extensive, colossal message! ' +
        'Very big, large, sizable, substantial, great, huge, immense, enormous, extensive, colossal message! ' +
        'Very big, large, sizable, substantial, great, huge, immense, enormous, extensive, colossal message! ' +
        'Very big, large, sizable, substantial, great, huge, immense, enormous, extensive, colossal message! ' +
        'Very big, large, sizable, substantial, great, huge, immense, enormous, extensive, colossal message! ' +
        'Very big, large, sizable, substantial, great, huge, immense, enormous, extensive, colossal message! ' +
        'Very big, large, sizable, substantial, great, huge, immense, enormous, extensive, colossal message! ' +
        'Very big, large, sizable, substantial, great, huge, immense, enormous, extensive, colossal message! ' +
        'Very big, large, sizable, substantial, great, huge, immense, enormous, extensive, colossal message! ' +
        'Very big, large, sizable, substantial, great, huge, immense, enormous, extensive, colossal message! ' +
        'Very big, large, sizable, substantial, great, huge, immense, enormous, extensive, colossal message! ' +
        'Very big, large, sizable, substantial, great, huge, immense, enormous, extensive, colossal message! ' +
        'Very big, large, sizable, substantial, great, huge, immense, enormous, extensive, colossal message! ' +
        'Very big, large, sizable, substantial, great, huge, immense, enormous, extensive, colossal message! ' +
        'Very big, large, sizable, substantial, great, huge, immense, enormous, extensive, colossal message! ' +
        'Very big, large, sizable, substantial, great, huge, immense, enormous, extensive, colossal message! '
    }));
}

Steps

1. Run all needed processes

const spawn = require('child_process').spawn;

let sub =  spawn('node', ['subscribe.js'], {stdio: ['ignore', 'inherit', 'inherit']});

for (let i =0; i <20; i++ ) {
    let pub =  spawn('node', ['publish.js'], {stdio: ['ignore', 'ignore', 'ignore']});
}

2. Once we get error on Redis
   # Client id=280 addr=127.0.0.1:62272 fd=30 name= age=2 idle=0 flags=N db=0 sub=10000 psub=0 multi=-1 qbuf=0 qbuf-free=0 obl=0 oll=1636 omem=33555360 events=rw cmd=subscribe scheduled to be closed ASAP for overcoming of output buffer limits.

3. As the result: Sub client get end event and try to reconnect but hangs.

Reason

During the research I found out that redis-parser instance(this.reply_parser in index.js) contains buffer with corrupted message e.g.

*3
$7
message
$10
message_26
$4858

Next data from stream is response for info command that initiates by Sub client during reconnect

$2140
# Server
redis_version:3.2.6
redis_git_sha1:00000000
redis_git_dirty:0
redis_build_id:d7230ef0a2976d7c
redis_mode:standalone
os:Linux 4.4.17-boot2docker x86_64
arch_bits:64
multiplexing_api:epoll
gcc_version:4.9.2
process_id:1
run_id:a8efde8774bbde62cdbb1deddf59
....

But no one clears redis-parser buffer on recconnect also redis-parser can't process buffer with corrupted message so Sub client hangs.

• Version: node_redis v.2.6.4 and Redis official v3.2.6 docker image without specific configuration(https://hub.docker.com/_/redis/)
• Platform: Node.js v6 on Windows 7