hiredis blocking scenario

after_fixes

perf stat ./client
Error: Read timed out

 Performance counter stats for './client':

          0.963859 task-clock (msec)         #    0.081 CPUs utilized
                 3 context-switches          #    0.003 M/sec
                 0 cpu-migrations            #    0.000 K/sec
               236 page-faults               #    0.245 M/sec
   <not supported> cycles
   <not supported> stalled-cycles-frontend
   <not supported> stalled-cycles-backend
   <not supported> instructions
   <not supported> branches
   <not supported> branch-misses

       0.011916365 seconds time elapsed

before_fixes

~/src/opensource/hiredis $ time ./client
Reply: foo
5.065 secs

client.c

#include <stdio.h>
#include <hiredis/hiredis.h>

int main(int argc, char **argv)
{
  redisContext *c = redisConnect("localhost", 6379);
  redisReply *reply = redisCommand(c, "GET 1.1.1.1:repsheet:ip:blacklist");
  if (reply->type != REDIS_REPLY_NIL) {
    printf("Reply: %s\n", reply->str);
    freeReplyObject(reply);
  }
  return 0;
}

hiredis.c

int redisBufferRead(redisContext *c) {
    char buf[1024*16];
    int nread;

    /* Return early when the context has seen an error. */
    if (c->err)
        return REDIS_ERR;

    fd_set read_fds;
    FD_ZERO(&read_fds);
    FD_SET(c->fd, &read_fds);
    struct timeval timeout;
    timeout.tv_sec = 0;
    timeout.tv_usec = 10 * 1000;

    if (select(c->fd + 1, &read_fds, NULL, NULL, &timeout) == 1) {
      nread = read(c->fd,buf,sizeof(buf));
      if (nread == -1) {
        if ((errno == EAGAIN && !(c->flags & REDIS_BLOCK)) || (errno == EINTR)) {
          /* Try again later */
        } else {
          __redisSetError(c,REDIS_ERR_IO,NULL);
          return REDIS_ERR;                                                                                                                                                                }
      } else if (nread == 0) {
        __redisSetError(c,REDIS_ERR_EOF,"Server closed the connection");
        return REDIS_ERR;
      } else {
        if (redisReaderFeed(c->reader,buf,nread) != REDIS_OK) {
          __redisSetError(c,c->reader->err,c->reader->errstr);
          return REDIS_ERR;
        }
      }
    } else {
      /* timeout reached */
      __redisSetError(c,REDIS_ERR_IO,"Read timed out");
      return REDIS_ERR;
    }

    return REDIS_OK;
}

server.rb

#This simply fakes Redis and causes a slow request. Using this to simulate a Redis CPU spike or other anomaly.

require 'socket'

server = TCPServer.open(6379)

loop do
  client = server.accept

  5.times do
    p client.gets
  end

  sleep(5)

  client.print "$3\r\nfoo\r\n"
end

timeout.c

#include <stdio.h>
#include <stdlib.h>
#include <time.h>

#define SECONDS_TO_NANOS(x) x * 1000000000
#define MILLIS_TO_NANOS(x) x * 1000000

#define TIMEOUT -1

long current_nanos() {
  struct timespec current;
  clock_gettime(CLOCK_MONOTONIC, &current);
  return current.tv_nsec + SECONDS_TO_NANOS(current.tv_sec);
}

int try(int milliseconds) {
  struct timespec start, end;
  clock_gettime(CLOCK_MONOTONIC, &start);
  end = start;

  start.tv_nsec += SECONDS_TO_NANOS(start.tv_sec);
  end.tv_nsec = start.tv_nsec + MILLIS_TO_NANOS(milliseconds);

  while (end.tv_nsec > current_nanos()) {
    // Do stuff
  }

  return -1;
}

int main(int argc, char **argv)
{
  int result = try(strtol(argv[1], 0, 10));

  if (result == TIMEOUT) {
    printf("Operation timed out\n");
  }

  return 0;
}

timeout_run

perf stat ./timeout 10
Operation timed out

 Performance counter stats for './timeout 10':

         10.335301 task-clock (msec)         #    0.940 CPUs utilized
                 4 context-switches          #    0.387 K/sec
                 0 cpu-migrations            #    0.000 K/sec
               127 page-faults               #    0.012 M/sec
   <not supported> cycles
                 0 stalled-cycles-frontend   #    0.00% frontend cycles idle
                 0 stalled-cycles-backend    #    0.00% backend  cycles idle
   <not supported> instructions
   <not supported> branches
   <not supported> branch-misses

       0.010992224 seconds time elapsed