test_concurrency.py

import argparse
import multiprocessing
import threading

from pylibmc.test import make_test_client
import random

NUM_THREADS = 4
NUM_PROCS = 4
NUM_SETS = 1000


def inner_set_values(client):
    key, val = str(random.randrange(1, 10)), str(random.randrange(1, 10))
    for i in range(NUM_SETS):
        client.set(key, val)

    client.get(key)


def set_values(client):
    threads = []
    for _ in range(NUM_THREADS):
        th = threading.Thread(target=inner_set_values, args=(client, ))
        th.daemon = False
        th.start()
        threads.append(th)

    for t in threads:
        t.join()


def parse_arguments():
    parser = argparse.ArgumentParser(description='Stress test pylibmc set')
    parser.add_argument('-p', '--procs', default=NUM_PROCS, type=int)
    parser.add_argument('-t', '--threads', default=NUM_THREADS, type=int)
    parser.add_argument('-s', '--sets', default=NUM_SETS, type=int)

    return parser.parse_args()


def main():
    ns = parse_arguments()
    global NUM_PROCS, NUM_SETS, NUM_THREADS
    NUM_PROCS = ns.procs
    NUM_THREADS = ns.threads
    NUM_SETS = ns.sets

    client = make_test_client(binary=True)

    procs = [multiprocessing.Process(target=set_values, args=(client, )) for _ in range(NUM_PROCS)]
    for proc in procs:
        proc.start()

    for proc in procs:
        proc.join()

    client.disconnect_all()


if __name__ == '__main__':
    main()

It seems that if you try to use the same pylibmc Client object between multiple processes, this problem arises.
I found a solution to this problem by simply creating a new pylibmc Client for each call to set().

Bear in mind that on UNIX-like systems multiprocessing will by default call fork() for each new process, and fork will try to copy everything from the parent process to the child process. I did try to see if the Client object is shared between all processes, and it does return the same value when I call id(client), even with different PIDs.

I was running way too many processes, each of them using the pylibmc client. I reduced the number of processes and it worked.

+1. Separating pylibmc client by pid fixed the problem.