Majordomo worker API in Python

"""Majordomo Protocol Worker API, Python version

Implements the MDP/Worker spec at

Author: Min RK <moc.liamg|krnimajneb#moc.liamg|krnimajneb>
Based on Java example by Arkadiusz Orzechowski

import logging
import time
import zmq

from zhelpers import dump
# MajorDomo protocol constants:
import MDP

class MajorDomoWorker(object):
"""Majordomo Protocol Worker API, Python version

Implements the MDP/Worker spec at

HEARTBEAT_LIVENESS = 3 # 3-5 is reasonable
broker = None
ctx = None
service = None

worker = None # Socket to broker
heartbeat_at = 0 # When to send HEARTBEAT (relative to time.time(), so in seconds)
liveness = 0 # How many attempts left
heartbeat = 2500 # Heartbeat delay, msecs
reconnect = 2500 # Reconnect delay, msecs

# Internal state
expect_reply = False # False only at start

timeout = 2500 # poller timeout
verbose = False # Print activity to stdout

# Return address, if any
reply_to = None

def __init__(self, broker, service, verbose=False): = broker
self.service = service
self.verbose = verbose
self.ctx = zmq.Context()
self.poller = zmq.Poller()
logging.basicConfig(format="%(asctime)s %(message)s", datefmt="%Y-%m-%d %H:%M:%S",

def reconnect_to_broker(self):
"""Connect or reconnect to broker"""
if self.worker:
self.worker = self.ctx.socket(zmq.DEALER)
self.worker.linger = 0
self.poller.register(self.worker, zmq.POLLIN)
if self.verbose:"I: connecting to broker at %s…",

# Register service with broker
self.send_to_broker(MDP.W_READY, self.service, [])

# If liveness hits zero, queue is considered disconnected
self.liveness = self.HEARTBEAT_LIVENESS
self.heartbeat_at = time.time() + 1e-3 * self.heartbeat

def send_to_broker(self, command, option=None, msg=None):
"""Send message to broker.

If no msg is provided, creates one internally

if msg is None:
msg = []
elif not isinstance(msg, list):
msg = [msg]

if option:
msg = [option] + msg

msg = ['', MDP.W_WORKER, command] + msg
if self.verbose:"I: sending %s to broker", command)

def recv(self, reply=None):
"""Send reply, if any, to broker and wait for next request."""
# Format and send the reply if we were provided one
assert reply is not None or not self.expect_reply

if reply is not None:
assert self.reply_to is not None
reply = [self.reply_to, ''] + reply
self.send_to_broker(MDP.W_REPLY, msg=reply)

self.expect_reply = True

while True:
# Poll socket for a reply, with timeout
items = self.poller.poll(self.timeout)
except KeyboardInterrupt:
break # Interrupted

if items:
msg = self.worker.recv_multipart()
if self.verbose:"I: received message from broker: ")

self.liveness = self.HEARTBEAT_LIVENESS
# Don't try to handle errors, just assert noisily
assert len(msg) >= 3

empty = msg.pop(0)
assert empty == ''

header = msg.pop(0)
assert header == MDP.W_WORKER

command = msg.pop(0)
if command == MDP.W_REQUEST:
# We should pop and save as many addresses as there are
# up to a null part, but for now, just save one…
self.reply_to = msg.pop(0)
# pop empty
assert msg.pop(0) == ''

return msg # We have a request to process
elif command == MDP.W_HEARTBEAT:
# Do nothing for heartbeats
elif command == MDP.W_DISCONNECT:
else :
logging.error("E: invalid input message: ")

self.liveness -= 1
if self.liveness == 0:
if self.verbose:
logging.warn("W: disconnected from broker - retrying…")
except KeyboardInterrupt:

# Send HEARTBEAT if it's time
if time.time() > self.heartbeat_at:
self.heartbeat_at = time.time() + 1e-3*self.heartbeat

logging.warn("W: interrupt received, killing worker…")
return None

def destroy(self):
# context.destroy depends on pyzmq >= 2.1.10