Package ivy :: Module ivy
[hide private]
[frames] | no frames]

Source Code for Module ivy.ivy

   1  """ 
   2  Using an IvyServer 
   3  ------------------ 
   4   
   5  The following code is a typical example of use: 
   6   
   7  .. python:: 
   8      from ivy.ivy import IvyServer 
   9       
  10      class MyAgent(IvyServer): 
  11        def __init__(self, name): 
  12          IvyServer.__init__(self,'MyAgent') 
  13          self.name = name 
  14          self.start('127.255.255.255:2010') 
  15          self.bind_msg(self.handle_hello, 'hello .*') 
  16          self.bind_msg(self.handle_button, 'BTN ([a-fA-F0-9]+)') 
  17           
  18        def handle_hello(self, agent): 
  19          print '[Agent %s] GOT hello from %r'%(self.name, agent) 
  20         
  21        def handle_button(self, agent, btn_id): 
  22          print '[Agent %s] GOT BTN button_id=%s from %r'%(self.name, btn_id, agent) 
  23          # let's answer! 
  24          self.send_msg('BTN_ACK %s'%btn_id) 
  25       
  26      a=MyAgent('007') 
  27   
  28   
  29  Implementation details 
  30  ---------------------- 
  31   
  32  An Ivy client is made of several threads: 
  33   
  34    - an `IvyServer` instance 
  35   
  36    - a UDP server, lanched by the Ivy server, listening to incoming UDP 
  37      broadcast messages 
  38   
  39    - `IvyTimer` objects  
  40   
  41  :group Messages types: BYE, ADD_REGEXP, MSG, ERROR, DEL_REGEXP, END_REGEXP, 
  42    END_INIT, START_REGEXP, START_INIT, DIRECT_MSG, DIE 
  43  :group Separators: ARG_START, ARG_END 
  44  :group Misc. constants: DEFAULT_IVYBUS, PROTOCOL_VERSION, IVY_SHOULD_NOT_DIE 
  45    IvyApplicationConnected, IvyApplicationDisconnected, DEFAULT_TTL 
  46  :group Objects and functions related to logging: ivylogger, debug, log, warn, 
  47    error, ivy_loghdlr, ivy_logformatter 
  48   
  49  Copyright (c) 2005-2008 Sebastien Bigaret <sbigaret@users.sourceforge.net> 
  50  """ 
  51   
  52  import logging, os 
  53  ivylogger = logging.getLogger('Ivy') 
  54   
  55  if os.environ.get('IVY_LOG_TRACE'): 
  56    logging.TRACE=logging.DEBUG-1 
  57    logging.addLevelName(logging.TRACE, "TRACE") 
  58    trace = lambda *args, **kw: ivylogger.log(logging.TRACE, *args, **kw) 
  59  else: 
  60    trace = lambda *args, **kw: None 
  61   
  62  debug = ivylogger.debug 
  63  info = log = ivylogger.info 
  64  warn = ivylogger.warning 
  65  error = ivylogger.error 
  66   
  67  ivy_loghdlr = logging.StreamHandler() # stderr by default 
  68  ivy_logformatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s') 
  69   
  70  ivy_loghdlr.setFormatter(ivy_logformatter) 
  71  ivylogger.addHandler(ivy_loghdlr) 
  72  #ivylogger.setLevel(logging.DEBUG) 
  73  ivylogger.setLevel(logging.INFO) 
  74   
  75  ## 
  76  DEFAULT_IVYBUS = '127:2010' 
  77  PROTOCOL_VERSION = 3 
  78   
  79  # Message types. Refer to "The Ivy architecture and protocol" for details 
  80  BYE = 0 
  81  ADD_REGEXP = 1 
  82  MSG = 2 
  83  ERROR = 3 
  84  DEL_REGEXP = 4 
  85   
  86  # START_REGEXP and END_REGEXP are the ones declared in ivy.c 
  87  # however we'll use the aliases START_INIT and END_INIT here 
  88  END_REGEXP   = END_INIT   = 5 
  89  START_REGEXP = START_INIT = 6 
  90   
  91  DIRECT_MSG = 7 
  92  DIE = 8 
  93   
  94  # Other constants 
  95  ARG_START = '\002' 
  96  ARG_END = '\003' 
  97   
  98  # for multicast, arbitrary TTL value taken from ivysocket.c:SocketAddMember 
  99  DEFAULT_TTL = 64   
 100   
 101  IvyApplicationConnected = 1 
 102  IvyApplicationDisconnected = 2 
 103   
 104  IvyRegexpAdded = 3 
 105  IvyRegexpRemoved = 4 
 106   
 107  IVY_SHOULD_NOT_DIE = 'Ivy Application Should Not Die' 
 108   
 109   
110 -def void_function(*arg, **kw):
111 "A function that accepts any number of parameters and does nothing" 112 pass
113 114 #
115 -def UDP_init_and_listen(broadcast_addr, port, socket_server):
116 """ 117 Called by an IvyServer at startup; the method is responsible for: 118 119 - sending the initial UDP broadcast message, 120 121 - waiting for incoming UDP broadcast messages being sent by new clients 122 connecting on the bus. When it receives such a message, a connection 123 is established to that client and that connection (a socket) is then 124 passed to the IvyServer instance. 125 126 :Parameters: 127 - `broadcast_addr`: the broadcast address used on the Ivy bus 128 - `port`: the port dedicated to the Ivy bus 129 - `socket_server`: instance of an IvyServer handling communications 130 for our client. 131 """ 132 log('Starting Ivy UDP Server on %r:%r'%(broadcast_addr,port)) 133 import socket 134 s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) 135 on=1 136 s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, on) 137 if hasattr(socket, 'SO_REUSEPORT'): 138 s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, on) 139 s.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, on) 140 141 s.bind(('',port)) # '' means: INADDR_ANY 142 143 # Multicast 144 if is_multicast(broadcast_addr): 145 debug('Broadcast address is a multicast address') 146 import struct 147 ifaddr = socket.INADDR_ANY 148 mreq=struct.pack('4sl', 149 socket.inet_aton(broadcast_addr), 150 socket.htonl(ifaddr)) 151 s.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP, mreq) 152 s.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_TTL, DEFAULT_TTL) 153 # /Multicast 154 155 s.sendto("%li %s %s %s\n"%(PROTOCOL_VERSION,socket_server.port, 156 socket_server.agent_id, socket_server.agent_name), 157 (broadcast_addr, port)) 158 159 s.settimeout(0.1) 160 while socket_server.isAlive(): 161 try: 162 udp_msg, (ip, ivybus_port) = s.recvfrom(1024) 163 except socket.timeout: 164 continue 165 166 debug('UDP got: %r from: %r', udp_msg, ip) 167 168 appid = appname = None 169 try: 170 udp_msg_l = udp_msg.split(' ') 171 protocol_version, port_number = udp_msg_l[:2] 172 if len(udp_msg_l) > 2: 173 # "new" udp protocol, with id & appname 174 appid = udp_msg_l[2] 175 appname = ' '.join(udp_msg_l[3:]).strip('\n') 176 debug('IP %s has id: %s and name: %s', ip, appid, appname) 177 else: 178 debug('Received message w/o app. id & name from %r', ip) 179 180 port_number = int(port_number) 181 protocol_version = int(protocol_version) 182 183 except (ValueError): # unpack error, invalid literal for int() 184 warn('Received an invalid UDP message (%r) from :', udp_msg) 185 186 if protocol_version != PROTOCOL_VERSION: 187 error('Received a UDP broadcast msg. w/ protocol version:%s , expected: %s', protocol_version, PROTOCOL_VERSION) 188 continue 189 190 if appid == socket_server.agent_id: 191 # this is us! 192 debug('UDP from %r: ignored: we sent that one!', ip) 193 continue 194 195 # build a new socket and delegate its handling to the SocketServer 196 debug('New client connected: %s:%s', ip, port_number) 197 198 new_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 199 new_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, on) 200 trace('New client %s:%s, socket %r', ip, port_number, new_socket) 201 # Since we already have a client's name and id, lets register it 202 # (this was previously done in IvyHandler's __init__() only) 203 # but we want to check that we did not receive more than once a 204 # broadcast coming from the same 205 new_client = socket_server._get_client(ip, port_number, new_socket, 206 appid, appname) 207 if new_client is None: 208 # an agent with that app-id is already registered 209 info('UDP from %s:%s (%s): discarding message, an application w/ id=%s is already registered', ip, port_number, appname, appid) 210 continue 211 212 try: 213 new_socket.connect((ip, port_number)) 214 except: # e.g., timeout on connect 215 from traceback import format_exc 216 info('Client %r: failed to connect to its socket, ignoring it', 217 new_client) 218 debug('Client %r: failed to connect to its socket, got:%s', 219 new_client, format_exc()) 220 socket_server.remove(ip, port_number, 221 trigger_application_callback=False) 222 else: 223 socket_server.process_request(new_socket, (ip, port_number)) 224 log('UDP Server stopped')
225
226 -class IvyProtocolError(Exception): pass
227 -class IvyMalformedMessage(Exception): pass
228 -class IvyIllegalStateError(RuntimeError): pass
229
230 -def decode_msg(msg):
231 """ 232 233 :return: msg_id, numerical_id, parameters 234 :exception IvyMalformedMessage: 235 """ 236 try: 237 msg_id, _msg = msg.split(' ', 1) 238 msg_id = int(msg_id) 239 num_id, params = _msg.split(ARG_START, 1) 240 num_id = int(num_id) 241 242 if ARG_END in params: 243 # there is an extra ARG_END after the last parameter and 244 # before the newline 245 params = params[:-1].split(ARG_END) 246 247 except ValueError: 248 raise IvyMalformedMessage 249 return msg_id, num_id, params
250
251 -def encode_message(msg_type, numerical_id, params=''):
252 """ 253 254 params is string -> added as-is 255 params is list -> concatenated, separated by ARG_END 256 """ 257 msg = "%s %s"%(msg_type, numerical_id) + ARG_START 258 if type(params) is type(''): 259 msg += params 260 else: 261 msg += ARG_END.join(params) 262 msg += ARG_END 263 trace('encode_message(params: %s) -> %s'%(repr(params),repr(msg+'\n'))) 264 return msg + '\n'
265 266 267 268 NOT_INITIALIZED = 0 269 INITIALIZATION_IN_PROGRESS=1 270 INITIALIZED=2 271
272 -class IvyClient:
273 """ 274 Represents a client connected to the bus. Every callback methods 275 registered by an agent receive an object of this type as their first 276 parameter, so that they know which agent on the bus is the cause of the 277 event which triggered the callback. 278 279 An IvyClient is responsible for: 280 281 - managing the remote agent's subscriptions, 282 283 - sending messages to the remote agent. 284 285 It is **not** responsible for receiving messages from the client: another 286 object is in charge of that, namely an `IvyHandler` object. 287 288 The local IvyServer creates one IvyClient per agent on the bus. 289 290 MT-safety 291 --------- 292 See the discussion in `regexps`. 293 294 :group Protocol-related methods: start_init, end_init, 295 send_new_subscription, remove_subscription, wave_bye 296 :group Manipulating the remote agent's subscriptions: 297 add_regexp, remove_regexp 298 :group Sending messages: send_msg, send_direct_message, send_die_message 299 300 :ivar regexps: a dictionary mapping subscriptions' ids (as delivered by 301 `add_regexp`) to the corresponding regular expressions. Precisely, it 302 maps ids to tuples being ``(regexp_as_string, compiled_regexp)``. You 303 shouldn't directly access or manipulate this variable, use `add_regexp` 304 and `remove_regexp` instead; however if you really need/want to, you 305 must acquire the `regexp_lock` before accessing/modifying it. 306 307 :ivar regexp_lock: a non-reentrant lock protecting the variable 308 `regexps`. Used by methods `add_regexp`, `remove_regexp` and `send_msg`. 309 :type regexp_lock: `threading.Lock` 310 """ 311 agent_id = None 312 agent_name = None 313 port = None 314 socket = None 315 #struct _client in ivysocket.c
316 - def __init__(self, ip, port, client_socket, 317 agent_id=None, agent_name=None):
318 self.agent_id = agent_id 319 # agent_name will be overridden when start_init() is called 320 # but nevermind, 321 self.agent_name = agent_name 322 self.ip = ip 323 self.port = port 324 self.socket = client_socket 325 self.regexps = {} # maps regexp_id to (regexp_string, compiled_regexp) 326 import socket 327 self.fqdn = socket.getfqdn(ip) 328 self.status = NOT_INITIALIZED 329 self.socket.settimeout(0.1) 330 331 import threading 332 # regexps are modified w/ add_regexp and remove_regexp, called 333 # by the server, while they are accessed by send_message() 334 # called by the corresponding IvyHandler-thread --> needs to be 335 # protected against concurrent access. 336 self.regexps_lock=threading.Lock() # non-reentrant, faster than RLock
337
338 - def start_init(self, agent_name):
339 """ 340 Finalizes the initialization process by setting the client's 341 agent_name. This is a Ivy protocol requirement that an application 342 sends its agent-name only once during the initial handshake (beginning 343 with message of type ``START_INIT`` and ending with a type 344 ``END_INIT``). After this method is called, we expect to receive the 345 initial subscriptions for that client (or none); the initialization 346 process completes after `end_init` is called. 347 348 :exception IvyIllegalStateError: if the method has already been called 349 once 350 """ 351 if self.status != NOT_INITIALIZED: 352 raise IvyIllegalStateError 353 self.agent_name = agent_name 354 self.status = INITIALIZATION_IN_PROGRESS 355 debug('Client:%r: Starting initialization', self)
356
357 - def end_init(self):
358 """ 359 Should be called when the initalization process ends. 360 361 :exception IvyIllegalStateError: if the method has already been called 362 (and ``self.status`` has already been set to ``INITIALIZED``) 363 """ 364 if self.status is INITIALIZED: 365 raise IvyIllegalStateError 366 debug('Client:%r: Initialization ended', self) 367 self.status = INITIALIZED
368
369 - def add_regexp(self, regexp_id, regexp):
370 """ 371 :exception IvyIllegalStateError: if the client has not been fully 372 initialized yet (see `start_init`) 373 """ 374 if self.status not in ( INITIALIZATION_IN_PROGRESS, INITIALIZED ): 375 # initialization has not begun 376 raise IvyIllegalStateError 377 378 # TODO: handle error on compile 379 import re 380 debug('Client:%r: Adding regexp id=%s: %r', self, regexp_id, regexp) 381 self.regexps_lock.acquire() 382 try: 383 self.regexps[regexp_id] = (regexp, re.compile(regexp)) 384 finally: 385 self.regexps_lock.release()
386
387 - def remove_regexp(self, regexp_id):
388 """ 389 Removes a regexp 390 391 :return: the regexp that has been removed 392 :exception IvyIllegalStateError: if the client has not been fully 393 initialized yet (see `start_init`) 394 :exception KeyError: if no such subscription exists 395 """ 396 if self.status not in ( INITIALIZATION_IN_PROGRESS, INITIALIZED ): 397 # initialization has not begun 398 raise IvyIllegalStateError 399 debug('Client:%r: removing regexp id=%s', self, regexp_id) 400 regexp = None 401 402 self.regexps_lock.acquire() 403 try: 404 regexp = self.regexps.pop(regexp_id)[0] 405 finally: 406 self.regexps_lock.release() 407 return regexp
408
409 - def get_regexps(self):
410 self.regexps_lock.acquire() 411 try: 412 return [ (idx, s[0]) for idx, s in self.regexps.items()] 413 finally: 414 self.regexps_lock.release()
415
416 - def send_msg(self, msg):
417 """ 418 Sends a message to the client. The message is compared to the 419 client's subscriptions and it is sent if one of them matches. 420 421 :return: ``True`` if the message was actually sent to the client, that 422 is: if there is a regexp matching the message in the client's 423 subscriptions; returns ``False`` otherwise. 424 425 """ 426 if self.status is not INITIALIZED: 427 return 428 debug('Client:%r: Searching a subscription matching msg %r', 429 self, msg) 430 # TODO: if 2 regexps match a message, we should be able to tell 431 # TODO: which one is selected (for example, try them in the order 432 # TODO: of their subscriptions) 433 self.regexps_lock.acquire() 434 try: 435 436 for id, (s, r) in self.regexps.items(): 437 captures = r.match(msg) 438 if captures: 439 captures=captures.groups() 440 # The following is needed to reproduce the very same 441 # behaviour #observed w/ the C library 442 # (tested w/ pyhello.py and ivyprobe) 443 if len(captures)==0: 444 captures='' 445 debug('Client:%r: msg being sent: %r (regexp: %r)', 446 self,captures,s) 447 self._send(MSG, id, captures) 448 return True 449 return False 450 451 finally: 452 self.regexps_lock.release()
453
454 - def send_direct_message(self, num_id, msg):
455 """ 456 Sends a direct message 457 458 Note: the message will be encoded by `encode_message` with 459 ``numerical_id=num_id`` and ``params==msg``; this means that if `msg` 460 is not a string but a list or a tuple, the direct message will contain 461 more than one parameter. This is an **extension** of the original Ivy 462 design, supported by python, but if you want to inter-operate with 463 applications using the standard Ivy API the message you send *must* be 464 a string. See in particular in ``ivy.h``:: 465 466 typedef void (*MsgDirectCallback)( IvyClientPtr app, void *user_data, int id, char *msg ) ; 467 468 """ 469 if self.status is INITIALIZED: 470 debug('Client:%r: a direct message being sent: id: %r msg: %r', 471 self, id, msg) 472 self._send(DIRECT_MSG, num_id, msg)
473
474 - def send_die_message(self, num_id=0, msg=""):
475 """ 476 Sends a die message 477 """ 478 if self.status is INITIALIZED: 479 debug('Client:%r: die msg being sent: num_id: %r msg: %r', 480 self, num_id, msg) 481 self._send(DIE, num_id, msg)
482
483 - def send_new_subscription(self, idx, regexp):
484 """ 485 Notifies the remote agent that we (the local agent) subscribe to 486 a new type of messages 487 488 :Parameters: 489 - `idx`: the index/id of the new subscription. It is the 490 responsability of the local agent to make sure that every 491 subscription gets a unique id. 492 - `regexp`: a regular expression. The subscription consists in 493 receiving messages mathcing the regexp. 494 """ 495 self._send(ADD_REGEXP, idx, regexp)
496
497 - def remove_subscription(self, idx):
498 """ 499 Notifies the remote agent that we (the local agent) are not 500 interested in a given subscription. 501 502 :Parameters: 503 - `idx`: the index/id of a subscription previously registered with 504 `send_new_subscription`. 505 """ 506 self._send(DEL_REGEXP, idx)
507
508 - def wave_bye(self, id=0):
509 "Notifies the remote agent that we are about to quit" 510 self._send(BYE, id)
511
512 - def send_error(self, num_id, msg):
513 """ 514 Sends an error message 515 """ 516 self._send(ERROR, num_id, msg)
517
518 - def __eq__(self, client):
519 """ 520 cf. dict[client] or dict[(ip,port)] UNNEEDED FOR THE MOMENT 521 """ 522 if isinstance(client, IvyClient): 523 return self.ip==client.ip and self.port==client.port 524 525 import types 526 if type(client) in (types.TupleType, types.ListType) \ 527 and len(client) == 2: 528 return self.ip == client[0] and self.port == client[1] 529 530 return False
531
532 - def __hash__(self):
533 "``hash((self.ip, self.port))``" 534 return hash((self.ip, self.port))
535
536 - def __repr__(self):
537 "Returns ``'ip:port (agent_name)'``" 538 return "%s:%s (%s)"%(self.ip, self.port, self.agent_name)
539
540 - def __str__(self):
541 "Returns ``'agent_name@FQDN'``" 542 return "%s@%s"%(self.agent_name, self.fqdn)
543
544 - def _send(self, msg_type, *params):
545 """ 546 Internally used to send message to the remote agent through the opened 547 socket `self.socket`. This method catches all exceptions 548 `socket.error` and `socket.timeout` and ignores them, simply logging 549 them at the "info" level. 550 551 The errors that can occur are for example:: 552 553 socket.timeout: timed out 554 socket.error: (104, 'Connection reset by peer') 555 socket.error: (32, 'Broken pipe') 556 557 They can happen after a client disconnects abruptly (because it was 558 killed, because the network is down, etc.). We assume here that if and 559 when an error happens here, a disconnection will be detected shortly 560 afterwards by the server which then removes this agent from the bus. 561 Hence, we ognore the error; please also note that not ignoring the 562 error can have an impact on code, for example, IyServer.send_msg() 563 does not expect that IvyClient.send() fails and if it fails, it is 564 possible that the server does not send the message to all possible 565 subscribers. 566 567 .. note:: ``ivysocket.c:SocketSendRaw()`` also ignores error, simply 568 logging them. 569 570 """ 571 import socket 572 try: 573 self.socket.send(encode_message(msg_type, *params)) 574 except (socket.timeout, socket.error), exc: 575 log('[ignored] Error on socket with %r: %s', self, exc)
576 577 import SocketServer, threading
578 -class IvyServer(SocketServer.ThreadingTCPServer, threading.Thread):
579 """ 580 An Ivy server is responsible for receiving and handling the messages 581 that other clients send on an Ivy bus to a given agent. 582 583 An IvyServer has two important attributes: `usesDaemons` and 584 `server_termination`. 585 586 587 :ivar usesDaemons: 588 whether the threads are daemonic or not. Daemonic 589 threads do not prevent python from exiting when the main thread stop, 590 while non-daemonic ones do. Default is False. This attribute should 591 be set through at `__init__()` time and should not be modified 592 afterwards. 593 594 :ivar server_termination: 595 a `threading.Event` object that is set on server shutdown. It can be 596 used either to test whether the server has been stopped 597 (``server_termination.isSet()``) or to wait until it is stopped 598 (``server_termination.wait()``). Application code should not try to set 599 the Event directly, rather it will call `stop()` to terminate the 600 server. 601 602 :ivar port: tells on which port the TCP server awaits connection 603 604 MT-safety 605 --------- 606 All public methods (not starting with an underscore ``_``) are 607 MT-safe 608 609 :group Communication on the ivybus: start, send_msg, send_direct_message, 610 send_ready_message, handle_msg, stop 611 612 :group Inspecting the ivybus: get_clients, _get_client, get_client_with_name 613 614 :group Our own subscriptions: get_subscriptions, bind_msg, unbind_msg, 615 _add_subscription, _remove_subscription, _get_fct_for_subscription 616 617 """ 618 # Impl. note: acquiring/releasing the global lock in methods 619 # requiring it could be done w/ a decorator instead of repeating 620 # the acquire-try-finally-release block each time, but then we 621 # won't be compatible w/ py < 2.4 and I do not want this 622
623 - def __init__(self, agent_name, ready_msg="", 624 app_callback = void_function, 625 die_callback = void_function, 626 usesDaemons=False):
627 """ 628 Builds a new IvyServer. A client only needs to call `start()` on the 629 newly created instances to connect to the corresponding Ivy bus and to 630 start communicating with other applications. 631 632 MT-safety: both functions `app_callback` and `die_callback` must be 633 prepared to be called concurrently 634 635 :Parameters: 636 - `agent_name`: the client's agent name 637 - `ready_msg`: a message to send to clients when they connect 638 - `app_callback`: a function called each time a client connects or 639 disconnects. This function is called with a single parameter 640 indicating which event occured: `IvyApplicationConnected` or 641 `IvyApplicationDisconnected`. 642 - `die_callback`: called when the IvyServer receives a DIE message 643 - `usesDaemons`: see above. 644 645 :see: `bind_msg()`, `start()` 646 """ 647 threading.Thread.__init__(self, target=self.serve_forever) 648 649 # the empty string is equivalent to INADDR_ANY 650 SocketServer.TCPServer.__init__(self, ('',0),IvyHandler) 651 self.port = self.socket.getsockname()[1] 652 #self.allow_reuse_address=True 653 654 # private, maps (ip,port) to IvyClient! 655 self._clients = {} 656 657 # idx -> (regexp, function), see bind_msg() for details, below 658 self._subscriptions = {} 659 # the next index to use within the _subscriptions map. 660 self._next_subst_idx = 0 661 662 self.agent_name = agent_name 663 self.ready_message = ready_msg 664 665 # app_callback's parameter event=CONNECTED / DISCONNECTED 666 self.app_callback = app_callback 667 self.die_callback = die_callback 668 self.direct_callback = void_function 669 self.regexp_change_callback = void_function 670 671 # the global_lock protects: _clients, _subscriptions 672 # and _next_subst_idx 673 self._global_lock = threading.RLock() 674 675 self.usesDaemons = usesDaemons 676 self.setDaemon(self.usesDaemons) 677 self.server_termination = threading.Event() 678 679 import time,random 680 self.agent_id=agent_name+time.strftime('%Y%m%d%H%M%S')+"%05i"%random.randint(0,99999)+str(self.port)
681
682 - def serve_forever(self):
683 """ 684 Handle requests (calling `handle_request()`) until doomsday... or 685 until `stop()` is called. 686 687 This method is registered as the target method for the thread. 688 It is also responsible for launching the UDP server in a separate 689 thread, see `UDP_init_and_listen` for details. 690 691 You should not need to call this method, use `start` instead. 692 """ 693 broadcast, port = decode_ivybus(self.ivybus) 694 l=lambda server=self: UDP_init_and_listen(broadcast, port, server) 695 t2 = threading.Thread(target=l) 696 t2.setDaemon(self.usesDaemons) 697 log('Starting UDP listener') 698 t2.start() 699 700 self.socket.settimeout(0.1) 701 while not self.server_termination.isSet(): 702 self.handle_request() 703 log('TCP Ivy Server terminated')
704
705 - def start(self, ivybus=None):
706 """ 707 Binds the server to the ivybus. The server remains connected until 708 `stop` is called, or 709 """ 710 self.ivybus = ivybus 711 import socket 712 713 log('Starting IvyServer on port %li', self.port) 714 threading.Thread.start(self)
715
716 - def stop(self):
717 """ 718 Disconnects the server from the ivybus. It also sets the 719 `server_termination` event. 720 """ 721 self._global_lock.acquire() 722 try: 723 import socket 724 for client in self._clients.values(): 725 try: 726 client.wave_bye() 727 except socket.error, e: 728 pass 729 finally: 730 self._global_lock.release() 731 self.server_termination.set()
732
733 - def get_clients(self):
734 """ 735 Returns the list of the agent names of all connected clients 736 737 :see: get_client_with_name 738 """ 739 self._global_lock.acquire() 740 try: 741 return [c.agent_name for c in self._clients.values() 742 if c.status == INITIALIZED] 743 finally: 744 self._global_lock.release()
745
746 - def _get_client(self, ip, port, socket=None, 747 agent_id=None,agent_name=None):
748 """ 749 Returns the corresponding client, and create a new one if needed. 750 751 If agent_id is not None, the method checks whether a client with the 752 same id is already registered; if it exists, the method exits by 753 returning None. 754 755 You should not need to call this, use `get_client_with_name` instead 756 """ 757 self._global_lock.acquire() 758 try: 759 # if agent_id is provided, check whether it was already registered 760 if agent_id and agent_id in [c.agent_id for c in self._clients.values()]: 761 return None 762 return self._clients.setdefault( (ip,port), 763 IvyClient(ip, port, socket, 764 agent_id, agent_name) ) 765 finally: 766 self._global_lock.release()
767
768 - def get_client_with_name(self, name):
769 """ 770 Returns the list of the clients registered with a given agent-name 771 772 :see: get_clients 773 """ 774 clients=[] 775 self._global_lock.acquire() 776 try: 777 for client in self._clients.values(): 778 if client.agent_name == name: 779 clients.append(client) 780 return clients 781 finally: 782 self._global_lock.release()
783
784 - def handle_new_client(self, client):
785 """ 786 finalisation de la connection avec le client 787 TODO: peut-etre ajouter un flag (en cours de cnx) sur le client, 788 qui empecherait l'envoi de msg. etc. tant que la cnx. n'est pas 789 confirmee 790 """ 791 self.app_callback(client, IvyApplicationConnected)
792
793 - def handle_die_message(self, msg_id, from_client=None):
794 "" 795 should_die=(self.die_callback(from_client,msg_id) != IVY_SHOULD_NOT_DIE) 796 log("Received a die msg from: %s with id: %s -- should die=%s", 797 from_client or "<unknown>", msg_id, should_die) 798 if should_die: 799 self.stop() 800 return should_die
801
802 - def handle_direct_msg(self, client, num_id, msg):
803 "" 804 log("Received a direct msg from: %s with id: %s -- %s", 805 client or "<unknown>", num_id, msg) 806 self.direct_callback(client, num_id, msg)
807
808 - def handle_regexp_change(self, client, event, id, regexp):
809 """ 810 """ 811 log("Regexp change: %s %s regexp %d: %s", 812 client or "<unknown>", 813 event==ADD_REGEXP and "add" or "remove", 814 id, regexp) 815 if event==ADD_REGEXP: 816 event=IvyRegexpAdded 817 else: 818 event=IvyRegexpRemoved 819 self.regexp_change_callback(client, event, id, regexp)
820
821 - def remove_client(self, ip, port, trigger_application_callback=True):
822 """ 823 Removes a registered client 824 825 This method is responsible for calling ``server.app_callback`` 826 827 :return: the removed client, or None if no such client was found 828 829 .. note:: NO NETWORK CLEANUP IS DONE 830 """ 831 self._global_lock.acquire() 832 try: 833 try: 834 removed_client = self._clients[(ip,port)] 835 except KeyError: 836 debug("Trying to remove a non registered client %s:%s",ip,port) 837 return None 838 debug("Removing client %r", removed_client) 839 del self._clients[removed_client] 840 if trigger_application_callback: 841 self.app_callback(removed_client, IvyApplicationDisconnected) 842 843 return removed_client 844 finally: 845 self._global_lock.release()
846
847 - def send_msg(self, message):
848 """ 849 Examine the message and choose to send a message to the clients 850 that subscribed to such a msg 851 852 :return: the number of clients to which the message was sent 853 """ 854 self._global_lock.acquire() 855 count = 0 856 try: 857 for client in self._clients.values(): 858 if client.send_msg(message): 859 count = count + 1 860 finally: 861 self._global_lock.release() 862 863 return count
864
865 - def send_direct_message(self, agent_name, num_id, msg):
866 self._global_lock.acquire() 867 try: 868 for client in self._clients.values(): 869 # TODO: what if multiple clients w/ the same name?!! 870 if client.agent_name == agent_name: 871 self.client.send_direct_message(num_id, msg) 872 return True 873 return False 874 finally: 875 self._global_lock.release()
876
877 - def send_ready_message(self, client):
878 """ 879 """ 880 if self.ready_message: 881 client.send_msg(self.ready_message)
882
883 - def _add_subscription(self, regexp, fct):
884 """ 885 Registers a new regexp and binds it to the supplied fct. The id 886 assigned to the subscription and returned by method is **unique** 887 to that subscription for the life-time of the server object: even in 888 the case when a subscription is unregistered, its id will _never_ 889 be assigned to another subscription. 890 891 :return: the unique id for that subscription 892 """ 893 # explicit lock here: even if this method is private, it is 894 # responsible for the uniqueness of a subscription's id, so we 895 # prefer to lock it one time too much than taking the risk of 896 # forgetting it (hence, the need for a reentrant lock) 897 self._global_lock.acquire() 898 try: 899 idx = self._next_subst_idx 900 self._next_subst_idx += 1 901 self._subscriptions[idx] = (regexp, fct) 902 return idx 903 finally: 904 self._global_lock.release()
905
906 - def _remove_subscription(self, idx):
907 """ 908 Unregisters the corresponding regexp 909 910 .. warning:: this method is not MT-safe, callers must acquire the 911 global lock 912 913 :return: the regexp that has been removed 914 :except KeyError: if no such subscription can be found 915 """ 916 return self._subscriptions.pop(idx)[0]
917
918 - def _get_fct_for_subscription(self, idx):
919 """ 920 921 .. warning:: this method is not MT-safe, callers must acquire the 922 global lock 923 """ 924 return self._subscriptions[int(idx)][1]
925
926 - def handle_msg(self, client, idx, *params):
927 """ 928 Simply call the function bound to the subscription id `idx` with 929 the supplied parameters. 930 """ 931 self._global_lock.acquire() 932 try: 933 try: 934 return self._get_fct_for_subscription(int(idx))(client, *params) 935 except KeyError: 936 # it is possible that we receive a message for a regexp that 937 # was subscribed then unregistered 938 warn('Asked to handle an unknown subscription: id:%r params: %r' 939 ' --ignoring', idx, params) 940 finally: 941 self._global_lock.release()
942
943 - def get_subscriptions(self):
944 self._global_lock.acquire() 945 try: 946 return [ (idx, s[0]) for idx, s in self._subscriptions.items()] 947 finally: 948 self._global_lock.release()
949
950 - def bind_direct_msg(self, on_direct_msg_fct):
951 """ 952 """ 953 self.direct_callback = on_direct_msg_fct
954
955 - def bind_regexp_change(self, on_regexp_change_callback):
956 """ 957 """ 958 self.regexp_change_callback = on_regexp_change_callback
959
960 - def bind_msg(self, on_msg_fct, regexp):
961 """ 962 Registers a new subscriptions, by binding a regexp to a function, so 963 that this function is called whenever a message matching the regexp 964 is received. 965 966 :Parameters: 967 - `on_msg_fct`: a function accepting as many parameters as there is 968 groups in the regexp. For example: 969 970 - the regexp ``'^hello .*'`` corresponds to a function called w/ no 971 parameter, 972 - ``'^hello (.*)'``: one parameter, 973 - ``'^hello=([^ ]*) from=([^ ]*)'``: two parameters 974 975 - `regexp`: (string) a regular expression 976 977 :return: the binding's id, which can be used to unregister the binding 978 with `unbind_msg()` 979 """ 980 self._global_lock.acquire() 981 idx = self._add_subscription(regexp, on_msg_fct) 982 try: 983 for client in self._clients.values(): 984 client.send_new_subscription(idx, regexp) 985 finally: 986 self._global_lock.release() 987 return idx
988
989 - def unbind_msg(self, id):
990 """ 991 Unbinds a subscription 992 993 :param id: the binding's id, as returned by `bind_msg()` 994 995 :return: the regexp corresponding to the unsubscribed binding 996 :except KeyError: if no such subscription can be found 997 """ 998 self._global_lock.acquire() 999 regexp = None 1000 try: 1001 regexp = self._remove_subscription(id) # KeyError 1002 # notify others that we have no interest anymore in this regexp 1003 for client in self._clients.values(): 1004 client.remove_subscription(id) 1005 finally: 1006 self._global_lock.release() 1007 return regexp
1008
1009 -class IvyHandler(SocketServer.StreamRequestHandler): #BaseRequestHandler):
1010 """ 1011 An IvyHandler is associated to one IvyClient connected to our server. 1012 1013 It runs into a dedicate thread as long as the remote client is connected 1014 to us. 1015 1016 It is in charge of examining all messages that are received and to 1017 take any appropriate actions. 1018 1019 Implementation note: the IvyServer is accessible in ``self.server`` 1020 """
1021 - def handle(self):
1022 """ 1023 """ 1024 # self.request is the socket object 1025 # self.server is the IvyServer 1026 import time 1027 bufsize=1024 1028 socket = self.request 1029 ip = self.client_address[0] 1030 port = self.client_address[1] 1031 1032 trace('New IvyHandler for %s:%s, socket %r', ip, port, socket) 1033 1034 client = self.server._get_client(ip, port, socket) 1035 debug("Got a request from ip=%s port=%s", ip, port) 1036 1037 # First, send our initial subscriptions 1038 socket.send(encode_message(START_INIT, self.server.port, 1039 self.server.agent_name)) 1040 for idx, subscr in self.server.get_subscriptions(): 1041 socket.send(encode_message(ADD_REGEXP, idx, subscr)) 1042 socket.send(encode_message(END_REGEXP, 0)) 1043 1044 while self.server.isAlive(): 1045 from socket import error as socket_error 1046 from socket import timeout as socket_timeout 1047 try: 1048 msgs = socket.recv(bufsize) 1049 except socket_timeout, e: 1050 #debug('timeout on socket bound to client %r', client) 1051 continue 1052 except socket_error, e: 1053 log('Error on socket with %r: %s', client, e) 1054 self.server.remove_client(ip, port) 1055 break # the server will close the TCP connection 1056 1057 if not msgs: 1058 # client is not connected anymore 1059 log('Lost connection with %r', client) 1060 self.server.remove_client(ip, port) 1061 break # the server will close the TCP connection 1062 1063 # Sometimes the message is not fully read on the first try, 1064 # so we insist to get the final newline 1065 if msgs[-1:] != '\n': 1066 1067 # w/ the following idioms (also replicated a second time below) 1068 # we make sure that we wait until we get a message containing 1069 # the final newline, or if the server is terminated we stop 1070 # handling the request 1071 while self.server.isAlive(): 1072 try: 1073 _msg = socket.recv(bufsize) 1074 break 1075 except socket_timeout: 1076 continue 1077 if not self.server.isAlive(): 1078 break 1079 1080 msgs += _msg 1081 while _msg[-1:] != '\n' and self.server.isAlive(): 1082 1083 while self.server.isAlive(): 1084 try: 1085 _msg = socket.recv(bufsize) 1086 break 1087 except socket_timeout: 1088 continue 1089 1090 msgs += _msg 1091 1092 if not self.server.isAlive(): 1093 break 1094 1095 debug("Got a request from ip=%s port=%s: %r", ip, port, msgs) 1096 1097 msgs = msgs[:-1] 1098 1099 1100 msgs=msgs.split('\n') 1101 for msg in msgs: 1102 keep_connection_alive = self.process_ivymessage(msg, client) 1103 if not keep_connection_alive: 1104 self.server.remove_client(ip, port) 1105 break 1106 log('Closing connection to client %r', client)
1107
1108 - def process_ivymessage(self, msg, client):
1109 """ 1110 Examines the message (after passing it through the `decode_msg()` 1111 filter) and takes the appropriate actions depending on the message 1112 types. Please refer to the document `The Ivy Architecture and 1113 Protocol <http://www.tls.cena.fr/products/ivy/documentation>`_ and to 1114 the python code for further details. 1115 1116 1117 :Parameters: 1118 - `msg`: (should not include a newline at end) 1119 1120 :return: ``False`` if the connection should be terminated, ``True`` 1121 otherwise 1122 1123 """ 1124 # cf. static void Receive() in ivy.c 1125 try: 1126 msg_id, num_id, params = decode_msg(msg) 1127 except IvyMalformedMessage: 1128 warn('Received an incorrect message: %r from: %r', msg, client) 1129 # TODO: send back an error message 1130 return True 1131 1132 debug('Got: msg_id: %r, num_id: %r, params: %r', 1133 msg_id, num_id, params) 1134 1135 err_msg = "" 1136 try: 1137 if msg_id == BYE: 1138 # num_id: not meaningful. No parameter. 1139 log('%s waves bye-bye: disconnecting', client) 1140 return False 1141 1142 elif msg_id == ADD_REGEXP: 1143 # num_id=id for the regexp, one parameter: the regexp 1144 err_msg = 'Client %r was not properly initialized'%client 1145 log('%s sending a new subscription id:%r regexp:%r ', 1146 client, num_id, params) 1147 client.add_regexp(num_id, params) 1148 self.server.handle_regexp_change(client, ADD_REGEXP, 1149 num_id, params) 1150 # TODO: handle errors (e.g. 2 subscriptions w/ the same id) 1151 1152 elif msg_id == DEL_REGEXP: 1153 # num_id=id for the regexp to removed, no parameter 1154 err_msg = 'Client %r was not properly initialized'%client 1155 log('%s removing subscription id:%r', client, num_id) 1156 try: 1157 regexp = client.remove_regexp(num_id) 1158 self.server.handle_regexp_change(client, DEL_REGEXP, 1159 num_id, regexp) 1160 except KeyError: 1161 # TODO: what else? 1162 warn('%s tried to remove a non-registered subscription w/ id:%r', client, num_id) 1163 1164 elif msg_id == MSG: 1165 # num_id: regexp_id, parameters: the substrings captured by 1166 # the regexp 1167 log('From %s: (regexp=%s) %r', client, num_id, params) 1168 self.server.handle_msg(client, num_id, *params) 1169 1170 elif msg_id == ERROR: 1171 # num_id: not meaningful, parameter=error msg 1172 warn('Client %r sent a protocol error: %s', client, params) 1173 # TODO: send BYE and close connection, as in ivy.c? 1174 1175 elif msg_id == START_INIT: 1176 # num_id: tcp port number, parameter: the client's agentname 1177 err_msg = 'Client %r sent the initial subscription more than once'%client 1178 client.start_init(params) 1179 log('%s connected from %r', params, client) 1180 1181 elif msg_id == END_INIT: 1182 # num_id: not meaningful. No parameter. 1183 client.end_init() 1184 # app. callback 1185 self.server.handle_new_client(client) 1186 # send ready message 1187 self.server.send_ready_message(client) 1188 1189 elif msg_id == DIE: 1190 # num_id: not meaningful. No parameter. 1191 self.server.handle_die_message(num_id, client) 1192 1193 elif msg_id == DIRECT_MSG: 1194 # num_id: not meaningful. 1195 log('Client %r sent us a direct msg num_id:%s msg:%r', 1196 client, num_id, params) 1197 self.server.handle_direct_msg(client, num_id, params) 1198 1199 else: 1200 warn('Unhandled msg from %r: %r', client, msg) 1201 1202 except IvyIllegalStateError: 1203 raise IvyProtocolError, err_msg 1204 1205 return True
1206 1207 import threading
1208 -class IvyTimer(threading.Thread):
1209 """ 1210 An IvyTimer object is responsible for calling a function regularly. It is 1211 bound to an IvyServer and stops when its server stops. 1212 1213 Interacting with a timer object 1214 ------------------------------- 1215 1216 - Each timer gets a unique id, stored in the attribute ``id``. Note that 1217 a dead timer's id can be reassigned to another one (a dead timer is a 1218 timer that has been stopped) 1219 1220 - To start a timer, simply call its method ``start()`` 1221 1222 - To modify a timer's delay: simply assign ``timer.delay``, the 1223 modification will be taken into account after the next tick. The delay 1224 should be given in milliseconds. 1225 1226 - to stop a timer, assign ``timer.abort`` to ``True``, the timer will stop 1227 at the next tick (the callback won't be called) 1228 1229 MT-safety 1230 --------- 1231 **Please note:** ``start()`` starts a new thread; if the same function is 1232 given as the callback to different timers, that function should be 1233 prepared to be called concurrently. Specifically, if the callback 1234 accesses shared variables, they should be protected against concurrency 1235 problems (using locks e.g.). 1236 1237 """
1238 - def __init__(self, server, nbticks, delay, callback):
1239 """ 1240 Creates a new timer. After creation, call the timer's ``start()`` 1241 method to activate it. 1242 1243 :Parameters: 1244 - `server`: the `IvyServer` related to this timer --when the server 1245 stops, so does the timer. 1246 - `nbticks`: the number of repetition to make. ``0`` (zero) means: 1247 endless loop 1248 - `delay`: the delay, in milliseconds, between two ticks 1249 - `callback`: a function called at each tick. This function is 1250 called with one parameter, the timer itself 1251 1252 """ 1253 threading.Thread.__init__(self) 1254 self.server = server 1255 self.nbticks = nbticks 1256 self.delay = delay # milliseconds 1257 self.callback = callback 1258 self.abort = False 1259 self.id = id(self) 1260 self.setDaemon(server.usesDaemons)
1261
1262 - def run(self):
1263 import time 1264 ticks = -1 1265 while self.server.isAlive() and not self.abort and ticks<self.nbticks: 1266 1267 if self.nbticks: # 0 means: endless 1268 ticks += 1 1269 self.callback(self) 1270 time.sleep(self.delay/1000.0) 1271 log('IvyTimer %s terminated', id(self))
1272 1273
1274 -def is_multicast(ip):
1275 """ 1276 Tells whether the specified ip is a multicast address or not 1277 1278 :param ip: an IPv4 address in dotted-quad string format, for example 1279 192.168.2.3 1280 """ 1281 return int(ip.split('.')[0]) in range(224,239)
1282
1283 -def decode_ivybus(ivybus=None):
1284 """ 1285 Transforms the supplied string into the corrersponding broadcast address 1286 and port 1287 1288 :param ivybus: if ``None`` or empty, defaults to environment variable 1289 ``IVYBUS`` 1290 1291 :return: a tuple made of (broadcast address, port number). For example: 1292 :: 1293 1294 >>> print decode_ivybus('192.168.12:2010') 1295 ('192.168.12.255', 2010) 1296 1297 """ 1298 if not ivybus: 1299 import os 1300 ivybus = os.getenv('IVYBUS', DEFAULT_IVYBUS) 1301 1302 broadcast, port = ivybus.split(':', 1) 1303 port = int(port) 1304 broadcast = broadcast.strip('.') 1305 broadcast += '.' + '.'.join( ['255',]*(4-len(broadcast.split('.')))) 1306 # if broadcast is multicast it had 4 elements -> previous line added a '.' 1307 broadcast = broadcast.strip('.') 1308 debug('Decoded ivybus %s:%s', broadcast, port) 1309 return broadcast, port
1310 1311 1312 1313 if __name__=='__main__': 1314 s=IvyServer(agent_name='TEST_APP', 1315 ready_msg="[Youkou]") 1316 s.start() 1317 import time 1318
1319 - def dflt_fct(*args):
1320 log("DFLT_FCT: Received: %r", args)
1321 1322 time.sleep(1) 1323 1324 1325 for regexp in ('^test .*', '^test2 (.*)$', 'test3 ([^-]*)-?(.*)', '(.*)'): 1326 s.bind_msg(dflt_fct, regexp) 1327 time.sleep(1) 1328 1329 1330 s.send_msg('glop pas glop -et paf') 1331 s.send_msg('glosp pas glop -et paf') 1332 time.sleep(1000000) 1333