Client receiving two separate message as one with python socket

Question

I define the server class as follows (redacted):

class server:
 def __init__( self, ip = "", port = 0 ):
 self.SetAddress( ip, port )
 self.__players = []
 def __SetSocket( self, blocking = 0, queue = 4 ):
 self.__listener = socket.socket( socket.AF_INET, socket.SOCK_STREAM )
 self.__listener.bind( self.GetAddress() )
 self.__listener.setblocking( blocking )
 self.__listener.setsockopt( socket.SOL_SOCKET, socket.SO_REUSEADDR, 1 )
 self.__listener.listen( queue )
 self.__listener.settimeout( 5 )
 self.__read, self.__write, self.__error = [ self.__listener ], [], []
 def __AddClient( self, source ):
 c, a = source.accept()
 c.settimeout( 5 )
 self.__read.append( c )
 send( c, "Welcome!" )
 print a, "Connection established"
 return
 def __AddPlayer( self, source, nick ):
 if len( self.__players ) == 4:
 send( source, ('Error', "4 players already connected.") )
 self.__read.remove( source )
 return
 self.__players.append( nick )
 send( source, ('ID', self.__players.index(nick)) )
 def __RemovePlayer( self, source, gamer_id ):
 self.__players.pop( gamer_id )
 self.__read.remove( source )
 source.close()
 def __Connect( self ):
 joining = True
 while joining:
 r, w, x = select( self.__read, self.__write, self.__error, 0 )
 for s in r:
 if s is self.__listener:
 self.__AddClient( s )
 else:
 data = receive( s )
 if data:
 print data, s.getpeername()
 if self.__MaintainPlayers( s, data ):
 pass
 if len( self.__players ) == 4:
 joining = False
 return
 def __MaintainPlayers( self, source, data ):
 if data[0] == "Nick":
 self.__AddPlayer( source, data[1] )
 return True
 elif data[0] == "Quit":
 self.__RemovePlayer( source, data[1] )
 return True
 return False
 def run( self ):
 self.__SetSocket( 1, 4 )
 print "Waiting for players."
 self.__Connect()

where, the send and receive functions are as follows:

def send( channel, message ):
 try:
 channel.send( json.dumps(message) )
 return True
 except OSError as e:
 print e
 return False
def receive( channel, packet_size = 64 ):
 try:
 data = channel.recv( int(packet_size) )
 if not data:
 return None
 print data
 return json.loads( data.strip() )
 except OSError as e:
 print e
 return False

The client class is pretty simple (redacted):

class client:
 def __init__( self, name, srvIP, srvPort ):
 ip = socket.gethostbyname( socket.gethostname() )
 self.__server_address = self.__server_ip, self.__server_port = srvIP, srvPort
 self.__ID = None
 self.__nick = name
 self.__SetListener()
 def __SetListener( self ):
 self.__listener = socket.socket( socket.AF_INET, socket.SOCK_STREAM )
 self.__listener.settimeout( 5 )
 try:
 self.__listener.connect( self.__server_address )
 except Exception, e:
 print "Unable to connect", e
 raise e
 print "Connected to %s:%d." % self.__server_address
 send( self.__listener, ("Nick", self.__nick) )
 def run( self ):
 self.__read, self.__write, self.__error = [ self.__listener ], [], []
 while True:
 r, w, x = select( self.__read, self.__write, self.__error, 0 )
 for f in r:
 if f is self.__listener:
 data = receive( f )
 if data:
 print data
 if data[0] == "ID":
 self.__ID = int( data[1] )
 # More conditions

What happens is, my client objects receive the Welcome and ID message simultaneously. This throws an exception as follows:

$ client.py
Connected to 10.109.1.92:7777.
"Welcome!"["ID", 0]
Traceback (most recent call last):
 File "%PATH%\client.py", line 115, in <module>
 c.run()
 File "%PATH%\client.py", line 86, in run
 data = receive( f )
 File "%PATH%\connect.py", line 17, in receive
 return loads( data.strip() )
 File "%PYTHON%\lib\json\__init__.py", line 338, in loads
 return _default_decoder.decode(s)
 File "%PYTHON%\lib\json\decoder.py", line 369, in decode
 raise ValueError(errmsg("Extra data", s, end, len(s)))
ValueError: Extra data: line 1 column 28 - line 1 column 37 (char 27 - 36)

That is, the client receives the following as a single string:

"Welcome!"["ID", 0]

which raises an error in json.loads.

Is there some method to introduce any sort of delay between the messages?

• Ashwini Chaudhary: Currently in one of my chat application(using Twisted) I am using JSON to send messages, by JSON I mean something like {'text': "Welcome"}, {"text": ["ID", 0]} etc, this way it is easy to check whether currently the amount of data you've received can be parsed using json.loads or not. TCP has its limitation, so it may buffer the data or will break the data if it's huge.
• Ashwini Chaudhary: Sometimes you'll get multiple messages merged together and sometimes you'll get only partial messages. In case of merged messages like {'text': "Welcome"}{"text": ["ID", 0]} you can split these easily(split at }{) and then parse each of them individually, this can get a little tricky when the message itself contains }{, if it is coming from user then convert it to base64 first before sending it over to socket, For partial messages you need to accumulate the data till you've something that you can parse. This is still easier than using a plain string with some delimiter.
• hjpotter92: @AshwiniChaudhary But I am always sending json data. json.dumps in the send function.
• Ashwini Chaudhary: Yes, but as I mentioned you're not sending it in form of a dict: json.dumps('Welcome!') vs json.dumps({'text': 'Welcome!'})
• hjpotter92: @AshwiniChaudhary I'm still receiving merged message: {"text": "Welcome!"}{"ID": 0}. From what I understand, now I'd have to split this at }{? How will the split messages be parsed?

Answer 1

Alternatively, if you can guarantee your message won't contain a specific character (e.g. a null byte), you can append that to your string, and then server side, split the string on the null character:

Client:

socket.sendall(json_string + '0円')

Server:

recv_buffer = ""
while True:
 data = connection.recv(128)
 recv_buffer = recv_buffer + data
 strings = recv_buffer.split('0円')
 for s in strings[:-1]:
 print("Received: %s" % s)
 recv_buffer = strings[-1]

• Fejs: Interesting idea.

Answer 2

You need to add the size of the message when you send it so that when you receive it you can only return that message, and know that you have the whole message. There is nothing in the socket module to do this as they just implement a low level pipe.

When you send the message prefix it with the size of the message:

def send( channel, message ):
 try:
 msg = json.dumps(message)
 channel.send(struct.pack("i", len(msg)) + msg)
 return True
except OSError as e:
 print e
 return False

When you receive the message first retrieve the size, then repeatedly call recv until you have the whole message.

def receive( channel ):
 try:
 size = struct.unpack("i", channel.recv(struct.calcsize("i")))[0]
 data = ""
 while len(data) < size:
 msg = channel.recv(size - len(data))
 if not msg:
 return None
 data += msg.decode('utf-8')
 print data
 return json.loads( data.strip() )
 except OSError as e:
 print e
 return False

• hjpotter92: It does work wonderfully :). Although, struct.unpack returns a tuple which raises an error later for size - len(data). Can be fixed by accessing [0] index. Thanks once again. Sorry for the nearly 2 weeks delay. I was having some semester exams.
• Fejs: Old, but gold :-).
• TheLogicGuy: What happens if in channel.recv(struct.calcsize("i") you also get only part of the data?
• Andrew Wilkinson: Good point - there is a bug in this code. The recv for the message size should be in a while loop to make sure you have the all four bytes.