remote file follow using python

Question

I've created an on-demand class that can be initiated to collected any new data written to file from start_recording call until stop_recording which close the connection and retrieve the collected data so far.

It uses in test cases for obtaining relevant logs during the time where a certain operation was performed in order to verify its success.

I'm currently using this class for tests in remote machine, but would be happy to hear for any idea for improvements, correctness, etc.

import time
import paramiko
import select
from virtual_machine import utils
from multiprocessing import Queue
import multiprocess
class background():
 def __init__(self, config_file):
 self.q = Queue(1000)
 #this methods implemented elsewhere and read a config file into dictionary
 self.config = utils.get_params(config_file)
 @staticmethod
 def linesplit(socket):
 buffer_string = socket.recv(4048).decode('utf-8')
 done = False
 while not done:
 if '\n' in buffer_string:
 (line, buffer_string) = buffer_string.split('\n', 1)
 yield line + "\n"
 else:
 more = socket.recv(4048)
 if not more:
 done = True
 else:
 buffer_string = buffer_string + more.decode('utf-8')
 if buffer_string:
 yield buffer_string
 def do_tail(self, log_file):
 data = ""
 client = paramiko.SSHClient()
 client.load_system_host_keys()
 client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
 from os.path import expanduser
 home = expanduser("~")
 client.connect(self.config["ip"],self.config["username"],self.config["password"])
 grep_pattern = "grep_filter"
 remote_command = 'tail -0 -f {} '.format(log_file)
 print(remote_command)
 transport = client.get_transport()
 channel = transport.open_session()
 channel.exec_command(remote_command)
 while 1:
 try:
 rl, _, _ = select.select([channel], [], [], 0.0)
 if len(rl) > 0:
 print("ready to read")
 for line in background.linesplit(channel):
 self.q.put_nowait(line)
 except (KeyboardInterrupt, SystemExit):
 print('got ctrl+c')
 break
 client.close()
 return data
 def start_recording(self, log_file):
 q = Queue(100)
 self.p = multiprocess.Process(target=self.do_tail, args=(log_file,), daemon=True)
 self.p.start()
 def stop_recording(self):
 self.p.terminate()
 data = ""
 while not self.q.empty():
 data += self.q.get()
 return data
#example 
if __name__ == '__main__':
 x = background("config.conf")
 x.start_recording("/tmp/logfile.log")
 # doing some operation 
 data = x.stop_recording()
 print(data)

Answer 1

Your linesplit repeats a little bit of code; you could phrase it as

while True:
 buffer_bytes = socket.recv(4096)
 if len(buffer_bytes) == 0:
 break
 buffer_string = buffer_bytes.decode('utf-8')
 *lines, buffer_string = buffer_string.split('\n')
 yield from lines

'tail -0 -f {} '.format(log_file) could be simply 'tail -0 -f ' + log_file.

background should be Background.

Consider adding some PEP484 type hints.

rl, _, _ = select ...

can be

rl, *_ = select ...

The code between paramiko.SSHClient() and client.close() has no close-guarantee protection. There are various options, including a try/finally, or making your current class (or a factored-out tail-specific class) a context manager.

• Zohar81: Thanks for your comments. Btw, I've encountered where the log recording was lagging behind, and where it did, the interesting part already finished. I wonder if there's any kind of system wide lock that can block the start_recording from returning, until the subprocess reach the ready to read line. Can you recommend anything ? thanks !