Monday, 8 November 2021

Python Websocket module continues to start processes that dont get killed leading to memory issues

I have this code that runs on a server that is used to split audio files. I am using websockets for this process to be triggered the browser.

The problem is that each trigger of this splitting process creates 10 new python processes that do not get killed leading to memory errors after it running for a while. I cannot figure out why these processes get created and how I can improve the script. Right now I am using a hacky method of a cron job running once per day killing any leftover processes. Any pointers on what these processes do or why they get created and how I can kill them automatically after the script has ran would be great.

Here you can see the processes that get created each time the split function is ran: Unidentified Websocket Process

The code:

#!/usr/bin/env python


import websockets
#import aiohttp
import asyncio
from pathlib import Path
from io import BytesIO
from spleeter.separator import Separator
import sys
import tempfile
import os
from zipfile import ZipFile

async def split(websocket, path):
    print("split started")
    stems = await websocket.recv()
    audioFormat = await websocket.recv()
    bitRate = await websocket.recv()
    audio_bytes = await websocket.recv()
    separator = Separator('spleeter:'+stems+'stems')
    temp, pathname = tempfile.mkstemp()
    file_paths = []
    p = Path('/tmp/filename/'+str(pathname).split('/')[2])
    if not (os.path.exists(p)):
        print('New Path')
        with open(pathname, mode='wb') as f:
            f.write(audio_bytes)
            print('Writing to file ' + str(f) + 'with seperator: ' + str(separator))
        if audioFormat == "mp3":
            print("mp3")
            separator.separate_to_file(pathname, "/tmp/filename", codec="mp3", bitrate=bitRate)
        else:
            print("wav")
            separator.separate_to_file(pathname, "/tmp/filename")
        separator.join()
        print('Separating finished')
    oldwd = os.getcwd()
    os.chdir(p)
    for entry in os.scandir(path='.'):
        print('Appending file: ' + str(entry))
        if entry.is_file():
            file_paths.append(entry)
    with ZipFile(p / 'files.zip','w') as zip:
        for f in file_paths:
            zip.write(f)
    print('Zip created')
    os.chdir(oldwd)
    
    f=open(p / 'files.zip', "rb")
    contents = f.read()
    await websocket.send(contents)

    print('Zip sent to client')

async def main():
    print("main called")
    async with websockets.serve(
        split,
        # "0.0.0.0",
        "localhost",
        9014,
        max_size = None, 
        create_protocol=websockets.basic_auth_protocol_factory(
                realm="my dev server",
                credentials=("TEST", "CREDS"),
            )
        ):
            await asyncio.Future() # run forever

asyncio.run(main())


from Python Websocket module continues to start processes that dont get killed leading to memory issues

No comments:

Post a Comment