From 9043f7863cb1b0c4febc18ab9db9ad1ee22c87c7 Mon Sep 17 00:00:00 2001 From: "Yang, Bo" Date: Thu, 2 May 2024 14:33:58 -0700 Subject: [PATCH 1/3] [BugFix] Prevent the task of `_force_log` from being garbage collected The task returned by `asyncio.create_task` should not be discarded. See https://github.com/python/cpython/issues/88831 --- vllm/entrypoints/openai/api_server.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/vllm/entrypoints/openai/api_server.py b/vllm/entrypoints/openai/api_server.py index 8b3c5ea9de9..947d2db0f4b 100644 --- a/vllm/entrypoints/openai/api_server.py +++ b/vllm/entrypoints/openai/api_server.py @@ -33,6 +33,7 @@ openai_serving_completion: OpenAIServingCompletion logger = init_logger(__name__) +_running_tasks = set() @asynccontextmanager async def lifespan(app: fastapi.FastAPI): @@ -43,7 +44,9 @@ async def _force_log(): await engine.do_log_stats() if not engine_args.disable_log_stats: - asyncio.create_task(_force_log()) + task = asyncio.create_task(_force_log()) + _running_tasks.add(task) + task.add_done_callback(_running_tasks.remove) yield From 783e180a2b401b3b825746ccb645f349f9efc66d Mon Sep 17 00:00:00 2001 From: "Yang, Bo" Date: Thu, 2 May 2024 14:37:18 -0700 Subject: [PATCH 2/3] [BugFix] Fix mypy typing --- vllm/entrypoints/openai/api_server.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vllm/entrypoints/openai/api_server.py b/vllm/entrypoints/openai/api_server.py index 947d2db0f4b..f8d05a2079f 100644 --- a/vllm/entrypoints/openai/api_server.py +++ b/vllm/entrypoints/openai/api_server.py @@ -4,6 +4,7 @@ import re from contextlib import asynccontextmanager from http import HTTPStatus +from typing import Any, Set import fastapi import uvicorn @@ -33,7 +34,7 @@ openai_serving_completion: OpenAIServingCompletion logger = init_logger(__name__) -_running_tasks = set() +_running_tasks: Set[asyncio.Task[Any]] = set() @asynccontextmanager async def lifespan(app: fastapi.FastAPI): From 681e728d54e21a9cf3b72aabad3839d124f38e84 Mon Sep 17 00:00:00 2001 From: "Yang, Bo" Date: Thu, 2 May 2024 14:49:47 -0700 Subject: [PATCH 3/3] [BugFix] Fix yapf --- vllm/entrypoints/openai/api_server.py | 1 + 1 file changed, 1 insertion(+) diff --git a/vllm/entrypoints/openai/api_server.py b/vllm/entrypoints/openai/api_server.py index f8d05a2079f..f9e294af472 100644 --- a/vllm/entrypoints/openai/api_server.py +++ b/vllm/entrypoints/openai/api_server.py @@ -36,6 +36,7 @@ _running_tasks: Set[asyncio.Task[Any]] = set() + @asynccontextmanager async def lifespan(app: fastapi.FastAPI):