Skip to content

Commit 1a20247

Browse files
MintCat98MintCat98
committed
refactor(BA-502): Add the skeleton interface of vfolder CRUD handlers in storage-proxy (#3434)
1 parent 219cf14 commit 1a20247

File tree

9 files changed

+636
-252
lines changed

9 files changed

+636
-252
lines changed

‎src/ai/backend/storage/api/vfolder/BUILD

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
python_sources()

‎src/ai/backend/storage/api/vfolder/client.py

-58
This file was deleted.

‎src/ai/backend/storage/api/vfolder/manager_handler.py

+67-87
Original file line numberDiff line numberDiff line change
@@ -1,108 +1,88 @@
1-
from typing import AsyncContextManager, Type, TypeVar, cast
2-
3-
import weakref
1+
import uuid
42

53
from aiohttp import web
64

7-
8-
from ai.backend.storage.api.manager import token_auth_middleware
95
from ai.backend.storage.api.vfolder.manager_service import VFolderService
10-
from ai.backend.storage.context import RootContext, PrivateContext
11-
12-
13-
T = TypeVar("T")
6+
from ai.backend.storage.api.vfolder.types import (
7+
QuotaConfigModel,
8+
QuotaIDModel,
9+
VFolderCloneModel,
10+
VFolderIDModel,
11+
VFolderInfoRequestModel,
12+
VolumeIDModel,
13+
)
1414

1515

1616
class VFolderHandler:
1717
def __init__(self, storage_service: VFolderService) -> None:
1818
self.storage_service = storage_service
1919

2020
async def get_volume(self, request: web.Request) -> web.Response:
21-
return web.Response(text="Volume info")
21+
data = await request.json()
22+
data["volume_id"] = uuid.UUID(data["volume_id"])
23+
req = VolumeIDModel(**data)
24+
result = await self.storage_service.get_volume(req)
25+
return web.json_response(result)
2226

2327
async def get_volumes(self, request: web.Request) -> web.Response:
24-
return web.Response(text="Volumes list")
28+
result = await self.storage_service.get_volumes()
29+
# Assume that the volume_dict is a dictionary of VolumeInfoModel objects
30+
volumes_dict = result.volumes
31+
volumes_dict = {k: v for k, v in volumes_dict.items()}
32+
return web.json_response(volumes_dict)
33+
34+
async def create_quota_scope(self, request: web.Request) -> web.Response:
35+
data = await request.json()
36+
data["volume_id"] = uuid.UUID(data["volume_id"])
37+
req = QuotaConfigModel(**data)
38+
await self.storage_service.create_quota_scope(req)
39+
return web.Response(status=204)
40+
41+
async def get_quota_scope(self, request: web.Request) -> web.Response:
42+
data = await request.json()
43+
data["volume_id"] = uuid.UUID(data["volume_id"])
44+
req = QuotaIDModel(**data)
45+
result = await self.storage_service.get_quota_scope(req)
46+
return web.json_response(result)
47+
48+
async def update_quota_scope(self, request: web.Request) -> web.Response:
49+
data = await request.json()
50+
data["volume_id"] = uuid.UUID(data["volume_id"])
51+
req = QuotaConfigModel(**data)
52+
await self.storage_service.update_quota_scope(req)
53+
return web.Response(status=204)
54+
55+
async def delete_quota_scope(self, request: web.Request) -> web.Response:
56+
data = await request.json()
57+
data["volume_id"] = uuid.UUID(data["volume_id"])
58+
req = QuotaIDModel(**data)
59+
await self.storage_service.delete_quota_scope(req)
60+
return web.Response(status=204)
2561

2662
async def create_vfolder(self, request: web.Request) -> web.Response:
27-
return web.Response(text="VFolder created", status=204)
63+
data = await request.json()
64+
data["volume_id"] = uuid.UUID(data["volume_id"])
65+
req = VFolderIDModel(**data)
66+
await self.storage_service.create_vfolder(req)
67+
return web.Response(status=204)
2868

2969
async def clone_vfolder(self, request: web.Request) -> web.Response:
30-
return web.Response(text="VFolder cloned", status=204)
31-
32-
async def get_vfolders(self, request: web.Request) -> web.Response:
33-
return web.Response(text="VFolders list")
70+
data = await request.json()
71+
data["volume_id"] = uuid.UUID(data["volume_id"])
72+
req = VFolderCloneModel(**data)
73+
await self.storage_service.clone_vfolder(req)
74+
return web.Response(status=204)
3475

3576
async def get_vfolder_info(self, request: web.Request) -> web.Response:
36-
return web.Response(text="VFolder info")
37-
38-
async def update_vfolder_options(self, request: web.Request) -> web.Response:
39-
return web.Response(text="VFolder updated")
77+
data = await request.json()
78+
data["volume_id"] = uuid.UUID(data["volume_id"])
79+
req = VFolderInfoRequestModel(**data)
80+
result = await self.storage_service.get_vfolder_info(req)
81+
return web.json_response(result)
4082

4183
async def delete_vfolder(self, request: web.Request) -> web.Response:
42-
return web.Response(text="VFolder deleted", status=202)
43-
44-
# async def _extract_params(self, request: web.Request, schema: Type[T]) -> AsyncContextManager[T]:
45-
# """
46-
# pydantic에서 자주 활용되는 방식 찾아보기
47-
# middleware에서 처리하는 방식도 고려해보기"""
48-
# data = await request.json()
49-
# try:
50-
# params = schema(**data)
51-
# except TypeError as e:
52-
# raise web.HTTPBadRequest( # Backend.AI의 Exception 패키지 확인하기
53-
# reason=f"Invalid request parameters: {str(e)}"
54-
# )
55-
# # 데이터 검증을 위에서 같이 진행해서 check_params 제외함
56-
# return cast(AsyncContextManager[T], params)
57-
58-
59-
async def init_manager_app(ctx: RootContext) -> web.Application:
60-
storage_service = VFolderService(ctx)
61-
storage_handler = VFolderHandler(storage_service)
62-
63-
app = web.Application(
64-
middlewares=[
65-
token_auth_middleware,
66-
],
67-
)
68-
app["ctx"] = ctx
69-
app["app_ctx"] = PrivateContext(
70-
deletion_tasks=weakref.WeakValueDictionary())
71-
72-
# Volume
73-
app.router.add_route(
74-
"POST", "/volumes/{volume_id}", storage_handler.get_volume)
75-
app.router.add_route(
76-
"GET", "/volumes", storage_handler.get_volumes)
77-
# VFolder
78-
app.router.add_route(
79-
"POST", "/volumes/{volume_id}/vfolders/", storage_handler.create_vfolder
80-
)
81-
app.router.add_route(
82-
"POST", "/volumes/{volume_id}/vfolders/{vfolder_id}/clone", storage_handler.clone_vfolder)
83-
app.router.add_route(
84-
"GET", "/volumes/{volume_id}/vfolders", storage_handler.get_vfolders
85-
)
86-
app.router.add_route(
87-
"GET", "/volumes/{volume_id}/vfolders/{vfolder_id}", storage_handler.get_vfolder_info
88-
)
89-
app.router.add_route(
90-
"PUT", "/volumes/{volume_id}/vfolders/{vfolder_id}", storage_handler.update_vfolder_options
91-
)
92-
app.router.add_route(
93-
"DELETE", "/volumes/{volume_id}/vfolders/{vfolder_id}", storage_handler.delete_vfolder
94-
)
95-
96-
# evd = ctx.event_dispatcher
97-
# evd.subscribe(
98-
# DoVolumeMountEvent,
99-
# storage_service.handle_volume_mount,
100-
# name="storage.volume.mount"
101-
# )
102-
# evd.subscribe(
103-
# DoVolumeUnmountEvent,
104-
# storage_service.handle_volume_umount,
105-
# name="storage.volume.umount"
106-
# )
107-
108-
return app
84+
data = await request.json()
85+
data["volume_id"] = uuid.UUID(data["volume_id"])
86+
req = VFolderIDModel(**data)
87+
await self.storage_service.delete_vfolder(req)
88+
return web.Response(status=202)

‎src/ai/backend/storage/api/vfolder/manager_service.py

+63-56
Original file line numberDiff line numberDiff line change
@@ -1,83 +1,90 @@
1-
import asyncio
2-
from pathlib import Path
3-
from typing import AsyncIterator, Mapping, Protocol, Dict, Any
4-
import weakref
5-
6-
from ai.backend.common.events import DoVolumeMountEvent, DoVolumeUnmountEvent
7-
from ai.backend.common.types import VFolderID
8-
from ai.backend.storage.abc import AbstractVolume
9-
from ai.backend.storage.api.vfolder.types import VFolderData, VolumeBaseData
10-
from ai.backend.storage.context import RootContext
11-
from ai.backend.storage.types import VolumeInfo
1+
from pathlib import Path, PurePath
2+
from typing import Protocol
3+
4+
from ai.backend.common.types import BinarySize
5+
from ai.backend.storage.api.vfolder.types import (
6+
QuotaConfigModel,
7+
QuotaIDModel,
8+
QuotaScopeInfoModel,
9+
VFolderCloneModel,
10+
VFolderIDModel,
11+
VFolderInfoModel,
12+
VFolderInfoRequestModel,
13+
VolumeIDModel,
14+
VolumeInfoListModel,
15+
VolumeInfoModel,
16+
)
17+
from ai.backend.storage.types import CapacityUsage, TreeUsage
1218

1319

1420
class VFolderServiceProtocol(Protocol):
15-
async def get_volume(self, volume_data: VolumeBaseData) -> AsyncIterator[AbstractVolume]:
21+
async def get_volume(self, volume_data: VolumeIDModel) -> VolumeInfoModel:
1622
"""by volume_id"""
1723
...
1824

19-
async def get_volumes(self) -> Mapping[str, VolumeInfo]:
20-
...
25+
async def get_volumes(self) -> VolumeInfoListModel: ...
2126

22-
async def create_vfolder(self, volume_id: str, vfolder_id: VFolderID, options: VFolderOptions) -> None:
23-
...
27+
async def create_quota_scope(self, quota_config_data: QuotaConfigModel) -> None: ...
2428

25-
async def clone_vfolder(self, volume_id: str, vfolder_id: VFolderID, new_vfolder_id: VFolderID, options: VFolderOptions) -> None:
26-
...
29+
async def get_quota_scope(self, quota_data: QuotaIDModel) -> QuotaScopeInfoModel: ...
2730

28-
async def get_vfolders(self, volume_id: str) -> list[Dict[str, Any]]:
29-
...
31+
async def update_quota_scope(self, quota_config_data: QuotaConfigModel) -> None: ...
3032

31-
async def get_vfolder_info(self, volume_id: str, vfolder_id: VFolderID) -> Dict[str, Any]:
33+
async def delete_quota_scope(self, quota_data: QuotaIDModel) -> None:
34+
"""Previous: unset_quota"""
3235
...
3336

34-
"""TODO: options type 정의 필요
35-
create 시와 필드가 겹친다면 따로 정의 X"""
37+
async def create_vfolder(self, vfolder_data: VFolderIDModel) -> None: ...
3638

37-
async def update_vfolder_options(self, volume_id: str, vfolder_id: VFolderID, options: ...) -> None:
38-
...
39+
async def clone_vfolder(self, vfolder_clone_data: VFolderCloneModel) -> None: ...
3940

40-
async def delete_vfolder(self, volume_id: str, vfolder_id: VFolderID) -> None:
41+
async def get_vfolder_info(self, vfolder_info: VFolderInfoRequestModel) -> VFolderInfoModel:
42+
# Integration: vfolder_mount, metadata, vfolder_usage, vfolder_used_bytes, vfolder_fs_usage
4143
...
4244

45+
async def delete_vfolder(self, vfolder_data: VFolderIDModel) -> None: ...
4346

44-
class VFolderService(VFolderServiceProtocol):
45-
def __init__(self, ctx: RootContext) -> None:
46-
self.ctx = ctx
47-
48-
async def get_volume(self, volume_data: VolumeBaseData) -> AsyncIterator[AbstractVolume]:
49-
...
5047

51-
async def get_volumes(self) -> Mapping[str, VolumeInfo]:
52-
...
48+
class VFolderService:
49+
async def get_volume(self, volume_data: VolumeIDModel) -> VolumeInfoModel:
50+
return VolumeInfoModel(
51+
volume_id=volume_data.volume_id,
52+
backend="default-backend",
53+
path=Path("/default/path"),
54+
fsprefix=PurePath("/fsprefix"),
55+
capabilities=["read", "write"],
56+
options={"option1": "value1"},
57+
)
5358

54-
async def handle_volume_mount(self, event: DoVolumeMountEvent) -> None:
55-
...
59+
async def get_volumes(self) -> VolumeInfoListModel:
60+
return VolumeInfoListModel(volumes={})
5661

57-
async def handle_volume_umount(self, event: DoVolumeUnmountEvent) -> None:
58-
...
62+
async def create_quota_scope(self, quota_config_data: QuotaConfigModel) -> None:
63+
return None
5964

60-
async def create_vfolder(self, vfolder_data: VFolderData) -> None:
61-
...
65+
async def get_quota_scope(self, quota_data: QuotaIDModel) -> QuotaScopeInfoModel:
66+
return QuotaScopeInfoModel(used_bytes=0, limit_bytes=0)
6267

63-
async def clone_vfolder(self, vfolder_data: VFolderData, new_vfolder_id: VFolderID) -> None:
64-
...
68+
async def update_quota_scope(self, quota_config_data: QuotaConfigModel) -> None:
69+
return None
6570

66-
async def get_vfolders(self, volume_id: str) -> list[Dict[str, Any]]:
67-
...
71+
async def delete_quota_scope(self, quota_data: QuotaIDModel) -> None:
72+
return None
6873

69-
async def get_vfolder_info(self, volume_id: str, vfolder_id: VFolderID) -> Dict[str, Any]:
70-
...
74+
async def create_vfolder(self, vfolder_data: VFolderIDModel) -> None:
75+
return None
7176

72-
async def update_vfolder_options(self, volume_id: str, vfolder_id: VFolderID, options: ...) -> None:
73-
...
77+
async def clone_vfolder(self, vfolder_clone_data: VFolderCloneModel) -> None:
78+
return None
7479

75-
async def _delete_vfolder(
76-
self,
77-
vfolder_data: VFolderData,
78-
task_map: weakref.WeakValueDictionary[VFolderID, asyncio.Task]
79-
) -> None:
80-
...
80+
async def get_vfolder_info(self, vfolder_info: VFolderInfoRequestModel) -> VFolderInfoModel:
81+
return VFolderInfoModel(
82+
vfolder_mount=Path("/mount/point"),
83+
vfolder_metadata=b"",
84+
vfolder_usage=TreeUsage(file_count=0, used_bytes=0),
85+
vfolder_used_bytes=BinarySize(0),
86+
vfolder_fs_usage=CapacityUsage(used_bytes=0, capacity_bytes=0),
87+
)
8188

82-
async def delete_vfolder(self, vfolder_data: VFolderData) -> None:
83-
...
89+
async def delete_vfolder(self, vfolder_data: VFolderIDModel) -> None:
90+
return None

0 commit comments

Comments
 (0)