@@ -38,7 +38,7 @@ def __init__(self, settings: BaseSettings) -> None:
38
38
self ._expiration_max_items = 100
39
39
self ._expiration_secs : int = settings .getint ('HTTPCACHE_EXPIRATION_SECS' )
40
40
self ._spider : Spider | None = None
41
- self ._kv : KeyValueStore | None = None
41
+ self ._kvs : KeyValueStore | None = None
42
42
self ._fingerprinter : RequestFingerprinterProtocol | None = None
43
43
self ._async_thread : AsyncThread | None = None
44
44
@@ -47,19 +47,19 @@ def open_spider(self, spider: Spider) -> None:
47
47
logger .debug ('Using Apify key value cache storage' , extra = {'spider' : spider })
48
48
self ._spider = spider
49
49
self ._fingerprinter = spider .crawler .request_fingerprinter
50
- kv_name = f'httpcache-{ spider .name } '
50
+ kvs_name = f'httpcache-{ spider .name } '
51
51
52
- async def open_kv () -> KeyValueStore :
52
+ async def open_kvs () -> KeyValueStore :
53
53
config = Configuration .get_global_configuration ()
54
54
if config .is_at_home :
55
55
storage_client = ApifyStorageClient .from_config (config )
56
- return await KeyValueStore .open (name = kv_name , storage_client = storage_client )
57
- return await KeyValueStore .open (name = kv_name )
56
+ return await KeyValueStore .open (name = kvs_name , storage_client = storage_client )
57
+ return await KeyValueStore .open (name = kvs_name )
58
58
59
59
logger .debug ("Starting background thread for cache storage's event loop" )
60
60
self ._async_thread = AsyncThread ()
61
- logger .debug (f"Opening cache storage's { kv_name !r} key value store" )
62
- self ._kv = self ._async_thread .run_coro (open_kv ())
61
+ logger .debug (f"Opening cache storage's { kvs_name !r} key value store" )
62
+ self ._kvs = self ._async_thread .run_coro (open_kvs ())
63
63
64
64
def close_spider (self , _ : Spider , current_time : int | None = None ) -> None :
65
65
"""Close the cache storage for a spider."""
@@ -71,28 +71,28 @@ def close_spider(self, _: Spider, current_time: int | None = None) -> None:
71
71
if current_time is None :
72
72
current_time = int (time ())
73
73
74
- async def expire_kv () -> None :
75
- if self ._kv is None :
74
+ async def expire_kvs () -> None :
75
+ if self ._kvs is None :
76
76
raise ValueError ('Key value store not initialized' )
77
77
i = 0
78
- async for item in self ._kv .iterate_keys ():
79
- value = await self ._kv .get_value (item .key )
78
+ async for item in self ._kvs .iterate_keys ():
79
+ value = await self ._kvs .get_value (item .key )
80
80
try :
81
81
gzip_time = read_gzip_time (value )
82
82
except Exception as e :
83
83
logger .warning (f'Malformed cache item { item .key } : { e } ' )
84
- await self ._kv .set_value (item .key , None )
84
+ await self ._kvs .set_value (item .key , None )
85
85
else :
86
86
if self ._expiration_secs < current_time - gzip_time :
87
87
logger .debug (f'Expired cache item { item .key } ' )
88
- await self ._kv .set_value (item .key , None )
88
+ await self ._kvs .set_value (item .key , None )
89
89
else :
90
90
logger .debug (f'Valid cache item { item .key } ' )
91
91
if i == self ._expiration_max_items :
92
92
break
93
93
i += 1
94
94
95
- self ._async_thread .run_coro (expire_kv ())
95
+ self ._async_thread .run_coro (expire_kvs ())
96
96
97
97
logger .debug ('Closing cache storage' )
98
98
try :
@@ -108,13 +108,13 @@ def retrieve_response(self, _: Spider, request: Request, current_time: int | Non
108
108
"""Retrieve a response from the cache storage."""
109
109
if self ._async_thread is None :
110
110
raise ValueError ('Async thread not initialized' )
111
- if self ._kv is None :
111
+ if self ._kvs is None :
112
112
raise ValueError ('Key value store not initialized' )
113
113
if self ._fingerprinter is None :
114
114
raise ValueError ('Request fingerprinter not initialized' )
115
115
116
116
key = self ._fingerprinter .fingerprint (request ).hex ()
117
- value = self ._async_thread .run_coro (self ._kv .get_value (key ))
117
+ value = self ._async_thread .run_coro (self ._kvs .get_value (key ))
118
118
119
119
if value is None :
120
120
logger .debug ('Cache miss' , extra = {'request' : request })
@@ -140,7 +140,7 @@ def store_response(self, _: Spider, request: Request, response: Response) -> Non
140
140
"""Store a response in the cache storage."""
141
141
if self ._async_thread is None :
142
142
raise ValueError ('Async thread not initialized' )
143
- if self ._kv is None :
143
+ if self ._kvs is None :
144
144
raise ValueError ('Key value store not initialized' )
145
145
if self ._fingerprinter is None :
146
146
raise ValueError ('Request fingerprinter not initialized' )
@@ -153,7 +153,7 @@ def store_response(self, _: Spider, request: Request, response: Response) -> Non
153
153
'body' : response .body ,
154
154
}
155
155
value = to_gzip (data )
156
- self ._async_thread .run_coro (self ._kv .set_value (key , value ))
156
+ self ._async_thread .run_coro (self ._kvs .set_value (key , value ))
157
157
158
158
159
159
def to_gzip (data : dict , mtime : int | None = None ) -> bytes :
0 commit comments