Skip to content
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.

Commit 6b12feb

Browse files
amankrxamankrx
committedJan 13, 2025·
Added implementation for Google Cloud Store
1 parent b1df876 commit 6b12feb

15 files changed

+2687
-2
lines changed
 

‎.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ bazel-*
22
target/
33
.vscode/
44
.zed
5+
.idea/
56
.cache
67
.terraform*
78
.config

‎Cargo.lock

+94
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎nativelink-config/examples/README.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ The value of `stores` includes top-level keys, which are user supplied names sto
4141
### Store Type
4242

4343
Once the store has been named and its object exists,
44-
the next key is the type of store. The options are `filesystem`, `memory`, `compression`, `dedup`, `fast_slow`, `verify`, and `experimental_s3_store`.
44+
the next key is the type of store. The options are `filesystem`, `memory`, `compression`, `dedup`, `fast_slow`, `verify`, `experimental_s3_store` and `experimental_gcs_store`.
4545

4646
```json5
4747
{

‎nativelink-config/examples/gcs_backend.json5

+172
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,172 @@
1+
{
2+
"stores": {
3+
"CAS_MAIN_STORE": {
4+
"verify": {
5+
"backend": {
6+
"dedup": {
7+
"index_store": {
8+
"fast_slow": {
9+
"fast": {
10+
"filesystem": {
11+
"content_path": "/tmp/nativelink/data/content_path-index",
12+
"temp_path": "/tmp/nativelink/data/tmp_path-index",
13+
"eviction_policy": {
14+
// 500mb.
15+
"max_bytes": 500000000
16+
}
17+
}
18+
},
19+
"slow": {
20+
"experimental_gcs_store": {
21+
"service_email": "nativelink-test-aman@inbound-entity-447014-k2.iam.gserviceaccount.com",
22+
// Name of the bucket to upload to.
23+
"bucket": "test-bucket-aman-nativelink",
24+
"key_prefix": "test-prefix-index/",
25+
"retry": {
26+
"max_retries": 6,
27+
"delay": 0.3,
28+
"jitter": 0.5
29+
},
30+
"max_concurrent_uploads": 10
31+
}
32+
}
33+
}
34+
},
35+
"content_store": {
36+
"compression": {
37+
"compression_algorithm": {
38+
"lz4": {}
39+
},
40+
"backend": {
41+
"fast_slow": {
42+
"fast": {
43+
"filesystem": {
44+
"content_path": "/tmp/nativelink/data/content_path-content",
45+
"temp_path": "/tmp/nativelink/data/tmp_path-content",
46+
"eviction_policy": {
47+
// 2gb.
48+
"max_bytes": 2000000000
49+
}
50+
}
51+
},
52+
"slow": {
53+
"experimental_gcs_store": {
54+
"service_email": "nativelink-test-aman@inbound-entity-447014-k2.iam.gserviceaccount.com",
55+
// Name of the bucket to upload to.
56+
"bucket": "test-bucket-aman-nativelink",
57+
"key_prefix": "test-prefix-dedup-cas/",
58+
"retry": {
59+
"max_retries": 6,
60+
"delay": 0.3,
61+
"jitter": 0.5
62+
},
63+
"max_concurrent_uploads": 10
64+
}
65+
}
66+
}
67+
}
68+
}
69+
}
70+
}
71+
},
72+
"verify_size": true
73+
}
74+
},
75+
"AC_MAIN_STORE": {
76+
"fast_slow": {
77+
"fast": {
78+
"memory": {
79+
"eviction_policy": {
80+
// 100mb.
81+
"max_bytes": 100000000
82+
}
83+
},
84+
"filesystem": {
85+
"content_path": "/tmp/nativelink/data/content_path-ac",
86+
"temp_path": "/tmp/nativelink/data/tmp_path-ac",
87+
"eviction_policy": {
88+
// 500mb.
89+
"max_bytes": 500000000
90+
}
91+
}
92+
},
93+
"slow": {
94+
"experimental_gcs_store": {
95+
"service_email": "nativelink-test-aman@inbound-entity-447014-k2.iam.gserviceaccount.com",
96+
// Name of the bucket to upload to.
97+
"bucket": "test-bucket-aman-nativelink",
98+
"key_prefix": "test-prefix-ac/",
99+
"retry": {
100+
"max_retries": 6,
101+
"delay": 0.3,
102+
"jitter": 0.5
103+
},
104+
"max_concurrent_uploads": 10
105+
}
106+
}
107+
}
108+
},
109+
},
110+
"schedulers": {
111+
"MAIN_SCHEDULER": {
112+
"simple": {
113+
"supported_platform_properties": {
114+
"cpu_count": "minimum",
115+
"memory_kb": "minimum",
116+
"network_kbps": "minimum",
117+
"disk_read_iops": "minimum",
118+
"disk_read_bps": "minimum",
119+
"disk_write_iops": "minimum",
120+
"disk_write_bps": "minimum",
121+
"shm_size": "minimum",
122+
"gpu_count": "minimum",
123+
"gpu_model": "exact",
124+
"cpu_vendor": "exact",
125+
"cpu_arch": "exact",
126+
"cpu_model": "exact",
127+
"kernel_version": "exact",
128+
"docker_image": "priority",
129+
"lre-rs": "priority"
130+
}
131+
}
132+
}
133+
},
134+
"servers": [{
135+
"listener": {
136+
"http": {
137+
"socket_address": "0.0.0.0:50051"
138+
}
139+
},
140+
"services": {
141+
"cas": {
142+
"main": {
143+
"cas_store": "CAS_MAIN_STORE"
144+
}
145+
},
146+
"ac": {
147+
"main": {
148+
"ac_store": "AC_MAIN_STORE"
149+
}
150+
},
151+
"execution": {
152+
"main": {
153+
"cas_store": "CAS_MAIN_STORE",
154+
"scheduler": "MAIN_SCHEDULER"
155+
}
156+
},
157+
"capabilities": {
158+
"main": {
159+
"remote_execution": {
160+
"scheduler": "MAIN_SCHEDULER"
161+
}
162+
}
163+
},
164+
"bytestream": {
165+
"cas_stores": {
166+
"main": "CAS_MAIN_STORE"
167+
}
168+
},
169+
"health": {}
170+
}
171+
}]
172+
}

0 commit comments

Comments
 (0)
Please sign in to comment.