forked from Netflix/metaflow
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathlifecycle.dot
208 lines (173 loc) · 8.26 KB
/
lifecycle.dot
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
digraph Metaflow {
/*
LEGEND
palegreen2: environment
lightblue2: decorator
tan: command
lightgoldenrod1: metadata
lightpink2: function call
grey78: event / change in control
*/
graph [fontsize=10, fontname="Noto Mono"]
node [width=2.5,
height=1,
shape=record,
fontname="Noto Mono",
style=filled]
edge [fontname="Nimbus Mono L"]
subgraph cluster_init {
label="Initialization"
labeljust=l
fontsize=14
validate_env [label="{environment|validate_environment}", fillcolor=palegreen2]
flow_init [label="{decorator|flow_init}", fillcolor=lightblue2]
step_init [label="{decorator|step_init}", fillcolor=lightblue2]
choose_command [shape="circle", label="Choose\nCommand", width=1, fillcolor=grey78]
}
subgraph cluster_package {
label="Code Package"
labeljust=l
fontsize=14
validate_dag [label="{graph|validate}", fillcolor=lightpink2]
init_environment [label="{environment|init_environment}", fillcolor=palegreen2]
package_init [label="{decorator|package_init}", fillcolor=lightblue2]
add_custom_package [label="{decorator|add_to_package}", fillcolor=lightblue2]
add_to_package [label="{environment|add_to_package}", fillcolor=palegreen2]
package [label="{package|create}", fillcolor=lightpink2]
}
subgraph cluster_local_run {
label="Local Run"
labeljust=l
fontsize=14
command_run [label="{command|run}", fillcolor=tan]
new_run_id [label="{metadata|new_run_id}", fillcolor=lightgoldenrod1]
runtime_init [label="{decorator|runtime_init}", fillcolor=lightblue2]
local_params [label="{runtime|persist_constants}", fillcolor=lightpink2]
start_run_heartbeat [label="{metadata|start_run_heartbeat}", fillcolor=lightgoldenrod1]
schedule_local_task [shape="circle", label="Schedule\nTask", width=1, fillcolor=grey78]
runtime_finished [label="{decorator|runtime_finished}", fillcolor=lightblue2]
stop_run_heartbeat [label="{metadata|stop_run_heartbeat}", fillcolor=lightgoldenrod1]
}
subgraph cluster_init_deuce {
label="Initialization"
labeljust=l
fontsize=14
validate_env_deuce [label="{environment|validate_environment}", fillcolor=palegreen2]
flow_init_deuce [label="{decorator|flow_init}", fillcolor=lightblue2]
step_init_deuce [label="{decorator|step_init}", fillcolor=lightblue2]
choose_command_deuce [shape="circle", label="Choose\nCommand", width=1, fillcolor=grey78]
}
subgraph cluster_stepfunctions_deploy {
label="Deploy to AWS Step Functions"
labeljust=l
fontsize=14
stepfunctions_create [label="{command|step-functions create}", fillcolor=tan]
push_to_stepfunctions [shape="circle", label="Push to AWS\nStep Functions", width=1, fillcolor=grey78]
}
subgraph cluster_batch {
label="Launch on AWS Batch"
labeljust=l
fontsize=14
batch_step [label="{command|batch step}", fillcolor=tan]
launch_batch [label="{AWS Batch|launch_job}", fillcolor=lightpink2]
local_bootstrap_batch [shape="circle", label="Bootstrap\nAWS Batch", width=1, fillcolor=grey78]
}
subgraph cluster_stepfunctions_run {
label="AWS Step Functions Trigger"
labeljust=l
fontsize=14
stepfunctions_trigger [label="{command|step-functions trigger}", fillcolor=tan]
stepfunctions_run [label="{AWS Step Functions|start_execution}", fillcolor=lightpink2]
stepfunctions_bootstrap_batch [shape="circle", label="Bootstrap\nAWS Batch", width=1, fillcolor=grey78]
stepfunctions_init [label="{command|init}" fillcolor=tan]
stepfunctions_params [label="{runtime|persist_constants}", fillcolor=lightpink2]
stepfunctions_task [shape="circle", label="Execute\nTask", width=1, fillcolor=grey78]
}
subgraph cluster_local_task {
label="Initialize Local Task"
labeljust=l
fontsize=14
new_local_task [label="{metadata|new_task_id}", fillcolor=lightgoldenrod1]
runtime_task_created [label="{decorator|runtime_task_created}", fillcolor=lightblue2]
runtime_step_cli [label="{decorator|runtime_step_cli}", fillcolor=lightblue2]
launch_local [shape="circle", label="Execute\nTask", width=1, fillcolor=grey78]
}
subgraph cluster_task {
label="Execute Task"
labeljust=l
fontsize=14
task_entry [label="{command|step}" fillcolor=tan]
register_run [label="{metadata|register_run_id}", fillcolor=lightgoldenrod1]
register_task [label="{metadata|register_task_id}", fillcolor=lightgoldenrod1]
start_task_heartbeat [label="{metadata|start_task_heartbeat}", fillcolor=lightgoldenrod1]
task_pre_step [label="{decorator|task_pre_step}", fillcolor=lightblue2]
task_decorate [label="{decorator|task_decorate}", fillcolor=lightblue2]
user_code [shape="circle", label="Execute\nUser Code", width=1, fillcolor=grey78]
task_post_step [label="{decorator|task_post_step}", fillcolor=lightblue2]
task_exception [label="{decorator|task_exception}", fillcolor=lightblue2]
persist_artifacts [label="{datastore|persist}", fillcolor=lightpink2]
stop_task_heartbeat [label="{metadata|stop_task_heartbeat}", fillcolor=lightgoldenrod1]
register_artifacts [label="{metadata|register_artifacts}", fillcolor=lightgoldenrod1]
task_finished [label="{decorator|task_finished}", fillcolor=lightblue2]
}
/* initialize */
validate_env -> flow_init
flow_init -> step_init
step_init -> choose_command
choose_command -> validate_dag
validate_env_deuce -> flow_init_deuce
flow_init_deuce -> step_init_deuce
step_init_deuce -> choose_command_deuce
/* package */
validate_dag -> init_environment
init_environment -> package_init
package_init -> add_custom_package
add_custom_package -> add_to_package
add_to_package -> package
package -> command_run
package -> stepfunctions_create
/* stepfunctions deploy */
stepfunctions_create -> push_to_stepfunctions
/* local run */
command_run -> new_run_id
new_run_id -> runtime_init
runtime_init -> local_params
local_params -> start_run_heartbeat
start_run_heartbeat -> schedule_local_task
schedule_local_task -> new_local_task [label="for each task"]
schedule_local_task -> runtime_finished
runtime_finished -> stop_run_heartbeat [label="flow finished"]
/* local task */
new_local_task -> runtime_task_created
runtime_task_created -> runtime_step_cli
runtime_step_cli -> launch_local
launch_local -> validate_env_deuce
choose_command_deuce -> task_entry [label="local task"]
choose_command_deuce -> batch_step [label="AWS Batch task"]
/* batch run */
batch_step -> launch_batch
launch_batch -> local_bootstrap_batch
local_bootstrap_batch -> validate_env_deuce
/* step functions run */
stepfunctions_trigger -> stepfunctions_run
stepfunctions_run -> stepfunctions_bootstrap_batch
stepfunctions_bootstrap_batch -> stepfunctions_init [label="AWS Step Functions start"]
stepfunctions_bootstrap_batch -> stepfunctions_task [label="AWS Step Functions task"]
stepfunctions_init -> stepfunctions_params
stepfunctions_params -> stepfunctions_task
stepfunctions_task -> validate_env_deuce
/* task */
task_entry -> register_run
register_run -> register_task
register_task -> start_task_heartbeat
start_task_heartbeat -> task_pre_step
task_pre_step -> task_decorate
task_decorate -> user_code
user_code -> task_post_step [label="Task success"]
user_code -> task_exception [label="Task failed"]
task_post_step -> persist_artifacts
task_exception -> persist_artifacts
persist_artifacts -> stop_task_heartbeat
stop_task_heartbeat -> register_artifacts
register_artifacts -> task_finished
}