Skip to content

Commit c9109be

Browse files
committed
Dont choose null task
1 parent 49a6b14 commit c9109be

File tree

1 file changed

+5
-4
lines changed

1 file changed

+5
-4
lines changed

cpueff-goweb/spark/cpueff_stepchain_goweb.py

+5-4
Original file line numberDiff line numberDiff line change
@@ -70,9 +70,9 @@ def udf_step_extract(row):
7070
_fwjr_id = row['meta_data']['fwjr_id']
7171
_jobtype = row['meta_data']['jobtype']
7272
_ts = row['meta_data']['ts']
73-
if 'steps' in row and row['steps']:
73+
if 'steps' in row and row['steps'] is not None:
7474
for step in filter(None, row['steps']):
75-
if ('name' in step) and step['name'].lower().startswith('cmsrun'):
75+
if 'name' in step and step['name'].lower().startswith('cmsrun'):
7676
step_res = {'Task': _task_name, 'ts': _ts, 'fwjr_id': _fwjr_id, 'JobType': _jobtype}
7777

7878
count += 1
@@ -88,11 +88,11 @@ def udf_step_extract(row):
8888
step_res['threads_total_job_time'] = step_res['job_time'] * step_res['nthreads']
8989
except Exception:
9090
step_res['threads_total_job_time'] = None
91-
if step['output']:
91+
if 'output' in step and step['output'] is not None:
9292
for outx in step['output']:
9393
if outx['acquisitionEra']:
9494
step_res['acquisition_era'].append(outx['acquisitionEra'])
95-
if 'performance' in step:
95+
if 'performance' in step and step['performance'] is not None:
9696
performance = step['performance']
9797
if 'storage' in performance:
9898
if 'writeTotalMB' in performance['storage']:
@@ -174,6 +174,7 @@ def main(start_date, end_date, hdfs_out_dir, last_n_days):
174174
.filter(f"""data.meta_data.jobstate='success'
175175
AND data.wmats >= {start_date.timestamp()}
176176
AND data.wmats < {end_date.timestamp()}
177+
AND data.task IS NOT NULL
177178
""")
178179
.filter(col('data.meta_data.jobtype').isin(_PROD_CMS_JOB_TYPES_FILTER))
179180
)

0 commit comments

Comments
 (0)