Skip to content

Commit 64eab3c

Browse files
giannisdoukasgiannisdoukas
authored
Merge pull request #15 from giannisdoukas/build-in-plot
Add build in support for matplotlib figures
2 parents 6ebe351 + 90305e9 commit 64eab3c

8 files changed

+216
-71
lines changed

‎examples/intro.ipynb

+55-62
Large diffs are not rendered by default.

‎examples/new_data.png

-1.36 KB
Loading

‎examples/requirements.txt

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
pandas
2+
matplotlib

‎ipython2cwl/cwltoolextractor.py

+25-7
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
from nbformat.notebooknode import NotebookNode # type: ignore
1616

1717
from .iotypes import CWLFilePathInput, CWLBooleanInput, CWLIntInput, CWLStringInput, CWLFilePathOutput, \
18-
CWLDumpableFile, CWLDumpableBinaryFile, CWLDumpable
18+
CWLDumpableFile, CWLDumpableBinaryFile, CWLDumpable, CWLPNGPlot, CWLPNGFigure
1919
from .requirements_manager import RequirementsManager
2020

2121
with open(os.sep.join([os.path.abspath(os.path.dirname(__file__)), 'templates', 'template.dockerfile'])) as f:
@@ -64,9 +64,21 @@ class AnnotatedVariablesExtractor(ast.NodeTransformer):
6464
}
6565

6666
dumpable_mapper = {
67-
(CWLDumpableFile.__name__,): "with open('{var_name}', 'w') as f:\n\tf.write({var_name})",
68-
(CWLDumpableBinaryFile.__name__,): "with open('{var_name}', 'wb') as f:\n\tf.write({var_name})",
67+
(CWLDumpableFile.__name__,): (
68+
(None, "with open('{var_name}', 'w') as f:\n\tf.write({var_name})",),
69+
lambda node: node.target.id
70+
),
71+
(CWLDumpableBinaryFile.__name__,): (
72+
(None, "with open('{var_name}', 'wb') as f:\n\tf.write({var_name})"),
73+
lambda node: node.target.id
74+
),
6975
(CWLDumpable.__name__, CWLDumpable.dump.__name__): None,
76+
(CWLPNGPlot.__name__,): (
77+
(None, '{var_name}[-1].figure.savefig("{var_name}.png")'),
78+
lambda node: str(node.target.id) + '.png'),
79+
(CWLPNGFigure.__name__,): (
80+
('import matplotlib.pyplot as plt\nplt.figure()', '{var_name}[-1].figure.savefig("{var_name}.png")'),
81+
lambda node: str(node.target.id) + '.png'),
7082
}
7183

7284
def __init__(self, *args, **kwargs):
@@ -110,12 +122,18 @@ def _visit_input_ann_assign(self, node, annotation):
110122
return None
111123

112124
def _visit_default_dumper(self, node, dumper):
113-
dump_tree = ast.parse(dumper.format(var_name=node.target.id))
114-
self.to_dump.append(dump_tree.body)
125+
if dumper[0][0] is None:
126+
pre_code_body = []
127+
else:
128+
pre_code_body = ast.parse(dumper[0][0].format(var_name=node.target.id)).body
129+
if dumper[0][1] is None:
130+
post_code_body = []
131+
else:
132+
post_code_body = ast.parse(dumper[0][1].format(var_name=node.target.id)).body
115133
self.extracted_variables.append(_VariableNameTypePair(
116-
node.target.id, None, None, None, False, True, node.target.id)
134+
node.target.id, None, None, None, False, True, dumper[1](node))
117135
)
118-
return self.conv_AnnAssign_to_Assign(node)
136+
return [*pre_code_body, self.conv_AnnAssign_to_Assign(node), *post_code_body]
119137

120138
def _visit_user_defined_dumper(self, node):
121139
load_ctx = ast.Load()

‎ipython2cwl/iotypes.py

+47
Original file line numberDiff line numberDiff line change
@@ -158,3 +158,50 @@ class CWLDumpableBinaryFile(CWLDumpable):
158158
and at the CWL, the data, will be mapped as a output.
159159
"""
160160
pass
161+
162+
163+
class CWLPNGPlot(CWLDumpable):
164+
"""Use that annotation to define that after the assigment of that variable the plt.savefig() should
165+
be called.
166+
167+
>>> import matplotlib.pyplot as plt
168+
>>> data = [1,2,3]
169+
>>> new_data: 'CWLPNGPlot' = plt.plot(data)
170+
171+
the converter will tranform these lines to
172+
173+
>>> import matplotlib.pyplot as plt
174+
>>> data = [1,2,3]
175+
>>> new_data: 'CWLPNGPlot' = plt.plot(data)
176+
>>> plt.savefig('new_data.png')
177+
178+
179+
Note that by default if you have multiple plot statements in the same notebook will be written
180+
in the same file. If you want to write them in separates you have to do it in separate figures.
181+
To do that in your notebook you have to create a new figure before the plot command or use the CWLPNGFigure.
182+
183+
>>> import matplotlib.pyplot as plt
184+
>>> data = [1,2,3]
185+
>>> plt.figure()
186+
>>> new_data: 'CWLPNGPlot' = plt.plot(data)
187+
"""
188+
pass
189+
190+
191+
class CWLPNGFigure(CWLDumpable):
192+
"""The same with :class:`~ipython2cwl.iotypes.CWLPNGPlot` but creates new figures before plotting. Use that
193+
annotation of you don't want to write multiple graphs in the same image
194+
195+
>>> import matplotlib.pyplot as plt
196+
>>> data = [1,2,3]
197+
>>> new_data: 'CWLPNGPlot' = plt.plot(data)
198+
199+
the converter will tranform these lines to
200+
201+
>>> import matplotlib.pyplot as plt
202+
>>> data = [1,2,3]
203+
>>> plt.figure()
204+
>>> new_data: 'CWLPNGPlot' = plt.plot(data)
205+
>>> plt.savefig('new_data.png')
206+
207+
"""

‎test-requirements.txt

+2-1
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ coveralls>=2.0.0
44
virtualenv>=3.1.0
55
gitpython>=3.1.3
66
docker>=4.2.1
7-
git+https://github.com/giannisdoukas/cwltool.git#egg=cwltool
7+
cwltool==3.0.20200706173533
88
pandas==1.0.5
99
mypy
10+
matplotlib

‎tests/test_cwltoolextractor.py

+84
Original file line numberDiff line numberDiff line change
@@ -474,3 +474,87 @@ def test_AnnotatedIPython2CWLToolConverter_custom_dumpables(self):
474474
os.remove(f)
475475
except FileNotFoundError:
476476
pass
477+
478+
def test_AnnotatedIPython2CWLToolConverter_CWLPNGPlot(self):
479+
code = os.linesep.join([
480+
"import matplotlib.pyplot as plt",
481+
"new_data: 'CWLPNGPlot' = plt.plot([1,2,3,4])",
482+
])
483+
converter = AnnotatedIPython2CWLToolConverter(code)
484+
new_script = converter._wrap_script_to_method(
485+
converter._tree,
486+
converter._variables
487+
)
488+
try:
489+
os.remove('new_data.png')
490+
except FileNotFoundError:
491+
pass
492+
exec(new_script)
493+
locals()['main']()
494+
self.assertTrue(os.path.isfile('new_data.png'))
495+
os.remove('new_data.png')
496+
497+
tool = converter.cwl_command_line_tool()
498+
self.assertDictEqual(
499+
{
500+
'cwlVersion': "v1.1",
501+
'class': 'CommandLineTool',
502+
'baseCommand': 'notebookTool',
503+
'hints': {
504+
'DockerRequirement': {'dockerImageId': 'jn2cwl:latest'}
505+
},
506+
'arguments': ['--'],
507+
'inputs': {},
508+
'outputs': {
509+
'new_data': {
510+
'type': 'File',
511+
'outputBinding': {
512+
'glob': 'new_data.png'
513+
}
514+
}
515+
},
516+
},
517+
tool
518+
)
519+
520+
def test_AnnotatedIPython2CWLToolConverter_CWLPNGFigure(self):
521+
code = os.linesep.join([
522+
"import matplotlib.pyplot as plt",
523+
"new_data: 'CWLPNGFigure' = plt.plot([1,2,3,4])",
524+
])
525+
converter = AnnotatedIPython2CWLToolConverter(code)
526+
new_script = converter._wrap_script_to_method(
527+
converter._tree,
528+
converter._variables
529+
)
530+
try:
531+
os.remove('new_data.png')
532+
except FileNotFoundError:
533+
pass
534+
exec(new_script)
535+
locals()['main']()
536+
self.assertTrue(os.path.isfile('new_data.png'))
537+
os.remove('new_data.png')
538+
539+
tool = converter.cwl_command_line_tool()
540+
self.assertDictEqual(
541+
{
542+
'cwlVersion': "v1.1",
543+
'class': 'CommandLineTool',
544+
'baseCommand': 'notebookTool',
545+
'hints': {
546+
'DockerRequirement': {'dockerImageId': 'jn2cwl:latest'}
547+
},
548+
'arguments': ['--'],
549+
'inputs': {},
550+
'outputs': {
551+
'new_data': {
552+
'type': 'File',
553+
'outputBinding': {
554+
'glob': 'new_data.png'
555+
}
556+
}
557+
},
558+
},
559+
tool
560+
)

‎tests/test_system_tests.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,8 @@ def test_repo2cwl(self):
2929
self.assertListEqual(['example1.cwl'], [f for f in os.listdir(output_dir) if not f.startswith('.')])
3030

3131
with open(os.path.join(output_dir, 'example1.cwl')) as f:
32-
print(20 * '=')
3332
print('workflow file')
33+
print(20 * '=')
3434
print(f.read())
3535
print(20 * '=')
3636

0 commit comments

Comments
 (0)