Skip to content

Commit 68c35f1

Browse files
pditommasopditommaso
committed
Refactor Conda and Spack support for Wave to Java
Signed-off-by: Paolo Di Tommaso <[email protected]>
1 parent f6e4b9b commit 68c35f1

23 files changed

+985
-516
lines changed

‎plugins/nf-wave/build.gradle

+1
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ dependencies {
3333
compileOnly 'org.slf4j:slf4j-api:2.0.7'
3434
compileOnly 'org.pf4j:pf4j:3.4.1'
3535
api 'org.apache.commons:commons-compress:1.21'
36+
api 'org.apache.commons:commons-lang3:3.12.0'
3637
api 'com.google.code.gson:gson:2.10.1'
3738

3839
testImplementation(testFixtures(project(":nextflow")))

‎plugins/nf-wave/src/main/io/seqera/wave/plugin/config/CondaOpts.groovy ‎plugins/nf-wave/src/main/io/seqera/wave/config/CondaOpts.java

+13-11
Original file line numberDiff line numberDiff line change
@@ -15,25 +15,27 @@
1515
*
1616
*/
1717

18-
package io.seqera.wave.plugin.config
18+
package io.seqera.wave.config;
19+
20+
import java.util.List;
21+
import java.util.Map;
1922

2023
/**
2124
* Conda build options
2225
*
2326
* @author Paolo Di Tommaso <[email protected]>
2427
*/
25-
class CondaOpts {
26-
27-
final public static String DEFAULT_MAMBA_IMAGE = 'mambaorg/micromamba:1.4.2'
28+
public class CondaOpts {
29+
final public static String DEFAULT_MAMBA_IMAGE = "mambaorg/micromamba:1.4.2";
2830

29-
final String mambaImage
30-
final List<String> commands
31-
final String basePackages
31+
final public String mambaImage;
32+
final public List<String> commands;
33+
final public String basePackages;
3234

33-
CondaOpts(Map opts) {
34-
this.mambaImage = opts.mambaImage ?: DEFAULT_MAMBA_IMAGE
35-
this.commands = opts.commands as List<String>
36-
this.basePackages = opts.basePackages
35+
public CondaOpts(Map<String,?> opts) {
36+
this.mambaImage = opts.containsKey("mambaImage") ? opts.get("mambaImage").toString(): DEFAULT_MAMBA_IMAGE;
37+
this.commands = opts.containsKey("commands") ? (List<String>)opts.get("commands") : null;
38+
this.basePackages = (String)opts.get("basePackages");
3739
}
3840

3941
}

‎plugins/nf-wave/src/main/io/seqera/wave/config/SpackOpts.java

+58
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
/*
2+
* Copyright 2013-2023, Seqera Labs
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*
16+
*/
17+
18+
package io.seqera.wave.config;
19+
20+
import java.util.List;
21+
import java.util.Map;
22+
23+
/**
24+
* Spack build options
25+
*
26+
* @author Marco De La Pierre <[email protected]>
27+
*/
28+
public class SpackOpts {
29+
30+
final static public String DEFAULT_SPACK_BUILDER_IMAGE = "spack/ubuntu-jammy:v0.19.2";
31+
final static public String DEFAULT_SPACK_RUNNER_IMAGE = "ubuntu:22.04";
32+
final static public String DEFAULT_SPACK_OSPACKAGES = "";
33+
final static public String DEFAULT_SPACK_FLAGS = "-O3";
34+
35+
public final Boolean checksum;
36+
public final String builderImage;
37+
public final String runnerImage;
38+
public final String osPackages;
39+
public final String cFlags;
40+
public final String cxxFlags;
41+
public final String fFlags;
42+
public final List<String> commands;
43+
44+
public SpackOpts() {
45+
this(Map.of());
46+
}
47+
public SpackOpts(Map<String,?> opts) {
48+
this.checksum = opts.get("checksum") == null || Boolean.parseBoolean(opts.get("checksum").toString());
49+
this.builderImage = opts.containsKey("builderImage") ? opts.get("builderImage").toString() : DEFAULT_SPACK_BUILDER_IMAGE;
50+
this.runnerImage = opts.containsKey("runnerImage") ? opts.get("runnerImage").toString() : DEFAULT_SPACK_RUNNER_IMAGE;
51+
this.osPackages = opts.containsKey("osPackages") ? opts.get("osPackages").toString() : DEFAULT_SPACK_OSPACKAGES;
52+
this.cFlags = opts.containsKey("cFlags") ? opts.get("cFlags").toString() : DEFAULT_SPACK_FLAGS;
53+
this.cxxFlags = opts.containsKey("cxxFlags") ? opts.get("cxxFlags").toString() : DEFAULT_SPACK_FLAGS;
54+
this.fFlags = opts.containsKey("fFlags") ? opts.get("fFlags").toString() : DEFAULT_SPACK_FLAGS;
55+
this.commands = opts.containsKey("commands") ? (List<String>)opts.get("commands") : null;
56+
}
57+
58+
}

‎plugins/nf-wave/src/main/io/seqera/wave/plugin/DescribeContainerResponse.groovy

+17
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,20 @@
1+
/*
2+
* Copyright 2013-2023, Seqera Labs
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*
16+
*/
17+
118
package io.seqera.wave.plugin
219

320
import java.time.Instant

‎plugins/nf-wave/src/main/io/seqera/wave/plugin/WaveClient.groovy

+6-104
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
package io.seqera.wave.plugin
1919

20+
import static io.seqera.wave.util.DockerHelper.*
2021

2122
import java.net.http.HttpClient
2223
import java.net.http.HttpRequest
@@ -54,12 +55,10 @@ import io.seqera.wave.plugin.packer.Packer
5455
import nextflow.Session
5556
import nextflow.SysEnv
5657
import nextflow.container.resolver.ContainerInfo
57-
import nextflow.executor.BashTemplateEngine
5858
import nextflow.fusion.FusionConfig
5959
import nextflow.processor.Architecture
6060
import nextflow.processor.TaskRun
6161
import nextflow.script.bundle.ResourcesBundle
62-
import nextflow.util.MustacheTemplateEngine
6362
import nextflow.util.SysHelper
6463
import org.slf4j.Logger
6564
import org.slf4j.LoggerFactory
@@ -380,10 +379,11 @@ class WaveClient {
380379
// map the recipe to a dockerfile
381380
if( isCondaLocalFile(attrs.conda) ) {
382381
condaFile = Path.of(attrs.conda)
383-
dockerScript = condaFileToDockerFile()
382+
dockerScript = condaFileToDockerFile(config.condaOpts())
384383
}
384+
// 'conda' attributes is resolved as the conda packages to be used
385385
else {
386-
dockerScript = condaRecipeToDockerFile(attrs.conda)
386+
dockerScript = condaPackagesToDockerFile(attrs.conda, condaChannels, config.condaOpts())
387387
}
388388
}
389389

@@ -399,10 +399,10 @@ class WaveClient {
399399
// map the recipe to a dockerfile
400400
if( isSpackFile(attrs.spack) ) {
401401
spackFile = Path.of(attrs.spack)
402-
dockerScript = spackFileToDockerFile(spackArch)
402+
dockerScript = spackFileToDockerFile(spackArch, config.spackOpts())
403403
}
404404
else {
405-
dockerScript = spackRecipeToDockerFile(attrs.spack, spackArch)
405+
dockerScript = spackPackagesToDockerFile(attrs.spack, spackArch, config.spackOpts())
406406
}
407407
}
408408

@@ -464,105 +464,7 @@ class WaveClient {
464464
}
465465
}
466466

467-
protected String condaFileToDockerFile() {
468-
final template = """\
469-
FROM {{base_image}}
470-
COPY --chown=\$MAMBA_USER:\$MAMBA_USER conda.yml /tmp/conda.yml
471-
RUN micromamba install -y -n base -f /tmp/conda.yml && \\
472-
{{base_packages}}
473-
micromamba clean -a -y
474-
""".stripIndent(true)
475-
final image = config.condaOpts().mambaImage
476-
477-
final basePackage = config.condaOpts().basePackages ? "micromamba install -y -n base ${config.condaOpts().basePackages} && \\".toString() : null
478-
final binding = ['base_image': image, 'base_packages': basePackage]
479-
final result = new MustacheTemplateEngine().render(template, binding)
480-
481-
return addCommands(result)
482-
}
483-
484-
// Dockerfile template adpated from the Spack package manager
485-
// https://github.com/spack/spack/blob/develop/share/spack/templates/container/Dockerfile
486-
// LICENSE APACHE 2.0
487-
protected String spackFileToDockerFile(String spackArch) {
488-
489-
String cmd_template = ''
490-
final binding = [
491-
'builder_image': config.spackOpts().builderImage,
492-
'c_flags': config.spackOpts().cFlags,
493-
'cxx_flags': config.spackOpts().cxxFlags,
494-
'f_flags': config.spackOpts().fFlags,
495-
'spack_arch': spackArch,
496-
'checksum_string': config.spackOpts().checksum ? '' : '-n ',
497-
'runner_image': config.spackOpts().runnerImage,
498-
'os_packages': config.spackOpts().osPackages,
499-
'add_commands': addCommands(cmd_template),
500-
]
501-
final template = WaveClient.class.getResource('/templates/spack/dockerfile-spack-file.txt')
502-
try(final reader = template.newReader()) {
503-
final result = new BashTemplateEngine().render(reader, binding)
504-
return result
505-
}
506-
}
507467

508-
protected String addCommands(String result) {
509-
if( config.condaOpts().commands )
510-
for( String cmd : config.condaOpts().commands ) {
511-
result += cmd + "\n"
512-
}
513-
if( config.spackOpts().commands )
514-
for( String cmd : config.spackOpts().commands ) {
515-
result += cmd + "\n"
516-
}
517-
return result
518-
}
519-
520-
protected String condaRecipeToDockerFile(String recipe) {
521-
final template = """\
522-
FROM {{base_image}}
523-
RUN \\
524-
micromamba install -y -n base {{channel_opts}} \\
525-
{{target}} \\
526-
{{base_packages}}
527-
&& micromamba clean -a -y
528-
""".stripIndent(true)
529-
530-
final channelsOpts = condaChannels.collect(it -> "-c $it").join(' ')
531-
final image = config.condaOpts().mambaImage
532-
final target = recipe.startsWith('http://') || recipe.startsWith('https://')
533-
? "-f $recipe".toString()
534-
: recipe
535-
final basePackage = config.condaOpts().basePackages ? "&& micromamba install -y -n base ${config.condaOpts().basePackages} \\".toString() : null
536-
final binding = [base_image: image, channel_opts: channelsOpts, target:target, base_packages: basePackage]
537-
final result = new MustacheTemplateEngine().render(template, binding)
538-
return addCommands(result)
539-
}
540-
541-
// Dockerfile template adpated from the Spack package manager
542-
// https://github.com/spack/spack/blob/develop/share/spack/templates/container/Dockerfile
543-
// LICENSE APACHE 2.0
544-
protected String spackRecipeToDockerFile(String recipe, String spackArch) {
545-
546-
String cmd_template = ''
547-
final binding = [
548-
'recipe': recipe,
549-
'builder_image': config.spackOpts().builderImage,
550-
'c_flags': config.spackOpts().cFlags,
551-
'cxx_flags': config.spackOpts().cxxFlags,
552-
'f_flags': config.spackOpts().fFlags,
553-
'spack_arch': spackArch,
554-
'checksum_string': config.spackOpts().checksum ? '' : '-n ',
555-
'runner_image': config.spackOpts().runnerImage,
556-
'os_packages': config.spackOpts().osPackages,
557-
'add_commands': addCommands(cmd_template),
558-
]
559-
final template = WaveClient.class.getResource('/templates/spack/dockerfile-spack-recipe.txt')
560-
561-
try(final reader = template.newReader()) {
562-
final result = new BashTemplateEngine().render(reader, binding)
563-
return result
564-
}
565-
}
566468

567469
static protected boolean isCondaLocalFile(String value) {
568470
if( value.contains('\n') )

‎plugins/nf-wave/src/main/io/seqera/wave/plugin/WaveObserver.groovy

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright 2020-2022, Seqera Labs
2+
* Copyright 2013-2023, Seqera Labs
33
*
44
* Licensed under the Apache License, Version 2.0 (the "License");
55
* you may not use this file except in compliance with the License.

‎plugins/nf-wave/src/main/io/seqera/wave/plugin/adapter/InstantAdapter.groovy

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright 2020-2022, Seqera Labs
2+
* Copyright 2013-2023, Seqera Labs
33
*
44
* Licensed under the Apache License, Version 2.0 (the "License");
55
* you may not use this file except in compliance with the License.

‎plugins/nf-wave/src/main/io/seqera/wave/plugin/config/ReportOpts.groovy

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright 2020-2022, Seqera Labs
2+
* Copyright 2013-2023, Seqera Labs
33
*
44
* Licensed under the Apache License, Version 2.0 (the "License");
55
* you may not use this file except in compliance with the License.

‎plugins/nf-wave/src/main/io/seqera/wave/plugin/config/RetryOpts.groovy

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright 2020-2022, Seqera Labs
2+
* Copyright 2013-2023, Seqera Labs
33
*
44
* Licensed under the Apache License, Version 2.0 (the "License");
55
* you may not use this file except in compliance with the License.

‎plugins/nf-wave/src/main/io/seqera/wave/plugin/config/SpackOpts.groovy

-52
This file was deleted.

‎plugins/nf-wave/src/main/io/seqera/wave/plugin/config/WaveConfig.groovy

+2
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@ package io.seqera.wave.plugin.config
1919

2020
import groovy.transform.CompileStatic
2121
import groovy.util.logging.Slf4j
22+
import io.seqera.wave.config.CondaOpts
23+
import io.seqera.wave.config.SpackOpts
2224
import nextflow.util.Duration
2325
/**
2426
* Model Wave client configuration

0 commit comments

Comments
 (0)