Merge branch 'master' into 6570-slurm-job-state-query

Author: tcutts

docker/Makefile

@@ -37,9 +37,9 @@ release: build
 
 dist/docker/amd64:
 	mkdir -p dist/linux/amd64
-	curl -fsSL https://download.docker.com/linux/static/stable/x86_64/docker-17.09.0-ce.tgz | tar --strip-components=1 -xvz -C dist/linux/amd64
+	curl -fsSL https://download.docker.com/linux/static/stable/x86_64/docker-19.03.15.tgz | tar --strip-components=1 -xvz -C dist/linux/amd64
 
 dist/docker/arm64:
 	mkdir -p dist/linux/arm64
-	curl -fsSL https://download.docker.com/linux/static/stable/aarch64/docker-17.09.0-ce.tgz | tar --strip-components=1 -xvz -C dist/linux/arm64
+	curl -fsSL https://download.docker.com/linux/static/stable/aarch64/docker-19.03.15.tgz | tar --strip-components=1 -xvz -C dist/linux/arm64
 

docs/reference/config.md

@@ -871,6 +871,7 @@ The following settings are available:
 : Enable Fusion snapshotting (preview, default: `false`). This feature allows Fusion to automatically restore a job when it is interrupted by a spot reclamation.
 
 `fusion.tags`
+: *Currently only supported for S3.*
 : The pattern that determines how tags are applied to files created via the Fusion client (default: `[.command.*|.exitcode|.fusion.*](nextflow.io/metadata=true),[*](nextflow.io/temporary=true)`). Set to `false` to disable tags.
 
 (config-google)=

docs/reference/process.md

@@ -1656,17 +1656,17 @@ The `stageOutMode` directive defines how output files are staged out from the sc
 : Output files are copied from the scratch directory to the work directory.
 
 `'fcp'`
-: :::{versionadded} 23.02.0-edge
+: :::{versionadded} 23.04.0
   :::
-: Output files are copied from the scratch directory to the work directory by using the [fcp](https://github.com/Svetlitski/fcp) utility (note: it must be available in your cluster computing nodes).
+: Output files are copied from the scratch directory to the work directory by using the [fcp](https://github.com/Svetlitski/fcp) utility (note: it must be available in the task environment).
 
 `'move'`
 : Output files are moved from the scratch directory to the work directory.
 
 `'rclone'`
-: :::{versionadded} 23.01.0-edge
+: :::{versionadded} 23.04.0
   :::
-: Output files are copied from the scratch directory to the work directory by using the [rclone](https://rclone.org) utility (note: it must be available in your cluster computing nodes).
+: Output files are copied from the scratch directory to the work directory by using the [rclone](https://rclone.org) utility (note: it must be available in the task environment).
 
 `'rsync'`
 : Output files are copied from the scratch directory to the work directory by using the `rsync` utility.

modules/nextflow/src/main/groovy/nextflow/config/Manifest.groovy

@@ -170,8 +170,11 @@ class Manifest implements ConfigScope {
                 .map(opts -> new Contributor(opts))
                 .toList()
         }
-        catch( ClassCastException | IllegalArgumentException e ){
-            throw new AbortOperationException("Invalid config option `manifest.contributors` -- should be a list of maps")
+        catch( IllegalArgumentException e ){
+            throw new AbortOperationException(e.message)
+        }
+        catch( ClassCastException e ){
+            throw new AbortOperationException("Invalid setting for `manifest.contributors` config option -- should be a list of maps")
         }
     }
 
@@ -208,17 +211,23 @@ class Manifest implements ConfigScope {
             affiliation = opts.affiliation as String
             email = opts.email as String
             github = opts.github as String
-            contribution = parseContributionTypes(opts.contribution)
+            contribution = parseContributionTypes(opts.contribution as List<String>)
             orcid = opts.orcid as String
         }
 
-        private List<ContributionType> parseContributionTypes(Object value) {
-            if( value == null )
+        private List<ContributionType> parseContributionTypes(List<String> values) {
+            if( values == null )
                 return []
-            return (value as List<String>).stream()
-                .map(c -> ContributionType.valueOf(c.toUpperCase()))
-                .sorted()
-                .toList()
+            final result = new LinkedList<ContributionType>()
+            for( final value : values ) {
+                try {
+                    result.add(ContributionType.valueOf(value.toUpperCase()))
+                }
+                catch( IllegalArgumentException e ) {
+                    throw new IllegalArgumentException("Invalid contribution type '$value' in `manifest.contributors` config option")
+                }
+            }
+            return result.toSorted()
         }
 
         Map toMap() {

modules/nextflow/src/main/groovy/nextflow/splitter/AbstractSplitter.groovy

@@ -141,24 +141,24 @@ abstract class AbstractSplitter<T> implements SplitterStrategy {
 
         setSource(source)
 
-
-        final chunks = collector = createCollector()
-        if( chunks instanceof CacheableCollector && chunks.checkCached() ) {
-            log.debug "Operator `$operatorName` reusing cached chunks at path: ${chunks.getBaseFile()}"
-            result = resumeFromCache(chunks)
+        this.collector = createCollector()
+        if( collector instanceof CacheableCollector && collector.checkCached() ) {
+            log.debug "Operator `$operatorName` reusing cached chunks at path: ${collector.getBaseFile()}"
+            result = resumeFromCache(collector)
         }
-
         else {
             try {
-                def stream = normalizeSource(source)
+                final stream = normalizeSource(source)
                 result = process(stream)
+
+                if( collector instanceof CacheableCollector )
+                    collector.markComplete()
             }
             catch ( StopSplitIterationException e ) {
                 log.trace 'Split iteration interrupted'
             }
         }
 
-
         /*
          * now close and return the result
          * - when the target it's a channel, send stop message
@@ -246,7 +246,7 @@ abstract class AbstractSplitter<T> implements SplitterStrategy {
      * @param index the current split count
      * @return Either {@link groovyx.gpars.dataflow.DataflowChannel} or a {@code List} which holds the splitted chunks
      */
-    protected abstract process( T targetObject )
+    abstract protected process( T targetObject )
 
     /**
      * Normalise the source object to be splitted

modules/nextflow/src/main/groovy/nextflow/splitter/AbstractTextSplitter.groovy

@@ -56,6 +56,7 @@ abstract class AbstractTextSplitter extends AbstractSplitter<Reader> {
 
     private long itemsCount
 
+    @Override
     AbstractTextSplitter options(Map options) {
         super.options(options)
 
@@ -108,6 +109,7 @@ abstract class AbstractTextSplitter extends AbstractSplitter<Reader> {
      * @return  A  {@link Reader} for the given object.
      * @throws IllegalArgumentException if the object specified is of a type not supported
      */
+    @Override
     protected Reader normalizeSource( obj ) {
         Reader reader = normalizeSource0( obj )
         // detect if starts with bom and position after it
@@ -178,6 +180,7 @@ abstract class AbstractTextSplitter extends AbstractSplitter<Reader> {
      * @param offset
      * @return
      */
+    @Override
     protected process( Reader targetObject ) {
 
         def result = null
@@ -244,6 +247,7 @@ abstract class AbstractTextSplitter extends AbstractSplitter<Reader> {
      * @return A {@link CollectorStrategy} object implementing a concrete
      * strategy according the user provided options
      */
+    @Override
     protected CollectorStrategy createCollector() {
 
         if( !isCollectorEnabled() )
@@ -326,7 +330,6 @@ abstract class AbstractTextSplitter extends AbstractSplitter<Reader> {
      */
     abstract protected fetchRecord( BufferedReader reader )
 
-
     protected int positionAfterBOM(Reader reader ){
         if( !reader.markSupported() )
             return 0

modules/nextflow/src/main/groovy/nextflow/splitter/TextFileCollector.groovy

@@ -79,6 +79,7 @@ class TextFileCollector implements CollectorStrategy, CacheableCollector, Header
         return file.resolveSibling( fileName )
     }
 
+    @Override
     void setHeader(String value) {
         this.header = value
     }
@@ -139,8 +140,6 @@ class TextFileCollector implements CollectorStrategy, CacheableCollector, Header
     @Override
     void close() throws IOException {
         closeWriter()
-        markComplete()
     }
 
-
 }

modules/nextflow/src/main/resources/nextflow/executor/command-run.txt

@@ -87,7 +87,10 @@ nxf_fs_move() {
 }
 
 nxf_fs_rsync() {
-  rsync -rRl $1 $2
+  local source=$1
+  local target=$2
+  mkdir -p $target
+  rsync -rRl $source $target
 }
 
 nxf_fs_rclone() {

modules/nextflow/src/test/groovy/nextflow/cli/CmdConfigTest.groovy

@@ -330,6 +330,48 @@ class CmdConfigTest extends Specification {
         folder.deleteDir()
     }
 
+    def 'should parse config file with nested configs' () {
+        given:
+        def folder = Files.createTempDirectory('test')
+        def CONFIG = folder.resolve('nextflow.config')
+        def CONFIG2 = folder.resolve('nextflow2.config')
+        def CONFIG3 = folder.resolve('nextflow3.config')
+        CONFIG.text  = '''
+        profiles {
+            test { includeConfig 'nextflow2.config' }
+        }
+        '''
+        CONFIG2.text = '''
+        includeConfig 'nextflow3.config'
+        '''
+        CONFIG3.text = '''
+        params {
+            x = { 1 + 2 }
+        }
+        '''
+        and:
+        def buffer = new ByteArrayOutputStream()
+        def cmd = new CmdConfig()
+        cmd.launcher = new Launcher(options: new CliOptions(config: [CONFIG.toString()]))
+        cmd.profile = 'test'
+        cmd.stdout = buffer
+        cmd.args = [ '.' ]
+
+        when:
+        cmd.run()
+
+        then:
+        def result = buffer.toString()
+        result == '''
+        params {
+           x = { 1 + 2 }
+        }
+        '''.stripIndent().leftTrim()
+
+        cleanup:
+        folder.deleteDir()
+    }
+
     def 'should handle variables' () {
         given:
         def folder = Files.createTempDirectory('test')

modules/nextflow/src/test/groovy/nextflow/config/ManifestTest.groovy

@@ -179,7 +179,8 @@ class ManifestTest extends Specification {
         ])
         manifest.contributors
         then:
-        thrown(AbortOperationException)
+        def e = thrown(AbortOperationException)
+        e.message.contains("Invalid contribution type 'owner' in `manifest.contributors` config option")
     }
 
 }