From 631dca52933820e93608b6bed56649fc31be6ec7 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Tue, 20 Aug 2024 06:42:57 +0800 Subject: [PATCH 1/4] . --- Readme.adoc | 51 +++--- build.sc | 25 +++ os/src/Model.scala | 4 +- os/src/ProcessOps.scala | 56 +++++- .../SpawningSubprocessesNewTests.scala | 173 ++++++++++++++++++ 5 files changed, 284 insertions(+), 25 deletions(-) create mode 100644 os/test/src-jvm/SpawningSubprocessesNewTests.scala diff --git a/Readme.adoc b/Readme.adoc index c777ebfc..7a55fa81 100644 --- a/Readme.adoc +++ b/Readme.adoc @@ -1467,12 +1467,12 @@ Often, if you are only interested in capturing the standard output of the subprocess but want any errors sent to the console, you might set `stderr = os.Inherit` while leaving `stdout = os.Pipe`. -==== `os.proc.call` +==== `os.call` [source,scala] ---- -os.proc(command: os.Shellable*) - .call(cwd: Path = null, +os.call(cmd: os.Shellable, + cwd: Path = null, env: Map[String, String] = null, stdin: ProcessInput = Pipe, stdout: ProcessOutput = Pipe, @@ -1483,6 +1483,8 @@ os.proc(command: os.Shellable*) propagateEnv: Boolean = true): os.CommandResult ---- +_Also callable via `os.proc(cmd).call(...)`_ + Invokes the given subprocess like a function, passing in input and returning a `CommandResult`. You can then call `result.exitCode` to see how it exited, or `result.out.bytes` or `result.err.string` to access the aggregated stdout and @@ -1508,7 +1510,7 @@ Note that redirecting `stdout`/`stderr` elsewhere means that the respective [source,scala] ---- -val res = os.proc('ls, wd/"folder2").call() +val res = os.call(cmd = ('ls, wd/"folder2")) res.exitCode ==> 0 @@ -1531,13 +1533,13 @@ res.out.bytes // Non-zero exit codes throw an exception by default val thrown = intercept[os.SubprocessException]{ - os.proc('ls, "doesnt-exist").call(cwd = wd) + os.call(cmd = ('ls, "doesnt-exist"), cwd = wd) } assert(thrown.result.exitCode != 0) // Though you can avoid throwing by setting `check = false` -val fail = os.proc('ls, "doesnt-exist").call(cwd = wd, check = false) +val fail = os.call(cmd = ('ls, "doesnt-exist"), cwd = wd, check = false) assert(fail.exitCode != 0) @@ -1547,11 +1549,11 @@ fail.out.text() ==> "" assert(fail.err.text().contains("No such file or directory")) // You can pass in data to a subprocess' stdin -val hash = os.proc("shasum", "-a", "256").call(stdin = "Hello World") +val hash = os.call(cmd = ("shasum", "-a", "256"), stdin = "Hello World") hash.out.trim() ==> "a591a6d40bf420404a011733cfb7b190d62c65bf0bcda32b57b277d9ad9f146e -" // Taking input from a file and directing output to another file -os.proc("base64").call(stdin = wd / "File.txt", stdout = wd / "File.txt.b64") +os.call(cmd = ("base64"), stdin = wd / "File.txt", stdout = wd / "File.txt.b64") os.read(wd / "File.txt.b64") ==> "SSBhbSBjb3c=" ---- @@ -1570,7 +1572,8 @@ of `os.proc.call` in a streaming fashion, either on groups of bytes: [source,scala] ---- var lineCount = 1 -os.proc('find, ".").call( +os.call( + cmd = ('find, "."), cwd = wd, stdout = os.ProcessOutput( (buf, len) => lineCount += buf.slice(0, len).count(_ == '\n') @@ -1584,7 +1587,8 @@ Or on lines of output: ---- lineCount ==> 22 var lineCount = 1 -os.proc('find, ".").call( +os.call( + cmd = ('find, "."), cwd = wd, stdout = os.ProcessOutput.Readlines( line => lineCount += 1 @@ -1593,12 +1597,12 @@ os.proc('find, ".").call( lineCount ==> 22 ---- -==== `os.proc.spawn` +==== `os.spawn` [source,scala] ---- -os.proc(command: os.Shellable*) - .spawn(cwd: Path = null, +os.spawn(cmd: os.Shellable, + cwd: Path = null, env: Map[String, String] = null, stdin: os.ProcessInput = os.Pipe, stdout: os.ProcessOutput = os.Pipe, @@ -1607,7 +1611,9 @@ os.proc(command: os.Shellable*) propagateEnv: Boolean = true): os.SubProcess ---- -The most flexible of the `os.proc` calls, `os.proc.spawn` simply configures and +_Also callable via `os.proc(cmd).spawn(...)`_ + +The most flexible of the `os.proc` calls, `os.spawn` simply configures and starts a subprocess, and returns it as a `os.SubProcess`. `os.SubProcess` is a simple wrapper around `java.lang.Process`, which provides `stdin`, `stdout`, and `stderr` streams for you to interact with however you like. e.g. You can sending @@ -1619,10 +1625,7 @@ as the stdin of a second spawned process. Note that if you provide `ProcessOutput` callbacks to `stdout`/`stderr`, the calls to those callbacks take place on newly spawned threads that execute in parallel with the main thread. Thus make sure any data processing you do in -those callbacks is thread safe! For simpler cases, it may be easier to use -`os.proc.stream` which triggers it's `onOut`/`onErr` callbacks -all on the calling thread, avoiding needing to think about multithreading and -concurrency issues. +those callbacks is thread safe! `stdin`, `stdout` and `stderr` are ``java.lang.OutputStream``s and ``java.lang.InputStream``s enhanced with the `.writeLine(s: String)`/`.readLine()` @@ -1631,8 +1634,10 @@ methods for easy reading and writing of character and line-based data. [source,scala] ---- // Start a long-lived python process which you can communicate with -val sub = os.proc("python", "-u", "-c", "while True: print(eval(raw_input()))") - .spawn(cwd = wd) +val sub = os.spawn( + cmd = ("python", "-u", "-c", "while True: print(eval(raw_input()))"), + cwd = wd +) // Sending some text to the subprocess sub.stdin.write("1 + 2") @@ -1654,9 +1659,9 @@ sub.stdout.read() ==> '8'.toByte sub.destroy() // You can chain multiple subprocess' stdin/stdout together -val curl = os.proc("curl", "-L" , "https://git.io/fpfTs").spawn(stderr = os.Inherit) -val gzip = os.proc("gzip", "-n").spawn(stdin = curl.stdout) -val sha = os.proc("shasum", "-a", "256").spawn(stdin = gzip.stdout) +val curl = os.spawn(cmd = ("curl", "-L" , "https://git.io/fpfTs"), stderr = os.Inherit) +val gzip = os.spawn(cmd = ("gzip", "-n"), stdin = curl.stdout) +val sha = os.spawn(cmd = ("shasum", "-a", "256"), stdin = gzip.stdout) sha.stdout.trim ==> "acc142175fa520a1cb2be5b97cbbe9bea092e8bba3fe2e95afa645615908229e -" ---- diff --git a/build.sc b/build.sc index b619614e..dd40a544 100644 --- a/build.sc +++ b/build.sc @@ -112,6 +112,31 @@ trait OsModule extends OsLibModule { outer => def scalaDocOptions = super.scalaDocOptions() ++ conditionalScalaDocOptions() + def generatedSources = T{ + val conversions = for(i <- Range.inclusive(2, 22)) yield { + val ts = Range.inclusive(1, i).map(n => s"T$n").mkString(", ") + val fs = Range.inclusive(1, i).map(n => s"f$n: T$n => R").mkString(", ") + val vs = Range.inclusive(1, i).map(n => s"f$n(t._$n)").mkString(", ") + s""" implicit def tuple${i}Conversion[$ts] + | (t: ($ts)) + | (implicit $fs): R = { + | this.flatten($vs) + | } + |""".stripMargin + } + _root_.os.write( + T.dest / "os" / "GeneratedTupleConversions.scala", + s"""package os + |trait GeneratedTupleConversions[R]{ + | protected def flatten(vs: R*): R + | ${conversions.mkString("\n")} + |} + | + |""".stripMargin, + createFolders = true + ) + Seq(PathRef(T.dest)) + } } object os extends Module { diff --git a/os/src/Model.scala b/os/src/Model.scala index 2d9fb45e..523d648f 100644 --- a/os/src/Model.scala +++ b/os/src/Model.scala @@ -215,7 +215,7 @@ case class SubprocessException(result: CommandResult) extends Exception(result.t * be "interpolated" directly into a subprocess call. */ case class Shellable(value: Seq[String]) -object Shellable { +object Shellable extends os.GeneratedTupleConversions[Shellable]{ implicit def StringShellable(s: String): Shellable = Shellable(Seq(s)) implicit def CharSequenceShellable(cs: CharSequence): Shellable = Shellable(Seq(cs.toString)) @@ -232,6 +232,8 @@ object Shellable { implicit def ArrayShellable[T](s: Array[T])(implicit f: T => Shellable): Shellable = Shellable(s.toIndexedSeq.flatMap(f(_).value)) + + protected def flatten(vs: Shellable*): Shellable = IterableShellable(vs) } /** diff --git a/os/src/ProcessOps.scala b/os/src/ProcessOps.scala index fa8cc973..d2992593 100644 --- a/os/src/ProcessOps.scala +++ b/os/src/ProcessOps.scala @@ -10,6 +10,61 @@ import java.util.concurrent.LinkedBlockingQueue import ProcessOps._ import scala.util.Try +object call { + /** + * @see [[os.proc.call]] + */ + def apply(cmd: Shellable, + cwd: Path = null, + env: Map[String, String] = null, + stdin: ProcessInput = Pipe, + stdout: ProcessOutput = Pipe, + stderr: ProcessOutput = os.Inherit, + mergeErrIntoOut: Boolean = false, + timeout: Long = -1, + check: Boolean = true, + propagateEnv: Boolean = true, + timeoutGracePeriod: Long = 100 + ): CommandResult = { + os.proc(cmd).call( + cwd = cwd, + env = env, + stdin = stdin, + stdout = stdout, + stderr = stderr, + mergeErrIntoOut = mergeErrIntoOut, + timeout = timeout, + check = check, + propagateEnv = propagateEnv, + timeoutGracePeriod = timeoutGracePeriod, + ) + } +} +object spawn { + /** + * @see [[os.proc.spawn]] + */ + def apply(cmd: Shellable, + cwd: Path = null, + env: Map[String, String] = null, + stdin: ProcessInput = Pipe, + stdout: ProcessOutput = Pipe, + stderr: ProcessOutput = os.Inherit, + mergeErrIntoOut: Boolean = false, + propagateEnv: Boolean = true + ): SubProcess = { + os.proc(cmd).spawn( + cwd = cwd, + env = env, + stdin = stdin, + stdout = stdout, + stderr = stderr, + mergeErrIntoOut = mergeErrIntoOut, + propagateEnv = propagateEnv, + ) + } +} + /** * Convenience APIs around [[java.lang.Process]] and [[java.lang.ProcessBuilder]]: * @@ -27,7 +82,6 @@ import scala.util.Try * the standard stdin/stdout/stderr streams, using whatever protocol you * want */ - case class proc(command: Shellable*) { def commandChunks: Seq[String] = command.flatMap(_.value) diff --git a/os/test/src-jvm/SpawningSubprocessesNewTests.scala b/os/test/src-jvm/SpawningSubprocessesNewTests.scala new file mode 100644 index 00000000..a0345888 --- /dev/null +++ b/os/test/src-jvm/SpawningSubprocessesNewTests.scala @@ -0,0 +1,173 @@ +package test.os + +import java.io.{BufferedReader, InputStreamReader} +import os.ProcessOutput + +import scala.collection.mutable + +import test.os.TestUtil.prep +import utest._ + +object SpawningSubprocessesNewTests extends TestSuite { + + def tests = Tests { + test("proc") { + test("call") { + test - prep { wd => + if (Unix()) { + val res = os.call(cmd = ("ls", wd / "folder2")) + + res.exitCode ==> 0 + + res.out.text() ==> + """nestedA + |nestedB + |""".stripMargin + + res.out.trim() ==> + """nestedA + |nestedB""".stripMargin + + res.out.lines() ==> Seq( + "nestedA", + "nestedB" + ) + + res.out.bytes + + val thrown = intercept[os.SubprocessException] { + os.call(cmd = ("ls", "doesnt-exist"), cwd = wd) + } + + assert(thrown.result.exitCode != 0) + + val fail = os.call(cmd = ("ls", "doesnt-exist"), cwd = wd, check = false, stderr = os.Pipe) + + assert(fail.exitCode != 0) + + fail.out.text() ==> "" + + assert(fail.err.text().contains("No such file or directory")) + + // You can pass in data to a subprocess' stdin + val hash = os.call(cmd = ("shasum", "-a", "256"), stdin = "Hello World") + hash.out.trim() ==> "a591a6d40bf420404a011733cfb7b190d62c65bf0bcda32b57b277d9ad9f146e -" + + // Taking input from a file and directing output to another file + os.call(cmd = ("base64"), stdin = wd / "File.txt", stdout = wd / "File.txt.b64") + + os.read(wd / "File.txt.b64") ==> "SSBhbSBjb3c=\n" + + if (false) { + os.call(cmd = ("vim"), stdin = os.Inherit, stdout = os.Inherit, stderr = os.Inherit) + } + } + } + test - prep { wd => + if (Unix()) { + val ex = intercept[os.SubprocessException] { + os.call(cmd = ("bash", "-c", "echo 123; sleep 10; echo 456"), timeout = 2000) + } + + ex.result.out.trim() ==> "123" + } + } + } + test("stream") { + test - prep { wd => + if (Unix()) { + var lineCount = 1 + os.call( + cmd = ("find", "."), + cwd = wd, + stdout = + os.ProcessOutput((buf, len) => lineCount += buf.slice(0, len).count(_ == '\n')) + ) + lineCount ==> 22 + } + } + test - prep { wd => + if (Unix()) { + var lineCount = 1 + os.call( + cmd = ("find", "."), + cwd = wd, + stdout = os.ProcessOutput.Readlines(line => lineCount += 1) + ) + lineCount ==> 22 + } + } + } + + test("spawn python") { + test - prep { wd => + if (TestUtil.isInstalled("python") && Unix()) { + // Start a long-lived python process which you can communicate with + val sub = os.spawn( + cmd = ( + "python", + "-u", + "-c", + if (TestUtil.isPython3()) "while True: print(eval(input()))" + else "while True: print(eval(raw_input()))"), + cwd = wd + ) + + // Sending some text to the subprocess + sub.stdin.write("1 + 2") + sub.stdin.writeLine("+ 4") + sub.stdin.flush() + sub.stdout.readLine() ==> "7" + + sub.stdin.write("'1' + '2'") + sub.stdin.writeLine("+ '4'") + sub.stdin.flush() + sub.stdout.readLine() ==> "124" + + // Sending some bytes to the subprocess + sub.stdin.write("1 * 2".getBytes) + sub.stdin.write("* 4\n".getBytes) + sub.stdin.flush() + sub.stdout.read() ==> '8'.toByte + + sub.destroy() + } + } + } + test("spawn curl") { + if ( + Unix() && // shasum seems to not accept stdin on Windows + TestUtil.isInstalled("curl") && + TestUtil.isInstalled("gzip") && + TestUtil.isInstalled("shasum") + ) { + // You can chain multiple subprocess' stdin/stdout together + val curl = + os.spawn(cmd = ("curl", "-L", ExampleResourcess.RemoteReadme.url), stderr = os.Inherit) + val gzip = os.spawn(cmd = ("gzip", "-n", "-6"), stdin = curl.stdout) + val sha = os.spawn(cmd = ("shasum", "-a", "256"), stdin = gzip.stdout) + sha.stdout.trim() ==> s"${ExampleResourcess.RemoteReadme.gzip6ShaSum256} -" + } + } + test("spawn callback") { + test - prep { wd => + if (TestUtil.isInstalled("echo") && Unix()) { + val output: mutable.Buffer[String] = mutable.Buffer() + val sub = os.spawn( + cmd = ("echo", "output"), + stdout = ProcessOutput((bytes, count) => output += new String(bytes, 0, count)) + ) + val finished = sub.join(5000) + sub.wrapped.getOutputStream().flush() + assert(finished) + assert(sub.exitCode() == 0) + val expectedOutput = "output\n" + val actualOutput = output.mkString("") + assert(actualOutput == expectedOutput) + sub.destroy() + } + } + } + } + } +} From 9db2e0566a94e0f6e27285ea9d1f1beee3846d6c Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Tue, 20 Aug 2024 06:45:44 +0800 Subject: [PATCH 2/4] . --- Readme.adoc | 6 ++- os/src/Model.scala | 2 +- os/src/ProcessOps.scala | 50 ++++++++++--------- .../SpawningSubprocessesNewTests.scala | 6 ++- 4 files changed, 36 insertions(+), 28 deletions(-) diff --git a/Readme.adoc b/Readme.adoc index 7a55fa81..ac7ddf17 100644 --- a/Readme.adoc +++ b/Readme.adoc @@ -1424,8 +1424,9 @@ os.owner.set(wd / "File.txt", originalOwner) === Spawning Subprocesses -Subprocess are spawned using `+os.proc(command: os.Shellable*).foo(...)+` calls, -where the `command: Shellable*` sets up the basic command you wish to run and +Subprocess are spawned using `+os.call(cmd: os.Shellable, ...)+` or +`+os.spawn(cmd: os.Shellable, ...)+` calls, +where the `cmd: Shellable` sets up the basic command you wish to run and `+.foo(...)+` specifies how you want to run it. `os.Shellable` represents a value that can make up part of your subprocess command, and the following values can be used as ``os.Shellable``s: @@ -1436,6 +1437,7 @@ be used as ``os.Shellable``s: * `os.RelPath` * `T: Numeric` * ``Iterable[T]``s of any of the above +* ``TupleN[T1, T2, ...Tn]``s of any of the above Most of the subprocess commands also let you redirect the subprocess's `stdin`/`stdout`/`stderr` streams via `os.ProcessInput` or `os.ProcessOutput` diff --git a/os/src/Model.scala b/os/src/Model.scala index 523d648f..f2389407 100644 --- a/os/src/Model.scala +++ b/os/src/Model.scala @@ -215,7 +215,7 @@ case class SubprocessException(result: CommandResult) extends Exception(result.t * be "interpolated" directly into a subprocess call. */ case class Shellable(value: Seq[String]) -object Shellable extends os.GeneratedTupleConversions[Shellable]{ +object Shellable extends os.GeneratedTupleConversions[Shellable] { implicit def StringShellable(s: String): Shellable = Shellable(Seq(s)) implicit def CharSequenceShellable(cs: CharSequence): Shellable = Shellable(Seq(cs.toString)) diff --git a/os/src/ProcessOps.scala b/os/src/ProcessOps.scala index d2992593..c1a0051c 100644 --- a/os/src/ProcessOps.scala +++ b/os/src/ProcessOps.scala @@ -11,21 +11,23 @@ import ProcessOps._ import scala.util.Try object call { + /** * @see [[os.proc.call]] */ - def apply(cmd: Shellable, - cwd: Path = null, - env: Map[String, String] = null, - stdin: ProcessInput = Pipe, - stdout: ProcessOutput = Pipe, - stderr: ProcessOutput = os.Inherit, - mergeErrIntoOut: Boolean = false, - timeout: Long = -1, - check: Boolean = true, - propagateEnv: Boolean = true, - timeoutGracePeriod: Long = 100 - ): CommandResult = { + def apply( + cmd: Shellable, + cwd: Path = null, + env: Map[String, String] = null, + stdin: ProcessInput = Pipe, + stdout: ProcessOutput = Pipe, + stderr: ProcessOutput = os.Inherit, + mergeErrIntoOut: Boolean = false, + timeout: Long = -1, + check: Boolean = true, + propagateEnv: Boolean = true, + timeoutGracePeriod: Long = 100 + ): CommandResult = { os.proc(cmd).call( cwd = cwd, env = env, @@ -36,23 +38,25 @@ object call { timeout = timeout, check = check, propagateEnv = propagateEnv, - timeoutGracePeriod = timeoutGracePeriod, + timeoutGracePeriod = timeoutGracePeriod ) } } object spawn { + /** * @see [[os.proc.spawn]] */ - def apply(cmd: Shellable, - cwd: Path = null, - env: Map[String, String] = null, - stdin: ProcessInput = Pipe, - stdout: ProcessOutput = Pipe, - stderr: ProcessOutput = os.Inherit, - mergeErrIntoOut: Boolean = false, - propagateEnv: Boolean = true - ): SubProcess = { + def apply( + cmd: Shellable, + cwd: Path = null, + env: Map[String, String] = null, + stdin: ProcessInput = Pipe, + stdout: ProcessOutput = Pipe, + stderr: ProcessOutput = os.Inherit, + mergeErrIntoOut: Boolean = false, + propagateEnv: Boolean = true + ): SubProcess = { os.proc(cmd).spawn( cwd = cwd, env = env, @@ -60,7 +64,7 @@ object spawn { stdout = stdout, stderr = stderr, mergeErrIntoOut = mergeErrIntoOut, - propagateEnv = propagateEnv, + propagateEnv = propagateEnv ) } } diff --git a/os/test/src-jvm/SpawningSubprocessesNewTests.scala b/os/test/src-jvm/SpawningSubprocessesNewTests.scala index a0345888..b753d1e6 100644 --- a/os/test/src-jvm/SpawningSubprocessesNewTests.scala +++ b/os/test/src-jvm/SpawningSubprocessesNewTests.scala @@ -41,7 +41,8 @@ object SpawningSubprocessesNewTests extends TestSuite { assert(thrown.result.exitCode != 0) - val fail = os.call(cmd = ("ls", "doesnt-exist"), cwd = wd, check = false, stderr = os.Pipe) + val fail = + os.call(cmd = ("ls", "doesnt-exist"), cwd = wd, check = false, stderr = os.Pipe) assert(fail.exitCode != 0) @@ -109,7 +110,8 @@ object SpawningSubprocessesNewTests extends TestSuite { "-u", "-c", if (TestUtil.isPython3()) "while True: print(eval(input()))" - else "while True: print(eval(raw_input()))"), + else "while True: print(eval(raw_input()))" + ), cwd = wd ) From eb32f762a553730ca7d5cc889fd49e410d15644e Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Tue, 20 Aug 2024 06:57:35 +0800 Subject: [PATCH 3/4] . --- os/src/ProcessOps.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/os/src/ProcessOps.scala b/os/src/ProcessOps.scala index c1a0051c..2170c795 100644 --- a/os/src/ProcessOps.scala +++ b/os/src/ProcessOps.scala @@ -17,8 +17,8 @@ object call { */ def apply( cmd: Shellable, - cwd: Path = null, env: Map[String, String] = null, + cwd: Path = null, stdin: ProcessInput = Pipe, stdout: ProcessOutput = Pipe, stderr: ProcessOutput = os.Inherit, @@ -49,8 +49,8 @@ object spawn { */ def apply( cmd: Shellable, - cwd: Path = null, env: Map[String, String] = null, + cwd: Path = null, stdin: ProcessInput = Pipe, stdout: ProcessOutput = Pipe, stderr: ProcessOutput = os.Inherit, From dd6ff480f2b9d9ea00c94ea16600c4e2e943730a Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Tue, 20 Aug 2024 06:59:07 +0800 Subject: [PATCH 4/4] . --- os/src/ProcessOps.scala | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/os/src/ProcessOps.scala b/os/src/ProcessOps.scala index 2170c795..389b34f0 100644 --- a/os/src/ProcessOps.scala +++ b/os/src/ProcessOps.scala @@ -18,6 +18,8 @@ object call { def apply( cmd: Shellable, env: Map[String, String] = null, + // Make sure `cwd` only comes after `env`, so `os.call("foo", path)` is a compile error + // since the correct syntax is `os.call(("foo", path))` cwd: Path = null, stdin: ProcessInput = Pipe, stdout: ProcessOutput = Pipe, @@ -49,6 +51,8 @@ object spawn { */ def apply( cmd: Shellable, + // Make sure `cwd` only comes after `env`, so `os.spawn("foo", path)` is a compile error + // since the correct syntax is `os.spawn(("foo", path))` env: Map[String, String] = null, cwd: Path = null, stdin: ProcessInput = Pipe,