added awk-like line processing api

Author: holgerbrandl

src/main/kotlin/kscript/AwkLike.kt

@@ -0,0 +1,69 @@
+package kscript
+
+/**
+ * Utility methods to allow for awk-like data processing using kscript.
+ *
+ * The general usage pattern is to start with a `Sequence<String>` that is typically provided as `lines` by [kscript.resolveArgFile], and to end with one of the `print` extension methods provided below.
+ *
+ * In one-liners `lines` is implicitly added by `kscript` and the alle elements from `kscript.*` are imported. This allows for constructs such as
+ * ```
+ * kscript 'lines.filter{ it.contains("foo") }.print()' some_file.txt
+ * kscript 'lines.filter{ it.contains("foo") }.print()'
+ * cat some_file.txt  | kscript 'lines.filter{ it.contains("foo") }.print()'
+ *
+ * ```
+ *
+ * @author Holger Brandl
+ */
+
+
+// for top-level vs member extensions see https://kotlinlang.org/docs/reference/extensions.html#scope-of-extensions
+
+
+/** For sake of readability we refer to a list of strings as a row here. */
+typealias Row = List<String>
+
+//typealias RowSequence = Sequence<Row>
+//class RowSequence(input: Sequence<String>){
+//}
+
+
+/** Splits the lines of an input stream into [Row]s.
+ *
+ * @param separator The used separator character which defaults to tabs.
+ */
+fun Sequence<String>.split(separator: String = "\t"): Sequence<Row> {
+    return this.map { it.split(separator) }
+}
+
+/** awk-like convenience wrapper around columns->map->print */
+fun Sequence<String>.awk(separator: String = "\t", rule: (Row) -> String) = split(separator).map { rule(it) }.print()
+
+
+fun Sequence<Row>.map(vararg rules: (Row) -> String): Sequence<Row> {
+    return map { splitLine -> rules.map { it(splitLine) } }
+}
+
+/** Adds a new column to a row. */
+fun Sequence<Row>.add(rule: (Row) -> String): Sequence<Row> {
+    return map { row -> listOf(*row.toTypedArray(), rule(row)) }
+}
+
+
+//@Deprecated("use kscript.awk() instead")
+//fun Sequence<String>.splitMap(vararg rules: (Row) -> String, separator: String = "\t", joinWith: String = separator) {
+//    map { it.split(separator).let { splitLine -> rules.map { it(splitLine) } } }.print()
+//}
+
+
+fun Sequence<Row>.join(separator: String = "\t") = map { it.joinToString(separator) }
+
+/** Joins rows with the provided `separator` and print them to `stdout`. */
+fun Sequence<Row>.print(separator: String = "\t") = join(separator).print()
+
+
+fun List<Row>.print() = forEach { println(it) }
+
+
+// todo  add krangl ColNames interface here
+

src/main/kotlin/kscript/FileUtil.kt

@@ -1,34 +1,67 @@
 package kscript
 
-import java.io.BufferedReader
-import java.io.File
-import java.io.FileReader
+import java.io.*
+import java.util.zip.GZIPOutputStream
 
 
-// for top-level vs member extensions see https://kotlinlang.org/docs/reference/extensions.html#scope-of-extensions
-//object KscriptHelpers {}
+/** a `Sequence<String>` iterator for standard input */
+val stdin by lazy { generateSequence() { readLine() } }
 
-val stdin = generateSequence() { readLine() }
+fun linesFrom(file: File) = BufferedReader(FileReader(file)).lineSequence()
 
+// just used for testing and development
+fun linesFrom(vararg lines: String) = lines.asSequence()
 
-/** Read lines stdin or a file argument. Example 'argLines(1).map { it+"foo"}'. could be used either with "-" or file argument. */
-fun argLines(arg: String, stdinNames: List<String> = listOf("-", "stdin")): Sequence<String> {
-    if (stdinNames.contains(arg)) return stdin
 
-    val inputFile = File(arg)
+/**
+ * File argument processor that works similar to awk. If data is available on stdin, use it. If not expect a file argument and read from that one instead.
+ * */
+fun resolveArgFile(args: Array<String>, position: Int = 0): Sequence<String> {
+    if (stdin.iterator().hasNext()) return stdin
 
-    stopIfNot(inputFile.canRead()) { "Can not read from '${arg}'" }
+    stopIfNot(args.isNotEmpty()) { "Missing file or input input stream" }
+    stopIfNot(args.size >= position) { "arg position ${position} exceeds number of arguments ${args.size} " }
+
+    val fileArg = args[position]
+
+    //    stdinNames: List<String> = listOf("-", "stdin")
+    //    if (stdinNames.contains(fileArg)) return stdin
+
+    val inputFile = File(fileArg)
+
+    stopIfNot(inputFile.canRead()) { "Can not read from '${fileArg}'" }
 
     // todo we don't close the buffer with this approach
     //    BufferedReader(FileReader(inputFile )).use { return it }
+
     return BufferedReader(FileReader(inputFile)).lineSequence()
 }
 
-fun argMap(arg: String, stdinNames: List<String> = listOf("-", "stdin"), trafo: (String) -> String) =
-        argLines(arg, stdinNames).map { trafo(it) }.print()
 
-fun argFilter(arg: String, stdinNames: List<String> = listOf("-", "stdin"), trafo: (String) -> Boolean) =
-        argLines(arg, stdinNames).filter { trafo(it) }.print()
+/** Endpoint for a kscript pipe. */
+fun Sequence<String>.print() = forEach { println(it) }
+
+/** Endpoint for a kscript pipe. */
+fun Iterable<String>.print() = forEach { println(it) }
+
+
+
+//https://dzone.com/articles/readingwriting-compressed-and
+/** Save a list of items into a file. Output can be option ally zipped and a the stringifying operation can be changed from toString to custom operation if needed. */
+fun <T> Iterable<T>.saveAs(f: File,
+                           transform: (T) -> String = { it.toString() },
+                           separator: Char = '\n',
+                           overwrite: Boolean = true,
+                           compress: Boolean = f.name.let { it.endsWith(".zip") || it.endsWith(".gz") }) {
+
+    // ensure that file is not yet there or overwrite flag is set
+    require(!f.isFile || overwrite) { "$f is present already. Use overwrite=true to enforce file replacement." }
+
+    val p = if (!compress) PrintWriter(f) else BufferedWriter(OutputStreamWriter(GZIPOutputStream(FileOutputStream(f))))
+
+    toList().forEach { p.write(transform(it) + separator) }
+
+    p.close()
+}
 
 
-fun Sequence<String>.print() = forEach { println(it) }

src/main/kotlin/kscript/Support.kt src/main/kotlin/kscript/KscriptUtil.ktsrc/main/kotlin/kscript/Support.kt renamed to src/main/kotlin/kscript/KscriptUtil.kt

@@ -25,3 +25,8 @@ fun processStdin(trafo: (String) -> String) {
         println(trafo(it))
     }
 }
+
+
+operator fun File.div(childName: String): File {
+    return this.resolve(childName)
+}

src/main/kotlin/kscript/StreamUtil.kt

@@ -0,0 +1,20 @@
+package kscript.experimental
+
+import kscript.resolveArgFile
+
+/**
+ * @author Holger Brandl
+ */
+
+abstract class OneLinerContext(args: Array<String>) {
+
+    val arg by lazy { resolveArgFile(args) }
+    //    val stdin by lazy {  kscript.stdin }
+
+    init {
+        apply(arg)
+    }
+
+    abstract fun apply(lines: Sequence<String>)
+}
+

src/main/kotlin/kscript/experimental/Incubator.kt

@@ -0,0 +1,83 @@
+package kscript.examples
+
+import kscript.*
+import kscript.experimental.OneLinerContext
+
+/**
+ * @author Holger Brandl
+ */
+
+val args = arrayOf("flights.txt")
+
+object AwkExample : OneLinerContext(args) {
+    override fun apply(lines: Sequence<String>) {
+        lines.split().map({ it[1] }, { it[2] }).print()
+
+
+        lines.split().filter { it[3].matches("UA".toRegex()) }.print()
+
+        // remove header
+        lines.drop(1).split().filter { it[3].matches("UA".toRegex()) }.print()
+
+
+        lines.awk { it[1] + it[2] }
+
+        lines.awk { it[1] + it[2] }
+        lines.awk { it[3] }
+
+        // http://stackoverflow.com/questions/15361632/delete-a-column-with-awk-or-sed
+        lines.split().map { it.toMutableList().apply { removeAt(3) } }.print()
+
+
+        //        http@ //tuxgraphics.org/~guido/scripts/awk-one-liner.html
+        //        Print the next two (i=2) lines after the line matching regexp:
+        //        awk '/regexp/{i=2;next;}{if(i){i--; print;}}' file.txt
+
+
+        fun <T> List<T>.sliding(windowSize: Int): List<List<T>> {
+            return this.dropLast(windowSize - 1).mapIndexed { i, s -> this.subList(i, i + windowSize) }
+        }
+
+        val regex = "a[bc]+".toRegex()
+
+        resolveArgFile(args).
+                toList().sliding(4).
+                filter { it[0].matches(regex) }.
+                flatten().print()
+
+
+
+        // number lines (from http://tuxgraphics.org/~guido/scripts/awk-one-liner.html)
+        //        awk '{print FNR "\t" $0}'
+        lines.mapIndexed { num, line -> num.toString() + " " + line }.print()
+
+
+        //        Remove duplicate consecutive lines (uniq):
+        //        awk 'a !~ $0{print}; {a=$0}'
+
+
+        //        Delete trailing white space (spaces, tabs)
+        //        awk '{sub(/[ \t]*$/, "");print}' file.txt
+        lines.map { it.trim() }.print()
+
+        //        Count lines (wc -l):
+        //        awk 'END{print NR}'
+        println(lines.fold(0) { cur, _ -> cur + 1 })
+        println(lines.mapIndexed { i, _ -> i }.last())
+        // don't
+        println(lines.toList().size)
+
+
+
+        //        Print the lines from a file starting at the line matching "start" until the line matching "stop":
+        //        awk '/start/,/stop/' file.txt
+
+        lines.dropWhile { it.startsWith("foo") }.takeWhile { it.startsWith("bar") }.print()
+
+        val arg by lazy { resolveArgFile(args) }
+        arg.filter { true }.print()
+    }
+}
+//file:///Users/brandl/Desktop/awk_cheatsheets.pdf
+
+

src/main/kotlin/kscript/experimental/OneLinerContext.kt

@@ -0,0 +1,23 @@
+package kscript.examples
+
+import kscript.add
+import kscript.split
+import kscript.experimental.OneLinerContext
+import kscript.print
+
+/**
+ * One-liner kscript example. To ease devlopment simply extend [OneLinerContext] as shown, which will provide
+ * the same context as `kscript` when running in single line mode.
+ *
+ * @author Holger Brandl
+ */
+
+fun main(args: Array<String>) {
+
+    object : OneLinerContext(args) {
+
+        override fun apply(lines: Sequence<String>) {
+            lines.split().filter { it[3] == "UA" }.add { it[3] + ":" + it[3] }.print()
+        }
+    }
+}

src/test/kotlin/kscript/examples/AwkComparison.kt

@@ -1,6 +1,5 @@
 package kscript.test
 
-import io.kotlintest.matchers.shouldBe
 import io.kotlintest.specs.StringSpec
 import kscript.*
 import java.io.File
@@ -18,8 +17,8 @@ class SupportApiTest : StringSpec() { init {
         //        "hello".length shouldBe 5
         //        stopIfNot("FOO"=="BAR"){"condition not met"}
         println("current dir is " + File(".").absolutePath)
-        argMap("src/test/resources/flights_head.txt") { it.split("\t")[7] }
-        argFilter("src/test/resources/flights_head.txt") { it.split("\t")[7] == "UA" }
+        linesFrom(File("src/test/resources/flights_head.txt")).map { it.split("\t")[7] }.print()
+        linesFrom(File("src/test/resources/flights_head.txt")).filter { it.split("\t")[7] == "UA" }.print()
     }
 }
 }