bytecount working better

Author: fileformat

README.md

@@ -30,6 +30,7 @@
 - [ ] `ghash`: calculate various [hashes available in the Go standard library](https://pkg.go.dev/crypto#Hash)
 - [ ] `body`: prints specific lines of a file (in between `head` and `tail`)
 - [ ] `trilobyte`: translates bytes according to a map
+- [ ] `trune`: translates Unicode codepoints (runes) according to a map
 - [ ] `ustrings`: like the standard [`strings`](https://man7.org/linux/man-pages/man1/strings.1.html) utility, but with Unicode support
 - [ ] `utf8ify`: convert to UTF-8
 - [ ] `unhexdump`: convert the (edited) output of [`hexdump -C`](https://man7.org/linux/man-pages/man1/hexdump.1.html) back to binary

TODO.md

@@ -18,6 +18,7 @@
 
 ## Uniwhat and Unicount
 
+- [ ] Flag for whitespace (i.e. separate from control)
 - [ ] Flag to include/exclude specific scripts (not just ASCII)
 - [ ] Flag to print script
 - [ ] Flag to print block

cmd/bytecount/bytecount.go

@@ -4,11 +4,14 @@ import (
 	"bufio"
 	"fmt"
 	"io"
+	"log"
 	"os"
 
+	"github.com/FileFormatInfo/fftools/internal"
 	"github.com/olekukonko/tablewriter"
 	"github.com/olekukonko/tablewriter/renderer"
 	"github.com/olekukonko/tablewriter/tw"
+	"github.com/spf13/pflag"
 	"golang.org/x/text/language"
 	"golang.org/x/text/message"
 )
@@ -32,43 +35,91 @@ func outputPretty(out io.Writer, counts map[byte]int) {
 	)
 
 	header := []string{""}
-	for i := 0; i <= 0xf0; i += 0x10 {
+	for i := 0; i <= 0x0f; i += 1 {
 		header = append(header, fmt.Sprintf("0x%02X", i))
 	}
+	header = append(header, "")
 	table.Header(header)
 
-	for row := 0; row <= 0x0F; row += 0x01 {
+	for row := 0; row <= 0xF0; row += 0x10 {
 		data := []string{fmt.Sprintf("0x%02X", row)}
-		for col := 0; col <= 0xF0; col += 0x10 {
+		for col := 0; col <= 0x0F; col += 0x01 {
 			i := row + col
 			data = append(data, prettyPrinter.Sprintf("%d", counts[byte(i)]))
 		}
+		data = append(data, fmt.Sprintf("0x%02X", row))
 		table.Append(data)
 	}
 
 	table.Render()
 
 }
 
-func main() {
+func processFile(fileName string) error {
 	counts := make(map[byte]int)
 	for i := 0; i < 256; i++ {
 		counts[byte(i)] = 0
 	}
 
-	reader := bufio.NewReaderSize(os.Stdin, 1024*1024)
+	file, err := os.Open(fileName)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	reader := bufio.NewReaderSize(file, 1024*1024)
 	for {
 		b, err := reader.ReadByte()
 		if err == io.EOF {
 			break
 		}
 		if err != nil {
-			fmt.Fprintf(os.Stderr, "ERROR: unable to read input: %v\n", err)
-			os.Exit(1)
+			return err
 		}
 		counts[b]++
 	}
 
 	//LATER: other output formats: plain, JSON, CSV
 	outputPretty(os.Stdout, counts)
+
+	return nil
+}
+
+func main() {
+
+	var help = pflag.BoolP("help", "h", false, "Show help message")
+	var version = pflag.Bool("version", false, "Print version information")
+
+	pflag.Parse()
+
+	if *version {
+		internal.PrintVersion("bytecount")
+		return
+	}
+
+	if *help {
+		// LATER: print man page
+		fmt.Printf("Usage: bytecount [options] [file...]\n\n")
+		fmt.Printf("Options:\n")
+		pflag.PrintDefaults()
+		return
+	}
+
+	args := pflag.Args()
+	if len(args) == 0 {
+		fmt.Printf("Usage: bytecount [options] file ...\n\n")
+		return
+	}
+
+	for _, arg := range args {
+		if arg == "-" {
+			arg = "/dev/stdin"
+		}
+
+		if len(args) > 1 {
+			fmt.Printf("Processing file: %s\n", arg)
+		}
+
+		processFile(arg)
+	}
 }

cmd/uniwhat/uniwhat.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"bufio"
+	_ "embed"
 	"fmt"
 	"io"
 	"log"
@@ -12,14 +13,21 @@ import (
 	"golang.org/x/text/unicode/runenames"
 )
 
+//go:embed README.md
+var helpText string
+
 func main() {
 
-	var control = pflag.Bool("control", false, "Include control characters")
 	var ascii = pflag.Bool("ascii", false, "Include ASCII characters")
 	var codepoint = pflag.Bool("codepoint", true, "Print the U+XXXX codepoint")
+	var line = pflag.Bool("line", true, "Print the line number")
 	var offset = pflag.Bool("offset", true, "Print the offset")
 	var char = pflag.Bool("char", false, "Print the character itself")
-	var version = pflag.Bool("version", false, "Print version information")
+
+	var first = pflag.Bool("first", false, "Only print the first occurrence of each character")
+
+	var help = pflag.Bool("help", false, "Detailed help")
+	var version = pflag.Bool("version", false, "Version info")
 
 	pflag.Parse()
 
@@ -28,11 +36,20 @@ func main() {
 		return
 	}
 
+	if *help {
+		fmt.Printf("%s\n", helpText)
+		return
+	}
+
 	args := pflag.Args()
 	if len(args) == 0 {
-		args = []string{"-"}
+		fmt.Printf("Usage: uniwhat [options] file ...\n\n")
+		pflag.PrintDefaults()
+		return
 	}
 
+	firstMap := make(map[rune]bool)
+
 	for _, arg := range args {
 		if arg == "-" {
 			arg = "/dev/stdin"
@@ -46,9 +63,10 @@ func main() {
 		}
 		defer file.Close()
 
-		reader := bufio.NewReader(file)
+		reader := bufio.NewReaderSize(file, 1024*1024)
 
 		var pos int = 0
+		var lineNum int = 1
 
 		// Loop to read runes one by one
 		for {
@@ -59,21 +77,32 @@ func main() {
 				}
 				log.Fatalf("Error reading rune: %v", err)
 			}
-			if r < 0x1F && !*control {
-				pos += rsize
-				continue // Skip control characters if --control is not set
+			pos += rsize
+
+			if r == '\n' {
+				lineNum++
 			}
-			if r <= 0x7E && !*ascii {
-				pos += rsize
+
+			if !*ascii && ((r >= 0x20 && r <= 0x7E) || r == 0x09 || r == 0x0A || r == 0x0D) {
 				continue // Skip ASCII characters if --ascii is not set
 			}
+
+			if *first {
+				if _, exists := firstMap[r]; exists {
+					continue
+				}
+				firstMap[r] = true
+			}
+
 			name := runenames.Name(r)
 			if name == "" {
 				name = "<unknown>"
 			}
 			if *offset {
-				// Note: Getting the exact byte offset of the rune is complex; this is a placeholder
-				fmt.Printf("%08x ", pos)
+				fmt.Printf("%08x ", pos-rsize)
+			}
+			if *line {
+				fmt.Printf("%6d ", lineNum)
 			}
 			if *codepoint {
 				fmt.Printf("U+%04X ", r)
@@ -82,8 +111,6 @@ func main() {
 				fmt.Printf("%c ", r)
 			}
 			fmt.Printf("%s\n", name)
-
-			pos += rsize
 		}
 	}
 }

internal/Logger.go

@@ -10,6 +10,8 @@ import (
 
 var Logger = initLogger()
 var LogLevel = slog.LevelInfo
+const LevelTrace  = slog.Level(-8)
+
 
 // level=Trace is for stuff that should not be logged in production, either because it's too verbose or because it contains sensitive information.
 

internal/Version.go

@@ -13,7 +13,7 @@ var (
 )
 
 func PrintVersion(name string) {
-	if LogLevel >= slog.LevelInfo {
+	if LogLevel <= slog.LevelInfo {
 		slog.Info("Version information", "name", name, "version", VERSION, "lastmod", LASTMOD, "commit", COMMIT, "builder", BUILDER)
 	} else {
 		fmt.Printf("%s version %s (built on %s from %s by %s)\n", name, VERSION, LASTMOD, COMMIT, BUILDER)