More tools (uniwhat, unicount), started on goreleaser

Author: fileformat

.gitignore

@@ -23,3 +23,5 @@ tmp-*
 venv/
 .venv/
 _work/
+# Added by goreleaser init:
+dist/

.goreleaser.yaml

@@ -0,0 +1,153 @@
+version: 2
+
+project_name: fftools
+
+before:
+  hooks:
+    # You may remove this if you don't use go modules.
+    - go mod tidy
+    # you may remove this if you don't need go generate
+    - go generate ./...
+
+builds:
+  - id: asciify
+    main: ./cmd/asciify/asciify.go
+    binary: asciify
+    env:
+      - CGO_ENABLED=0
+    goos:
+      - linux
+      - windows
+      - darwin
+
+  - id: asciitable
+    main: ./cmd/asciitable/asciitable.go
+    binary: asciitable
+    env:
+      - CGO_ENABLED=0
+    goos:
+      - linux
+      - windows
+      - darwin
+
+  - id: bytecount
+    main: ./cmd/bytecount/bytecount.go
+    binary: bytecount
+    env:
+      - CGO_ENABLED=0
+    goos:
+      - linux
+      - windows
+      - darwin
+
+  - id: certinfo
+    main: ./cmd/certinfo/certinfo.go
+    binary: certinfo
+    env:
+      - CGO_ENABLED=0
+    goos:
+      - linux
+      - windows
+      - darwin
+
+  - id: hexdumpc
+    main: ./cmd/hexdumpc/hexdumpc.go
+    binary: hexdumpc
+    env:
+      - CGO_ENABLED=0
+    goos:
+      - linux
+      - windows
+      - darwin
+
+  - id: unicount
+    main: ./cmd/unicount/unicount.go
+    binary: unicount
+    env:
+      - CGO_ENABLED=0
+    goos:
+      - linux
+      - windows
+      - darwin
+
+  - id: uniwhat
+    main: ./cmd/uniwhat/uniwhat.go
+    binary: uniwhat
+    env:
+      - CGO_ENABLED=0
+    goos:
+      - linux
+      - windows
+      - darwin
+
+archives:
+  - formats: [tar.gz]
+    # this name template makes the OS and Arch compatible with the results of `uname`.
+    name_template: >-
+      {{ .ProjectName }}_
+      {{- title .Os }}_
+      {{- if eq .Arch "amd64" }}x86_64
+      {{- else if eq .Arch "386" }}i386
+      {{- else }}{{ .Arch }}{{ end }}
+      {{- if .Arm }}v{{ .Arm }}{{ end }}
+    # use zip for windows archives
+    format_overrides:
+      - goos: windows
+        formats: [zip]
+    ids:
+      - asciify
+      - asciitable
+      - bytecount
+      - certinfo
+      - hexdumpc
+      - unicount
+      - uniwhat
+
+nfpms:
+  - id: fftools
+    package_name: fftools
+    ids:
+      - asciify
+      - asciitable
+      - bytecount
+      - certinfo
+      - hexdumpc
+      - unicount
+      - uniwhat
+    vendor: FileFormatInfo
+    homepage: "https://www.fileformat.info/"
+    maintainer: Andrew Marcuse <fileformat@gmail.com>
+    description: |-
+      Utilities for analyzing, viewing and fixing file formats.
+    license: GPL-3.0-and-later
+    formats:
+      - deb
+      - rpm
+
+release:
+  github:
+    owner: FileFormatInfo
+    name: fftools
+
+report_sizes: true
+
+source:
+  enabled: true
+  name_template: "{{ .ProjectName }}.src"
+
+upx:
+  - enabled: true
+
+changelog:
+  sort: asc
+  filters:
+    exclude:
+      - "^docs:"
+      - "^test:"
+
+release:
+  footer: >-
+
+    ---
+
+    Released by [GoReleaser](https://github.com/goreleaser/goreleaser).

README.md

@@ -1,23 +1,35 @@
 # File Format Tools [<img alt="Logo for fftools" src="docs/favicon.svg" height="96" align="right"/>](https://www.fileformat.info/)
 
+
+## Programs
+
+- `asciify`: converts to ASCII using [anyascii](https://github.com/anyascii/anyascii)
+- `asciitable`: prints out an table of ASCII characters
+- `bytecount`: counts the number of occurrences of each byte
+- `certinfo`: print info about an x509 (aka SSL/HTTPS) certificate
+- `hexdumpc`: generate canonical hexdump (`hexdump -C`) output in case you don't have [`hexdump`](https://man7.org/linux/man-pages/man1/hexdump.1.html) installed
+- `unicount`: count Unicode codepoints in a file
+- `uniwhat`: print the names of each Unicode character in a file
+
 ## Credits
 
 [![Git](https://www.vectorlogo.zone/logos/git-scm/git-scm-ar21.svg)](https://git-scm.com/ "Version control")
 [![Github](https://www.vectorlogo.zone/logos/github/github-ar21.svg)](https://github.com/ "Code hosting")
 [![golang](https://www.vectorlogo.zone/logos/golang/golang-ar21.svg)](https://golang.org/ "Programming language")
 [![svgrepo](https://www.vectorlogo.zone/logos/svgrepo/svgrepo-ar21.svg)](https://www.svgrepo.com/svg/276165/gardening-tools-rake "Icon")
 
-## To Do
+## Scripts To Do
 
-- [x] `asciify`: converts to ASCII using [anyascii](https://github.com/anyascii/anyascii)
-- [x] `asciitable`: prints out an table of ASCII characters
-- [x] `bytecount`: counts the number of occurrences of each byte
-- [x] `certinfo`: print info about an x509 (aka SSL/HTTPS) certificate
 - [ ] `ghash`: calculate various [hashes available in the Go standard library](https://pkg.go.dev/crypto#Hash)
-- [x] `hexdumpc`: generate canonical hexdump (`hexdump -C`) output in case you don't have [`hexdump`](https://man7.org/linux/man-pages/man1/hexdump.1.html) installed
+- [ ] `body`: prints specific lines of a file (in between `head` and `tail`)
 - [ ] `trilobyte`: translates bytes according to a map
 - [ ] `ustrings`: like the standard [`strings`](https://man7.org/linux/man-pages/man1/strings.1.html) utility, but with Unicode support
+- [ ] `utf8ify`: convert to UTF-8
 - [ ] `unhexdump`: convert the (edited) output of [`hexdump -C`](https://man7.org/linux/man-pages/man1/hexdump.1.html) back to binary
-- [ ] proper command line parsing (cobra)
+
+
+## General
+
 - [ ] man pages
-- [ ] distribution packages
+- [ ] goreleaser
+- [ ] [release security](https://github.com/goreleaser/example-secure)

bin/build.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+#
+# build binaries
+#
+
+set -o errexit
+set -o pipefail
+set -o nounset
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)"
+
+DIST_DIR="${REPO_DIR}/dist"
+
+echo "INFO: starting at $(date -u +%Y-%m-%dT%H:%M:%SZ)"
+
+if [ ! -d "${DIST_DIR}" ]; then
+	echo "INFO: creating dist directory ${DIST_DIR}"
+	mkdir "${REPO_DIR}/dist"
+fi
+
+FILES=$(ls "${REPO_DIR}/cmd")
+
+for f in $FILES; do
+	if [ -f "${DIST_DIR}/${f}" ]; then
+		echo "WARNING: file ${DIST_DIR}/${f} already exists"
+		continue
+	fi
+	echo "INFO: compiling ${f} to dist directory"
+	go build -o "${DIST_DIR}/${f}" "${REPO_DIR}/cmd/${f}"
+done
+
+echo "INFO: complete at $(date -u +%Y-%m-%dT%H:%M:%SZ)"

bin/clean.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+#
+# clean up build artifacts
+#
+
+set -o errexit
+set -o pipefail
+set -o nounset
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)"
+
+DIST_DIR="${REPO_DIR}/dist"
+
+echo "INFO: starting at $(date -u +%Y-%m-%dT%H:%M:%SZ)"
+
+if [ -d "${DIST_DIR}" ]; then
+	echo "INFO: removing dist directory ${DIST_DIR}"
+	rm -rf "${DIST_DIR}"
+else
+	echo "INFO: dist directory ${DIST_DIR} does not exist, nothing to clean"
+fi
+
+echo "INFO: complete at $(date -u +%Y-%m-%dT%H:%M:%SZ)"

cmd/ghash/ghash.go

@@ -8,6 +8,10 @@ import (
 	_ "crypto/sha256"
 	_ "crypto/sha3"
 	_ "crypto/sha512"
+	"hash"
+	"io"
+	"regexp"
+	"strings"
 
 	"fmt"
 	"os"
@@ -20,7 +24,16 @@ import (
 	"github.com/spf13/pflag"
 )
 
-func printHashes(fileName string, hashes []string) error {
+// struct with name and hash
+type Hasher struct {
+	name string
+	hash hash.Hash
+}
+
+// map of name to Hasher
+var hasherMap = map[string]Hasher{}
+
+func printHashes(fileName string, hashers []Hasher) error {
 	// Open the file
 	file, err := os.Open(fileName)
 	if err != nil {
@@ -30,29 +43,82 @@ func printHashes(fileName string, hashes []string) error {
 
 	reader := bufio.NewReader(file)
 
+	// Read the file in chunks and update the hashes
+	buf := make([]byte, 64*1024)
+	for {
+		n, err := reader.Read(buf)
+		if n > 0 {
+			for _, h := range hashers {
+				h.hash.Write(buf[:n])
+			}
+		}
+		if err != nil {
+			if err == io.EOF {
+				break
+			}
+			return err
+		}
+	}
+
+	// Print the results
+	for _, h := range hashers {
+		if len(hashers) > 1 {
+			fmt.Printf("%x: %s %s\n", h.hash.Sum(nil), h.name, fileName)
+		} else {
+			fmt.Printf("%x: %s\n", h.hash.Sum(nil), fileName)
+		}
+	}
+
 	return nil
 }
 
+var nonAlphaNumeric = regexp.MustCompile(`[^a-zA-Z0-9]`)
+
 func main() {
-	var hashes []string
+	var hashNames []string
 	var list bool
-	pflag.StringArrayVar(&hashes, "hash", []string{}, "Hash algorithms to use (e.g., sha256, sha512)")
+	pflag.StringArrayVar(&hashNames, "hash", []string{}, "Hash algorithms to use (e.g., sha256, sha512)")
 	pflag.BoolVar(&list, "list", false, "List available hash algorithms")
 	//LATER: output format
 
 	pflag.Parse()
 
 	if list {
 		for i := crypto.MD4; i <= crypto.BLAKE2b_512; i++ {
+			hashName := nonAlphaNumeric.ReplaceAllString(strings.ToLower(i.String()), "")
 			if i.Available() {
-				fmt.Fprintf(os.Stdout, "%s: %s\n", i, "true")
+				fmt.Fprintf(os.Stdout, "%s: %s\n", hashName, "true")
 			} else {
-				fmt.Fprintf(os.Stdout, "%s: %s\n", i, "false")
+				fmt.Fprintf(os.Stdout, "%s: %s\n", hashName, "false")
 			}
 		}
 		return
 	}
 
+	// build hashers map
+	for i := crypto.MD4; i <= crypto.BLAKE2b_512; i++ {
+		if i.Available() {
+			hashName := nonAlphaNumeric.ReplaceAllString(strings.ToLower(i.String()), "")
+			theHash := Hasher{name: hashName, hash: i.New()}
+			hasherMap[hashName] = theHash
+			hasherMap[strings.ToLower(i.String())] = theHash
+		}
+	}
+
+	var hashers = []Hasher{}
+	if len(hashNames) == 0 {
+		hashers = append(hashers, hasherMap["sha256"])
+	} else {
+		for _, name := range hashNames {
+			var h = hasherMap[name]
+			if h.hash == nil {
+				fmt.Fprintf(os.Stderr, "Unknown or unavailable hash algorithm: %s\n", name)
+				os.Exit(1)
+			}
+			hashers = append(hashers, h)
+		}
+	}
+
 	args := pflag.Args()
 	if len(args) == 0 {
 		args = []string{"-"}
@@ -62,7 +128,7 @@ func main() {
 			arg = "/dev/stdin"
 		}
 
-		if err := printHashes(arg, hashes); err != nil {
+		if err := printHashes(arg, hashers); err != nil {
 			panic(err)
 		}
 	}