Go as a Glue Language

(March 2017)

Does Go (golang) work as a glue language? Yes, better that I expected.

Get command line arguments

Using the “os” package:

package main

import ("fmt"; "os")

func main() {
	programName := os.Args[0]
	fmt.Println(programName)
	if len(os.Args) > 1 { // Gratuitous in this example, but remember not to exceed the length of the array.
		for i := 1; i < len(os.Args); i++  {
			fmt.Println(os.Args[i])
		}
	}
}

Also see the “flag” package:

package main

import ("fmt"; "flag")

func main() {
	flagA := flag.Bool("a", false, "A command-line flag.")
	flagB := flag.String("b", "default value", "Another command-line flag.")
	flag.Parse()
	for _, arg := range flag.Args() {
		fmt.Println("Other argument: ", arg)
	}
	if *flagA {
		fmt.Println("Flag -a set!")
	}
	if *flagB != "default value" {
		fmt.Println("Flag -b non-default value: ", *flagB)
	}
}

The “flag” package insists on invocation like cmd -a -b argval not cmd -ab argval nor cmd -a foo bar -b argval.

Reading a file line by line by line

package main

import("bufio"; "fmt"; "log"; "os")

func main() {
	file, err := os.Open(os.Args[1])
	if err != nil {
		log.Fatal(err)
	}
	defer file.Close()
	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		fmt.Println(scanner.Text())
	}
	if err := scanner.Err(); err != nil {
		log.Fatal(err)
	}
}

Scanner does not deal well with lines longer than 65,536 characters.

Slurp in an entire (not too large) file at once

package main

import("fmt"; "io/ioutil"; "log"; "os")

func main() {
	wholeFile, err := ioutil.ReadFile(os.Args[1])
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println(string(wholeFile)) // ReadFile returns a byte array. Convert it to a string before printing.
}

Check if a file exists

If we want to do this in preparation for reading or writing the file, it’s better to just try to read or write and handle any error. However:

package main

import("fmt"; "os")

func main() {
	if _, err := os.Stat(os.Args[1]); os.IsNotExist(err) {
		fmt.Println("The file does NOT exist.")
	}
	if _, err := os.Stat(os.Args[1]); err == nil {
		fmt.Println("The file exists.")
	}
}

https://golang.org/pkg/os/#FileInfo

Write to a file

package main

import("fmt"; "log"; "os"; "strings")

func main() {
	if len(os.Args) < 3 {
		fmt.Println("Usage: writefile myfile.txt \"Foo bar!\"")
		os.Exit(1)
	}
	file, err := os.Create(os.Args[1])
	if err != nil {
		log.Fatal("Cannot create file", err)
	}
	defer file.Close()
	s := strings.Join(os.Args[2:], " ")
	fmt.Fprintf(file, s)
}

Recurse through directory tree

package main

import("fmt"; "log"; "os"; "path/filepath")

func walk(path string, f os.FileInfo, err error) error {
	if err != nil {
		log.Fatal(err)
	} else {
		fmt.Printf("Walked: %s\n", path)
	}
	return err
}
func main() {
	root := os.Args[1]
	err := filepath.Walk(root, walk)
	if err != nil {
		log.Fatal(err)
	}
}

https://golang.org/pkg/path/filepath/#Walk

All text files in a directory non-recursively

package main

import("fmt"; "log"; "path/filepath")

func main() {
	textFiles, err := filepath.Glob("/home/paulgorman/tmp/*txt")
	if err != nil {
		log.Fatal(err)
	}
	for _, file := range textFiles {
		fmt.Println(file)
	}
}

Print to STDERR

package main

import("log"; "os")

func main() {
	log.Println("Log writes to STDERR.")
	// But I don't want the timestamp prefix that "log" adds!
	lerr := log.New(os.Stderr, "", 0)
	lerr.Println("Boop.")
	lerr.Fatal("I'm out.")
}

https://golang.org/pkg/log/

Escape a string for the shell

Unnecessary. Exec’s *Cmd is safely parameterized. Quotes can be escaped like \“.

Run a shell command (system)

package main

import("log"; "os/exec")

func main() {
	cmd := exec.Command("touch", "-r", "/etc/hosts", "/tmp/foo")
	err := cmd.Run()
	if err != nil {
		log.Fatal(err)
	}
}

https://golang.org/pkg/os/exec/

Get output of a shell command

package main

import ("fmt"; "log"; "os/exec")

func main() {
	out, err := exec.Command("date").Output()
	if err != nil {
		log.Fatal(err)
	}
	fmt.Printf("The date is %s", out)
}

Check the numeric unix exit code of a shell command

package main

import ("log"; "os/exec"; "syscall")

func main() {
	cmd := exec.Command("ping", "-c 3", "-w 3", "10.99.99.99")
	if err := cmd.Start(); err != nil {
		log.Fatal(err)
	}
	if err := cmd.Wait(); err != nil { // cmd.Wait() sets err to <nil> on zero exit code.
		log.Println(err) // "2017/04/18 12:15:17 exit status 1" but err is not a comparable/testable int.
		exitstatus := cmd.ProcessState.Sys().(syscall.WaitStatus).ExitStatus()
		if exitstatus != 0 {
			log.Printf("Unix exit status '%d' (non-zero).", exitstatus)
		}
	}
}

Get the name of the current user

package main

import ( "fmt"; "log"; "os/user")

func main() {
	u, err := user.Current()
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println(u.Username)
}

https://golang.org/pkg/os/user/

Get the hostname

package main

import ("fmt"; "log"; "os")

func main() {
	h, err := os.Hostname()
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println(h)
}

Write a simple filter

package main

import ("bufio"; "fmt"; "io"; "log"; "os"; "strings")

func main() {
	scanner := bufio.NewScanner(os.Stdin)
	for scanner.Scan() { // Scan() return false when it hits EOF or an error.
		line := scanner.Text()
		fmt.Println(strings.ToUpper(line))
	}
	if err := scanner.Err(); err != nil { // Scan() returns nil if the error was io.EOF.
		log.Fatal(err)
	}
}

Hook up to the shell’s standard handles

See:

Spawn a new process, and don’t wait for it to exit

package main

import ("log"; "os/exec")

func main() {
	cmd := exec.Command("sleep", "5")
	err := cmd.Start()
	if err != nil {
		log.Fatal(err)
	}
}

https://golang.org/pkg/os/exec/#example_Cmd_Start

Inter-Process Communication

In my x-as-glue-language notes, I normally have several examples about IPC involving forking, unix sockets, named pipes, etc. Go is different.

Go has ForkExec(), for example, but for most uses Go offers better alternatives, like goroutines.

package main

import("fmt"; "math/rand"; "sync")

type Worker struct {
	id int
}
func work(w *Worker) {
	defer wg.Done()
	fmt.Println("Hello from", w.id)
}
var wg sync.WaitGroup // https://golang.org/pkg/sync/#WaitGroup
func main() {
	fmt.Println("Hello from PARENT")
	worker1 := &Worker{ id: rand.Int() }
	worker2 := &Worker{ id: rand.Int() }
	wg.Add(2)
	go work(worker1)
	go work(worker2)
	wg.Wait()
}

Read from and Write to a Unix Named Pipe (FIFO)

Web serve the contents of a directory

package main

import ("flag"; "log"; "net/http")

func main() {
	port := flag.String("p", "8080", "Port number from which to serve")
	dir := flag.String("d", "/usr/share/doc", "Directory to serve")
	flag.Parse()
	log.Fatal(http.ListenAndServe(":" + *port, http.FileServer(http.Dir(*dir))))
}

Web serve a simple message

Fetch URL (GET)

Fetch URL (POST)

Connect to MySQL, Sqlite

Although Go includes a “database/sql” package, it requires a database-specific driver.

$  go get github.com/go-sql-driver/mysql
$  go get github.com/mattn/go-sqlite3

Apart from the sql.Open() arguments, the “database/sql” package keeps the implementation detail identical regardless of which database driver we use.

Assume for MySQL:

MariaDB [(none)]> CREATE DATABASE testdb;
MariaDB [(none)]> USE testdb;
MariaDB [testdb]> CREATE TABLE testtable(id INT NOT NULL AUTO_INCREMENT, color VARCHAR(100), planet VARCHAR(100), scheduled DATETIME DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY(id));
MariaDB [testdb]> CREATE USER 'testuser'@'localhost' IDENTIFIED BY 'mypasswd';
MariaDB [testdb]> GRANT SELECT, INSERT, UPDATE, DELETE ON testdb.* TO 'testuser'@'localhost';
MariaDB [testdb]> FLUSH PRIVILEGES;

MySQL:

package main

import ("database/sql"; _ "github.com/go-sql-driver/mysql"; "fmt"; "log")

func main() {
	db, err := sql.Open("mysql", "testuser:mypasswd@/testdb")
	_, err = db.Exec("INSERT INTO testtable (color, planet) values (?, ?)", "red", "Mars")
	_, err = db.Exec("INSERT INTO testtable (color, planet) values (?, ?)", "green", "Venus")
	if err != nil { log.Fatal(err) }
	defer db.Close()
	rows, err := db.Query("SELECT planet FROM testtable WHERE color=?", "red")
	if err != nil { log.Fatal(err) }
	defer rows.Close()
	for rows.Next() {
		var planet string
		if err := rows.Scan(&planet); err != nil { log.Fatal(err) }
		fmt.Println(planet)
	}
	var planet string
	err = db.QueryRow("SELECT planet FROM testtable WHERE id=?", 1).Scan(&planet)
	switch {
	case err == sql.ErrNoRows:
		log.Printf("No planet with that ID. :(")
	case err != nil:
		log.Fatal(err)
	default:
		fmt.Printf("By Jove, it's the planet %s!\n", planet)
	}
}

Assume for Sqlite:

$  echo "CREATE TABLE testtable(id INTEGER PRIMARY KEY, color VARCHAR(100), planet VARCHAR(100), scheduled DATETIME DEFAULT CURRENT_TIMESTAMP);" | sqlite3 /tmp/test.db

Sqlite:

package main

import ("database/sql"; _ "github.com/mattn/go-sqlite3"; "fmt"; "log")

func main() {
	db, err := sql.Open("sqlite3", "/tmp/test.db")
	_, err = db.Exec("INSERT INTO testtable (color, planet) values (?, ?)", "red", "Mars")
	_, err = db.Exec("INSERT INTO testtable (color, planet) values (?, ?)", "green", "Venus")
	if err != nil { log.Fatal(err) }
	defer db.Close()
	rows, err := db.Query("SELECT planet FROM testtable WHERE color=?", "red")
	if err != nil { log.Fatal(err) }
	defer rows.Close()
	for rows.Next() {
		var planet string
		if err := rows.Scan(&planet); err != nil { log.Fatal(err) }
		fmt.Println(planet)
	}
	var planet string
	err = db.QueryRow("SELECT planet FROM testtable WHERE id=?", 1).Scan(&planet)
	switch {
	case err == sql.ErrNoRows:
		log.Printf("No planet with that ID. :(")
	case err != nil:
		log.Fatal(err)
	default:
		fmt.Printf("By Jove, it's the planet %s!\n", planet)
	}
}

SQL and Goroutines

For most databases (including MySQL), sql.Open() returns the handle to a pool of connections, rather than a single connection. This connection pool plays nicely with goroutines, so we don’t have to worry when sharing the database handle between them.

However, SQLite works differently. It allows multiple simultaneous readers but only one writer at a time. Unlike with other databases, make one sql.Open() call per goroutine.

With significant contention, goroutines hitting SQLite may get “database is locked” errors. We might minimize this problem by sending a few flags to SQLite, like “_busy_timeout”.

db, err := sql.Open("sqlite3", "database_file.sqlite?_busy_timeout=5000&cache=shared&mode=rwc")

More thoughts about SQLite from 2019

SQLite itself can be safe for concurrent writes, as long as each concurrent thing makes a new fopen() call.
SQLite keeps this safe by managing its own write locks and mutexes.
However, the Go database/sql expects to use connection pooling — keeping a group of open database connection to share between goroutines in order to minimize the overhead of opening new connections. Connection pooling is pointless (and potentially dangerous) for embedded databases like SQLite, where initiating new connections is a simple fopen().
One of the first and most popular Go packages for SQLite, github.com/mattn/go-sqlite3, is not safe for concurrent writes because it follows the the database/sql connection-pooling model. A package like github.com/bvinc/go-sqlite-lite that eschews the database/sql model and just lightly wraps the SQLite C calls can promise concurrency safety as long as we open a new database connection for each goroutine.
For performance reasons, SQLite may be a poor choice for an application with heavy concurrent writes. But to quote the SQLite FAQ: “experience suggests that most applications need much less concurrency than their designers imagine”.
https://www.sqlite.org/lockingv3.html https://www.sqlite.org/wal.html

More thoughts about SQLite from 2020

SQLite itself is safe for concurrent connections.
The connection pooling model of Go’s database/sql package is a poor fit with the locking model of SQLite.
The safest course of action when using SQLite with a Go driver like github.com/mattn/go-sqlite3 is to guard database access with a mutex (sync.Mutex).
Do we need a mutex on both read and write?

Safety and SQL Injection

Using parameterized queries (i.e. where data are substituted into the query at “?”) should be safe. If in doubt, use Prepare() explicitly, although Go does this implicitly anytime we do something like db.Query(sql, param1, param2). https://golang.org/pkg/database/sql/#DB.Prepare

Regular Expressions

package main

import ("fmt"; "regexp")

func main() {
	match, _ := regexp.MatchString(".([a-z]{2})ch", "beach")
	fmt.Println(match) // --> true
	re, _ := regexp.Compile(".([a-z]{2})ch")
	fmt.Println(re.MatchString("peach")) // --> true
	fmt.Println(re.FindString("Somewhere beyond the starry reaches, they vanished.")) // --> reach
	fmt.Println(re.FindStringSubmatch("Somewhere beyond the starry reaches, they vanished.")) // --> [reach ea]
	fmt.Println(re.FindStringIndex("Somewhere beyond the starry reaches, they vanished.")) // --> [28 33]
	fmt.Println(re.ReplaceAllString("Somewhere beyond the starry reaches, they vanished.", "speech"))
}

https://golang.org/pkg/regexp/

Build for Multiple Platforms (Conditional Compilation)

Just as Go supports multiple platforms through the GOOS environment variable, go build is smart enough to use variant code based on file names.

main.go:

package main

import ("fmt"; "os/exec")

func main() {
	cmd := GetCmd()
	out, err := exec.Command(cmd[0], cmd[1:]...).Output()
	if err != nil {
		fmt.Println(err)
	}
	fmt.Println(out)
}

main_linux.go:

package main

func GetCmd() []string {
	return []string{"ip", "address", "show"}
}

main_windows.go:

package main

func GetCmd() []string {
	return []string{"ifconfig", "/all"}
}

Note that “unix” is not a valid GOOS value, but we can specify what we mean with +build tag. main_unix.go:

// +build openbsd freebsd netbsd darwin

package main

func GetCmd() []string {
	return []string{"ifconfig", "-a"}
}

(Note that build tags can also specify GOARCH like // +build 386.)

https://www.youtube.com/watch?v=hsgkdMrEJPs