paulgorman.org/technical

Go as a Glue Language

(March 2017)

Does Go (golang) work as a glue language? Yes, better that I expected.

Get command line arguments

Using the “os” package:

package main
import ("fmt"; "os")
func main() {
	programName := os.Args[0]
	fmt.Println(programName)
	if len(os.Args) > 1 { // Gratuitous in this example, but remember not to exceed the length of the array.
		for i := 1; i < len(os.Args); i++  {
			fmt.Println(os.Args[i])
		}
	}
}

Also see the “flag” package:

package main
import ("fmt"; "flag")
func main() {
	flagA := flag.Bool("a", false, "A command-line flag.")
	flagB := flag.String("b", "default value", "Another command-line flag.")
	flag.Parse()
	for _, arg := range flag.Args() {
		fmt.Println("Other argument: ", arg)
	}
	if *flagA {
		fmt.Println("Flag -a set!")
	}
	if *flagB != "default value" {
		fmt.Println("Flag -b non-default value: ", *flagB)
	}
}

The “flag” package insists on invocation like cmd -a -b argval not cmd -ab argval nor cmd -a foo bar -b argval.

Reading a file line by line by line

package main
import("bufio"; "fmt"; "log"; "os")
func main() {
	file, err := os.Open(os.Args[1])
	if err != nil {
		log.Fatal(err)
	}
	defer file.Close()
	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		fmt.Println(scanner.Text())
	}
	if err := scanner.Err(); err != nil {
		log.Fatal(err)
	}
}

Scanner does not deal well with lines longer than 65,536 characters.

Slurp in an entire (not too large) file at once

package main
import("fmt"; "io/ioutil"; "log"; "os")
func main() {
	wholeFile, err := ioutil.ReadFile(os.Args[1])
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println(string(wholeFile)) // ReadFile returns a byte array. Convert it to a string before printing.
}

Check if a file exists

If we want to do this in preparation for reading or writing the file, it’s better to just try to read or write and handle any error. However:

package main
import("fmt"; "os")
func main() {
	if _, err := os.Stat(os.Args[1]); os.IsNotExist(err) {
		fmt.Println("The file does NOT exist.")
	}
	if _, err := os.Stat(os.Args[1]); err == nil {
		fmt.Println("The file exists.")
	}
}

https://golang.org/pkg/os/#FileInfo

Write to a file

package main
import("fmt"; "log"; "os"; "strings")
func main() {
	if len(os.Args) < 3 {
		fmt.Println("Usage: writefile myfile.txt \"Foo bar!\"")
		os.Exit(1)
	}
	file, err := os.Create(os.Args[1])
	if err != nil {
		log.Fatal("Cannot create file", err)
	}
	defer file.Close()
	s := strings.Join(os.Args[2:], " ")
	fmt.Fprintf(file, s)
}

Recurse through directory tree

package main
import("fmt"; "log"; "os"; "path/filepath")
func walk(path string, f os.FileInfo, err error) error {
	if err != nil {
		log.Fatal(err)
	} else {
		fmt.Printf("Walked: %s\n", path)
	}
	return err
}
func main() {
	root := os.Args[1]
	err := filepath.Walk(root, walk)
	if err != nil {
		log.Fatal(err)
	}
}

https://golang.org/pkg/path/filepath/#Walk

All text files in a directory non-recursively

package main
import("fmt"; "log"; "path/filepath")
func main() {
	textFiles, err := filepath.Glob("/home/paulgorman/tmp/*txt")
	if err != nil {
		log.Fatal(err)
	}
	for _, file := range textFiles {
		fmt.Println(file)
	}
}
package main
import("log"; "os")
func main() {
	log.Println("Log writes to STDERR.")
	// But I don't want the timestamp prefix that "log" adds!
	lerr := log.New(os.Stderr, "", 0)
	lerr.Println("Boop.")
	lerr.Fatal("I'm out.")
}

https://golang.org/pkg/log/

Escape a string for the shell

Unnecessary. Exec’s *Cmd is safely parameterized. Quotes can be escaped like \“.

Run a shell command (system)

package main
import("log"; "os/exec")
func main() {
	cmd := exec.Command("touch", "-r", "/etc/hosts", "/tmp/foo")
	err := cmd.Run()
	if err != nil {
		log.Fatal(err)
	}
}

https://golang.org/pkg/os/exec/

Get output of a shell command

package main
import ("fmt"; "log"; "os/exec")
func main() {
	out, err := exec.Command("date").Output()
	if err != nil {
		log.Fatal(err)
	}
	fmt.Printf("The date is %s", out)
}

Check the numeric unix exit code of a shell command

package main
import ("log"; "os/exec"; "syscall")
func main() {
	cmd := exec.Command("ping", "-c 3", "-w 3", "10.99.99.99")
	if err := cmd.Start(); err != nil {
		log.Fatal(err)
	}
	if err := cmd.Wait(); err != nil { // cmd.Wait() sets err to <nil> on zero exit code.
		log.Println(err) // "2017/04/18 12:15:17 exit status 1" but err is not a comparable/testable int.
		exitstatus := cmd.ProcessState.Sys().(syscall.WaitStatus).ExitStatus()
		if exitstatus != 0 {
			log.Printf("Unix exit status '%d' (non-zero).", exitstatus)
		}
	}
}

Get the name of the current user

package main
import ( "fmt"; "log"; "os/user")
func main() {
	u, err := user.Current()
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println(u.Username)
}

https://golang.org/pkg/os/user/

Get the hostname

package main
import ("fmt"; "log"; "os")
func main() {
	h, err := os.Hostname()
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println(h)
}

Write a simple filter

package main
import ("bufio"; "fmt"; "io"; "log"; "os"; "strings")
func main() {
	scanner := bufio.NewScanner(os.Stdin)
	for scanner.Scan() { // Scan() return false when it hits EOF or an error.
		line := scanner.Text()
		fmt.Println(strings.ToUpper(line))
	}
	if err := scanner.Err(); err != nil { // Scan() returns nil if the error was io.EOF.
		log.Fatal(err)
	}
}

Hook up to the shell’s standard handles

See:

Spawn a new process, and don’t wait for it to exit

package main
import ("log"; "os/exec")
func main() {
	cmd := exec.Command("sleep", "5")
	err := cmd.Start()
	if err != nil {
		log.Fatal(err)
	}
}

https://golang.org/pkg/os/exec/#example_Cmd_Start

Inter-Process Communication

In my x-as-glue-language notes, I normally have several examples about IPC involving forking, unix sockets, named pipes, etc. Go is different.

Go has ForkExec(), for example, but for most uses Go offers better alternatives, like goroutines.

package main
import("fmt"; "math/rand"; "sync")
type Worker struct {
	id int
}
func work(w *Worker) {
	defer wg.Done()
	fmt.Println("Hello from", w.id)
}
var wg sync.WaitGroup // https://golang.org/pkg/sync/#WaitGroup
func main() {
	fmt.Println("Hello from PARENT")
	worker1 := &Worker{ id: rand.Int() }
	worker2 := &Worker{ id: rand.Int() }
	wg.Add(2)
	go work(worker1)
	go work(worker2)
	wg.Wait()
}

Read from and Write to a Unix Named Pipe (FIFO)

Web serve the contents of a directory

package main
import ("flag"; "log"; "net/http")
func main() {
	port := flag.String("p", "8080", "Port number from which to serve")
	dir := flag.String("d", "/usr/share/doc", "Directory to serve")
	flag.Parse()
	log.Fatal(http.ListenAndServe(":" + *port, http.FileServer(http.Dir(*dir))))
}

Web serve a simple message

Fetch URL (GET)

Fetch URL (POST)

Connect to MySQL, Sqlite

Although Go includes a “database/sql” package, it requires a database-specific driver.

$ go get github.com/go-sql-driver/mysql
$ go get github.com/mattn/go-sqlite3

Apart from the sql.Open() arguments, the “database/sql” package keeps the implementation detail identical regardless of which database driver we use.

Assume for MySQL:

MariaDB [(none)]> CREATE DATABASE testdb;
MariaDB [(none)]> USE testdb;
MariaDB [testdb]> CREATE TABLE testtable(id INT NOT NULL AUTO_INCREMENT, color VARCHAR(100), planet VARCHAR(100), scheduled DATETIME DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY(id));
MariaDB [testdb]> CREATE USER 'testuser'@'localhost' IDENTIFIED BY 'mypasswd';
MariaDB [testdb]> GRANT SELECT, INSERT, UPDATE, DELETE ON testdb.* TO 'testuser'@'localhost';
MariaDB [testdb]> FLUSH PRIVILEGES;

MySQL:

package main
import ("database/sql"; _ "github.com/go-sql-driver/mysql"; "fmt"; "log")
func main() {
	db, err := sql.Open("mysql", "testuser:mypasswd@/testdb")
	_, err = db.Exec("INSERT INTO testtable (color, planet) values (?, ?)", "red", "Mars")
	_, err = db.Exec("INSERT INTO testtable (color, planet) values (?, ?)", "green", "Venus")
	if err != nil { log.Fatal(err) }
	defer db.Close()
	rows, err := db.Query("SELECT planet FROM testtable WHERE color=?", "red")
	if err != nil { log.Fatal(err) }
	defer rows.Close()
	for rows.Next() {
		var planet string
		if err := rows.Scan(&planet); err != nil { log.Fatal(err) }
		fmt.Println(planet)
	}
	var planet string
	err = db.QueryRow("SELECT planet FROM testtable WHERE id=?", 1).Scan(&planet)
	switch {
	case err == sql.ErrNoRows:
		log.Printf("No planet with that ID. :(")
	case err != nil:
		log.Fatal(err)
	default:
		fmt.Printf("By Jove, it's the planet %s!\n", planet)
	}
}

Assume for Sqlite:

$ echo "CREATE TABLE testtable(id INTEGER PRIMARY KEY, color VARCHAR(100), planet VARCHAR(100), scheduled DATETIME DEFAULT CURRENT_TIMESTAMP);" | sqlite3 /tmp/test.db

Sqlite:

package main
import ("database/sql"; _ "github.com/mattn/go-sqlite3"; "fmt"; "log")
func main() {
	db, err := sql.Open("sqlite3", "/tmp/test.db")
	_, err = db.Exec("INSERT INTO testtable (color, planet) values (?, ?)", "red", "Mars")
	_, err = db.Exec("INSERT INTO testtable (color, planet) values (?, ?)", "green", "Venus")
	if err != nil { log.Fatal(err) }
	defer db.Close()
	rows, err := db.Query("SELECT planet FROM testtable WHERE color=?", "red")
	if err != nil { log.Fatal(err) }
	defer rows.Close()
	for rows.Next() {
		var planet string
		if err := rows.Scan(&planet); err != nil { log.Fatal(err) }
		fmt.Println(planet)
	}
	var planet string
	err = db.QueryRow("SELECT planet FROM testtable WHERE id=?", 1).Scan(&planet)
	switch {
	case err == sql.ErrNoRows:
		log.Printf("No planet with that ID. :(")
	case err != nil:
		log.Fatal(err)
	default:
		fmt.Printf("By Jove, it's the planet %s!\n", planet)
	}
}

SQL and Goroutines

For most databases (including MySQL), sql.Open() returns the handle to a pool of connections, rather than a single connection. This connection pool plays nicely with goroutines, so we don’t have to worry when sharing the database handle between them.

However, SQLite works differently. It allows multiple simultaneous readers but only one writer at a time. Unlike with other databases, make one sql.Open() call per goroutine.

With significant contention, goroutines hitting SQLite may get “database is locked” errors. We can minimize this problem by sending a few flags to SQLite, notably “_busy_timeout”:

db, err := sql.Open("sqlite3", "database_file.sqlite?_busy_timeout=5000&cache=shared&mode=rwc")

Safety and SQL Injection

Using parameterized queries (i.e. where data are substituted into the query at “?”) should be safe. If in doubt, use Prepare() explicitly, although Go does this implicitly anytime we do something like db.Query(sql, param1, param2). https://golang.org/pkg/database/sql/#DB.Prepare

Regular Expressions

package main
import ("fmt"; "regexp")
func main() {
	match, _ := regexp.MatchString(".([a-z]{2})ch", "beach")
	fmt.Println(match) // --> true
	re, _ := regexp.Compile(".([a-z]{2})ch")
	fmt.Println(re.MatchString("peach")) // --> true
	fmt.Println(re.FindString("Somewhere beyond the starry reaches, they vanished.")) // --> reach
	fmt.Println(re.FindStringSubmatch("Somewhere beyond the starry reaches, they vanished.")) // --> [reach ea]
	fmt.Println(re.FindStringIndex("Somewhere beyond the starry reaches, they vanished.")) // --> [28 33]
	fmt.Println(re.ReplaceAllString("Somewhere beyond the starry reaches, they vanished.", "speech"))
}

https://golang.org/pkg/regexp/