Merge branch 'release/v0.2.0-alpha'

CHANGELOG.md

@@ -4,6 +4,20 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+## [v0.2.0-alpha] - 2017-02-07
+
+### Fixed
+- Shut down the UI 10 seconds after the crawler finished
+
+### Changed
+- Remove the TERABYTE constant from the byte formatter so cross-compilation for ARM works
+- Remove the `reset` command hint from the troubleshooting section of the README
+- Start the timer only after the links in the XML sitemap(s) have been read
+- Stop counting the seconds after the crawler finished
+
+### Removed
+- Remove the debug console from the command-line ui
+
 ## [v0.1.0-alpha] - 2017-02-07
 
 The Prototype

README.md

@@ -2,6 +2,19 @@
 
 The fast website crawler
 
+You can use「 gargantua 」to quickly and easily
+
+- **warm-up** your frontend caches
+- perform small **load-tests** against your publicly available pages
+- **measure** response times
+- **detect** broken links
+
+from your command line on Linux, macOS and Windows.
+
+![Animation: gargantua v0.1.0 crawling a website](files/gargantua-in-action-crawling-a-website.gif)
+
+> Note: Press `Q` to stop the current crawling process.
+
 ## Usage
 
 Crawl **www.sitemaps.org** with 5 concurrent workers:
@@ -10,25 +23,40 @@ Crawl **www.sitemaps.org** with 5 concurrent workers:
 gargantua crawl --url https://www.sitemaps.org/sitemap.xml --workers 5
 ```
 
-## Roadmap
+see also: [A short introduction video of gargantua on YouTube](https://www.youtube.com/watch?v=TSCMvUvc0qo)
 
-- Increase the number of workers at runtime
-- Personalized user agent string
-- Silent mode (only show statistics at the end)
-- CSV mode (print CSV output to stdout)
-- Dockerfile
-- Web-UI
+## Download
+
+You can download binaries for Linux, macOS and Windows from [github.com »andreaskoch » gargantua » releases](https://github.com/andreaskoch/gargantua/releases):
 
-## Troubleshooting
+```bash
+wget https://github.com/andreaskoch/gargantua/releases/download/v0.2.0-alpha/gargantua_linux_amd64
+```
+
+## Docker Image
 
-### My console is messed up after gargantua exits
+There is also a docker image that you can use to download or run the latest version of gargantua:
 
-This has to do with gargantua's usage of ncurses. You can use the `reset` command to fix your command line window.
+[andreaskoch/gargantua](https://hub.docker.com/r/andreaskoch/gargantua/)
 
 ```bash
-reset
+docker run --rm andreaskoch/gargantua:latest \
+       crawl \
+       --verbose \
+       --url https://www.sitemaps.org/sitemap.xml \
+       --workers 5
 ```
 
+**Note**: You will need the `--verbose` flag in order to prevent the command-line UI from loading. Otherwise gargantua will fail.
+
+## Roadmap
+
+- Increase the number of workers at runtime
+- Personalized user agent string
+- Silent mode (only show statistics at the end)
+- CSV mode (print CSV output to stdout)
+- Web-UI
+
 ## License
 
 「 gargantua 」is licensed under the Apache License, Version 2.0. See [LICENSE](LICENSE) for the full license text.

dashboard.go

@@ -7,7 +7,7 @@ import (
 	"github.com/gizak/termui"
 )
 
-func dashboard(startTime time.Time, stopTheCrawler chan bool) {
+func dashboard(stopTheUI, stopTheCrawler chan bool) {
 	if err := termui.Init(); err != nil {
 		panic(err)
 	}
@@ -69,16 +69,23 @@ func dashboard(startTime time.Time, stopTheCrawler chan bool) {
 	elapsedTime.BorderLabel = "Elapsed time"
 	elapsedTime.BorderFg = termui.ColorCyan
 
-	debugMessages := termui.NewList()
-	debugMessages.ItemFgColor = termui.ColorWhite
-	debugMessages.BorderLabel = "Debug"
-	debugMessages.Height = 6
-	debugMessages.Y = 0
-
 	draw := func() {
 
 		snapshot := stats.LastSnapshot()
 
+		// ignore empty updates
+		if snapshot.Timestamp().IsZero() {
+			return
+		}
+
+		// don't update if there is no new snapshot available
+		if len(snapshots) > 0 {
+			previousSnapShot := snapshots[len(snapshots)-1]
+			if snapshot.Timestamp() == previousSnapShot.Timestamp() {
+				return
+			}
+		}
+
 		// capture the latest snapshot
 		snapshots = append(snapshots, snapshot)
 
@@ -106,11 +113,8 @@ func dashboard(startTime time.Time, stopTheCrawler chan bool) {
 		// number of errors
 		numberOfErrors.Text = fmt.Sprintf("%d", snapshot.NumberOfErrors())
 
-		// debug messages
-		debugMessages.Items = errorMessages
-
 		// time since first snapshot
-		timeSinceStart := time.Now().Sub(startTime)
+		timeSinceStart := time.Now().Sub(snapshots[0].Timestamp())
 		elapsedTime.Text = fmt.Sprintf("%s", timeSinceStart)
 
 		termui.Render(termui.Body)
@@ -132,9 +136,6 @@ func dashboard(startTime time.Time, stopTheCrawler chan bool) {
 			termui.NewCol(3, 0, averageSizeInBytes),
 			termui.NewCol(3, 0, elapsedTime),
 		),
-		termui.NewRow(
-			termui.NewCol(12, 0, debugMessages),
-		),
 	)
 
 	termui.Body.Align()
@@ -158,5 +159,15 @@ func dashboard(startTime time.Time, stopTheCrawler chan bool) {
 		draw()
 	})
 
+	// stop when the crawler is done
+	go func() {
+		select {
+		case <-stopTheUI:
+			// wait 10 seconds before closing the ui
+			time.Sleep(time.Second * 10)
+			termui.StopLoop()
+		}
+	}()
+
 	termui.Loop()
 }

main.go

@@ -11,7 +11,7 @@ import (
 )
 
 const applicationName = "gargantua"
-const applicationVersion = "v0.1.0-alpha"
+const applicationVersion = "v0.2.0-alpha"
 
 var (
 	app = kingpin.New(applicationName, fmt.Sprintf(`「 %s 」%s crawls all URLs of your website - starting with the links in your sitemap.xml
@@ -62,6 +62,7 @@ func handleCommandlineArgument(arguments []string) {
 
 func startCrawling(targetURL url.URL, concurrentRequests, timeoutInSeconds int, debugModeIsEnabled bool) error {
 	stopTheCrawler := make(chan bool)
+	stopTheUI := make(chan bool)
 	crawlResult := make(chan error)
 
 	go func() {
@@ -70,6 +71,7 @@ func startCrawling(targetURL url.URL, concurrentRequests, timeoutInSeconds int,
 			Timeout:                    time.Second * time.Duration(timeoutInSeconds),
 		}, stopTheCrawler)
 
+		stopTheUI <- true
 		crawlResult <- result
 	}()
 
@@ -81,7 +83,7 @@ func startCrawling(targetURL url.URL, concurrentRequests, timeoutInSeconds int,
 
 		uiWaitGroup.Add(1)
 		go func() {
-			dashboard(time.Now(), stopTheCrawler)
+			dashboard(stopTheUI, stopTheCrawler)
 			uiWaitGroup.Done()
 		}()
 	}

util.go

@@ -10,17 +10,13 @@ const (
 	KILOBYTE = 1024 * BYTE
 	MEGABYTE = 1024 * KILOBYTE
 	GIGABYTE = 1024 * MEGABYTE
-	TERABYTE = 1024 * GIGABYTE
 )
 
 func formatBytes(numberOfBytes int) string {
 	unit := ""
 	value := float32(numberOfBytes)
 
 	switch {
-	case numberOfBytes >= TERABYTE:
-		unit = "T"
-		value = value / TERABYTE
 	case numberOfBytes >= GIGABYTE:
 		unit = "G"
 		value = value / GIGABYTE