From 5825cb6198a5f923d3ffd7de6231f7e6eb8e20ff Mon Sep 17 00:00:00 2001 From: Florian Loch Date: Tue, 5 Mar 2024 23:24:15 +0100 Subject: [PATCH] fix: document that we use CRLF as line ending, same as upstream API does, and update tests accordingly --- README.md | 9 +++++++++ cmd/sync/main.go | 2 +- export.go | 4 +++- export_test.go | 4 ++-- lib_test.go | 4 ++-- sync_test.go | 46 +++++++++++++++++++++++----------------------- 6 files changed, 40 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index 52f656f..b73d7ea 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,15 @@ HIBP#.Query("ABCDE") (io.ReadClose, error) // Returns the k-proximity API result All of them operate on disk but, depending on the medium, should provide access times that are probably good enough for all scenarios. A memory-based `tmpfs` will speed things up when necessary. +**Attention:** +The [official API](https://haveibeenpwned.com/API/v3#PwnedPasswords) states the following regarding the format: + +> Each password is stored as both a SHA-1 and an NTLM hash of a UTF-8 encoded password. +> The downloadable source data delimits the hash and the password count with a colon (:) and each line with a CRLF. + +The crucial part being that lines are ended with `\r\n`. +In order to be compatible with the upstream API this library sticks to this... + ## CLI diff --git a/cmd/sync/main.go b/cmd/sync/main.go index ae88ce0..9ae4bea 100644 --- a/cmd/sync/main.go +++ b/cmd/sync/main.go @@ -69,7 +69,7 @@ func run(dataDir string) error { return nil } - h := hibp.New(hibp.WithDataDir(dataDir), hibp.WithNoCompression()) + h := hibp.New(hibp.WithDataDir(dataDir)) if err := h.Sync( hibp.SyncWithProgressFn(updateProgressBar), diff --git a/export.go b/export.go index 1b65fad..12f35d2 100644 --- a/export.go +++ b/export.go @@ -7,7 +7,9 @@ import ( "io" ) -var lineSeparator = []byte("\n") +// The upstream Have-I-Been-Pwned API uses CRLF as line separator - so we are stuck with it, +// although it does not feel right. +var lineSeparator = []byte("\r\n") func export(from, to int64, store storage, w io.Writer) error { for i := from; i < to; i++ { diff --git a/export_test.go b/export_test.go index 037d02c..7871ad5 100644 --- a/export_test.go +++ b/export_test.go @@ -12,7 +12,7 @@ func TestExport(t *testing.T) { ctrl := gomock.NewController(t) storageMock := NewMockstorage(ctrl) - storageMock.EXPECT().LoadData("00000").Return(io.NopCloser(bytes.NewReader([]byte("suffix:counter11\nsuffix:counter12"))), nil) + storageMock.EXPECT().LoadData("00000").Return(io.NopCloser(bytes.NewReader([]byte("suffix:counter11\r\nsuffix:counter12"))), nil) storageMock.EXPECT().LoadData("00001").Return(io.NopCloser(bytes.NewReader([]byte("suffix:counter2"))), nil) storageMock.EXPECT().LoadData("00002").Return(io.NopCloser(bytes.NewReader([]byte("suffix:counter3"))), nil) @@ -26,7 +26,7 @@ func TestExport(t *testing.T) { // HIBP API looks like. // This has to be the case because `Export` iterates over all ranges; different from `Query` which only // queries a single range. - if buf.String() != "00000suffix:counter11\n00000suffix:counter12\n00001suffix:counter2\n00002suffix:counter3" { + if buf.String() != "00000suffix:counter11\r\n00000suffix:counter12\r\n00001suffix:counter2\r\n00002suffix:counter3" { t.Fatalf("unexpected output: %q", buf.String()) } } diff --git a/lib_test.go b/lib_test.go index 95311da..4189a04 100644 --- a/lib_test.go +++ b/lib_test.go @@ -12,7 +12,7 @@ func TestQuery(t *testing.T) { ctrl := gomock.NewController(t) storageMock := NewMockstorage(ctrl) - storageMock.EXPECT().LoadData("00000").Return(io.NopCloser(bytes.NewReader([]byte("suffix:counter11\nsuffix:counter12"))), nil) + storageMock.EXPECT().LoadData("00000").Return(io.NopCloser(bytes.NewReader([]byte("suffix:counter11\r\nsuffix:counter12"))), nil) i := HIBP{store: storageMock} @@ -29,7 +29,7 @@ func TestQuery(t *testing.T) { // We expect the lines to not be prefixed with the range as this is what the response from the official // HIBP API looks like. - if string(lines) != "suffix:counter11\nsuffix:counter12" { + if string(lines) != "suffix:counter11\r\nsuffix:counter12" { t.Fatalf("unexpected output: %q", string(lines)) } } diff --git a/sync_test.go b/sync_test.go index 6347464..dddae7a 100644 --- a/sync_test.go +++ b/sync_test.go @@ -23,63 +23,63 @@ func TestSync(t *testing.T) { Get("/range/00000"). Reply(200). AddHeader("ETag", "etag"). - BodyString("suffix1") + BodyString("suffix1:1") gock.New(baseURL). Get("/range/00001"). MatchHeader("If-None-Match", "etag received earlier"). Reply(http.StatusNotModified). AddHeader("ETag", "etag received earlier"). - BodyString("suffix2") + BodyString("suffix2:2") gock.New(baseURL). Get("/range/00002"). Reply(200). AddHeader("ETag", "etag"). - BodyString("suffix31:2\nsuffix32:3") + BodyString("suffix31:2\r\nsuffix32:3") gock.New(baseURL). Get("/range/00003"). Reply(200). AddHeader("ETag", "etag"). - BodyString("suffix4") + BodyString("suffix4:4") gock.New(baseURL). Get("/range/00004"). Reply(200). AddHeader("ETag", "etag"). - BodyString("suffix5") + BodyString("suffix5:5") gock.New(baseURL). Get("/range/00005"). Reply(200). AddHeader("ETag", "etag"). - BodyString("suffix6") + BodyString("suffix6:6") gock.New(baseURL). Get("/range/00006"). Reply(200). AddHeader("ETag", "etag"). - BodyString("suffix7") + BodyString("suffix7:7") gock.New(baseURL). Get("/range/00007"). Reply(200). AddHeader("ETag", "etag"). - BodyString("suffix8") + BodyString("suffix8:8") gock.New(baseURL). Get("/range/00008"). Reply(200). AddHeader("ETag", "etag"). - BodyString("suffix9") + BodyString("suffix9:9") gock.New(baseURL). Get("/range/00009"). Reply(200). AddHeader("ETag", "etag"). - BodyString("suffix10") + BodyString("suffix10:10") gock.New(baseURL). Get("/range/0000A"). Reply(200). AddHeader("ETag", "etag"). - BodyString("suffix11") + BodyString("suffix11:11") gock.New(baseURL). Get("/range/0000B"). Reply(200). AddHeader("ETag", "etag"). - BodyString("suffix12") + BodyString("suffix12:12") client := &hibpClient{ endpoint: defaultEndpoint, @@ -90,29 +90,29 @@ func TestSync(t *testing.T) { storageMock := NewMockstorage(ctrl) storageMock.EXPECT().LoadETag("00000").Return("", nil) - storageMock.EXPECT().Save("00000", "etag", []byte("suffix1")).Return(nil) + storageMock.EXPECT().Save("00000", "etag", []byte("suffix1:1")).Return(nil) storageMock.EXPECT().LoadETag("00001").Return("etag received earlier", nil) // 00001 does not need to be written as its ETag has not changed storageMock.EXPECT().LoadETag("00002").Return("", nil) - storageMock.EXPECT().Save("00002", "etag", []byte("suffix31:2\n00suffix32:3")).Return(nil) + storageMock.EXPECT().Save("00002", "etag", []byte("suffix31:2\r\nsuffix32:3")).Return(nil) storageMock.EXPECT().LoadETag("00003").Return("", nil) - storageMock.EXPECT().Save("00003", "etag", []byte("suffix4")).Return(nil) + storageMock.EXPECT().Save("00003", "etag", []byte("suffix4:4")).Return(nil) storageMock.EXPECT().LoadETag("00004").Return("", nil) - storageMock.EXPECT().Save("00004", "etag", []byte("suffix5")).Return(nil) + storageMock.EXPECT().Save("00004", "etag", []byte("suffix5:5")).Return(nil) storageMock.EXPECT().LoadETag("00005").Return("", nil) - storageMock.EXPECT().Save("00005", "etag", []byte("suffix6")).Return(nil) + storageMock.EXPECT().Save("00005", "etag", []byte("suffix6:6")).Return(nil) storageMock.EXPECT().LoadETag("00006").Return("", nil) - storageMock.EXPECT().Save("00006", "etag", []byte("suffix7")).Return(nil) + storageMock.EXPECT().Save("00006", "etag", []byte("suffix7:7")).Return(nil) storageMock.EXPECT().LoadETag("00007").Return("", nil) - storageMock.EXPECT().Save("00007", "etag", []byte("suffix8")).Return(nil) + storageMock.EXPECT().Save("00007", "etag", []byte("suffix8:8")).Return(nil) storageMock.EXPECT().LoadETag("00008").Return("", nil) - storageMock.EXPECT().Save("00008", "etag", []byte("suffix9")).Return(nil) + storageMock.EXPECT().Save("00008", "etag", []byte("suffix9:9")).Return(nil) storageMock.EXPECT().LoadETag("00009").Return("", nil) - storageMock.EXPECT().Save("00009", "etag", []byte("suffix10")).Return(nil) + storageMock.EXPECT().Save("00009", "etag", []byte("suffix10:10")).Return(nil) storageMock.EXPECT().LoadETag("0000A").Return("", nil) - storageMock.EXPECT().Save("0000A", "etag", []byte("suffix11")).Return(nil) + storageMock.EXPECT().Save("0000A", "etag", []byte("suffix11:11")).Return(nil) storageMock.EXPECT().LoadETag("0000B").Return("", nil) - storageMock.EXPECT().Save("0000B", "etag", []byte("suffix12")).Return(nil) + storageMock.EXPECT().Save("0000B", "etag", []byte("suffix12:12")).Return(nil) var callCounter atomic.Int64