diff options
| -rw-r--r-- | flat.go | 19 | ||||
| -rw-r--r-- | flat_test.go | 31 | ||||
| -rw-r--r-- | flatbot.go | 79 | ||||
| -rw-r--r-- | go.mod | 2 | ||||
| -rw-r--r-- | parse.go | 84 | ||||
| -rw-r--r-- | parse_test.go (renamed from flatbot_test.go) | 10 | ||||
| -rw-r--r-- | sent.go | 52 | ||||
| -rw-r--r-- | sent_test.go | 106 |
8 files changed, 304 insertions, 79 deletions
@@ -0,0 +1,19 @@ +package main + +import ( + "cmp" + "fmt" +) + +type flat struct { + ID int + Price string +} + +func (f *flat) URL() string { + return fmt.Sprintf("https://rightmove.co.uk/properties/%v", f.ID) +} + +func compareID(a, b flat) int { + return cmp.Compare(a.ID, b.ID) +} diff --git a/flat_test.go b/flat_test.go new file mode 100644 index 0000000..2ac3ddb --- /dev/null +++ b/flat_test.go @@ -0,0 +1,31 @@ +package main + +import ( + "testing" +) + +func TestURL(t *testing.T) { + f := flat{ID: 0, Price: "£1,000"} + got := f.URL() + want := "https://rightmove.co.uk/properties/0" + if got != want { + t.Errorf("URL call failed: got: %v, want: %v", got, want) + } +} + +func TestCompareID(t *testing.T) { + a := flat{ID: 0, Price: "£3,000"} + b := flat{ID: 1, Price: "£1,000"} + got := compareID(a, b) + if got != -1 { + t.Errorf("Wrong compare result: got: %v, want: %v", got, -1) + } + got = compareID(b, a) + if got != 1 { + t.Errorf("Wrong compare result: got: %v, want: %v", got, 1) + } + got = compareID(a, a) + if got != 0 { + t.Errorf("Wrong compare result: got: %v, want: %v", got, 0) + } +} @@ -1,15 +1,10 @@ package main import ( - "bytes" - "errors" "fmt" "io" "log" "net/http" - "strings" - - "golang.org/x/net/html" ) func main() { @@ -22,7 +17,13 @@ func main() { if err != nil { log.Fatal(err) } + sent, err := readSent("sent/sent.json") + if err != nil { + log.Fatal(err) + } + flats = removeAlreadySent(flats, sent) fmt.Println(flats) + writeSent(flats, "/tmp/sent.json") } func fetch(url string) ([]byte, error) { @@ -41,71 +42,3 @@ func fetch(url string) ([]byte, error) { } return body, nil } - -type flat struct { - URL string - Price string -} - -func parse(body []byte) ([]flat, error) { - doc, err := html.Parse(bytes.NewReader(body)) - if err != nil { - return make([]flat, 0), err - } - flats := make([]flat, 0) - for _, n := range findNodes(doc) { - flat, err := parseNode(n) - if err != nil { - continue - } - flats = append(flats, flat) - } - return flats, nil -} - -func findNodes(root *html.Node) []*html.Node { - flats := make([]*html.Node, 0) - for n := range root.Descendants() { - if n.Type != html.ElementNode { - continue - } - if n.Data != "a" { - continue - } - attr := matchAttr(n, "data-testid") - if attr == nil || attr.Val != "property-price" { - continue - } - flats = append(flats, n) - } - return flats -} - -func matchAttr(n *html.Node, key string) *html.Attribute { - for _, attr := range n.Attr { - if attr.Key == key { - return &attr - } - } - return nil -} - -func parseNode(root *html.Node) (flat, error) { - url := matchAttr(root, "href") - if url == nil { - return flat{}, errors.New("Couldn't find URL") - } - f := flat{URL: makeURL(url.Val), Price: ""} - for n := range root.Descendants() { - if price, found := strings.CutSuffix(n.Data, " pcm"); found { - f.Price = price - return f, nil - } - } - return flat{}, errors.New("Couldn't find price") -} - -func makeURL(path string) string { - prettySuffix, _ := strings.CutSuffix(path, "/?channel=RES_LET") - return fmt.Sprintf("https://rightmove.co.uk%v", prettySuffix) -} @@ -2,4 +2,4 @@ module ilvokhin.com/flatbot go 1.24.0 -require golang.org/x/net v0.35.0 // indirect +require golang.org/x/net v0.35.0 diff --git a/parse.go b/parse.go new file mode 100644 index 0000000..6be416e --- /dev/null +++ b/parse.go @@ -0,0 +1,84 @@ +package main + +import ( + "bytes" + "errors" + "fmt" + "strconv" + "strings" + + "golang.org/x/net/html" +) + +func parse(body []byte) ([]flat, error) { + doc, err := html.Parse(bytes.NewReader(body)) + if err != nil { + return make([]flat, 0), err + } + flats := make([]flat, 0) + for _, n := range findNodes(doc) { + flat, err := parseNode(n) + if err != nil { + continue + } + flats = append(flats, flat) + } + return flats, nil +} + +func findNodes(root *html.Node) []*html.Node { + flats := make([]*html.Node, 0) + for n := range root.Descendants() { + if n.Type != html.ElementNode { + continue + } + if n.Data != "a" { + continue + } + attr := matchAttr(n, "data-testid") + if attr == nil || attr.Val != "property-price" { + continue + } + flats = append(flats, n) + } + return flats +} + +func matchAttr(n *html.Node, key string) *html.Attribute { + for _, attr := range n.Attr { + if attr.Key == key { + return &attr + } + } + return nil +} + +func parseNode(root *html.Node) (flat, error) { + url := matchAttr(root, "href") + if url == nil { + return flat{}, errors.New("Couldn't find URL") + } + ID, err := parseID(url.Val) + if err != nil { + return flat{}, err + } + f := flat{ID, ""} + for n := range root.Descendants() { + if price, found := strings.CutSuffix(n.Data, " pcm"); found { + f.Price = price + return f, nil + } + } + return flat{}, errors.New("Couldn't find price") +} + +func parseID(path string) (int, error) { + s, _ := strings.CutPrefix(path, "/properties/") + maybeID, _ := strings.CutSuffix(s, "#/?channel=RES_LET") + ID, err := strconv.Atoi(maybeID) + if err != nil { + err := fmt.Errorf("Couldn't extract ID from %q", path) + return -1, err + } + return ID, err +} diff --git a/flatbot_test.go b/parse_test.go index 0308295..cb04638 100644 --- a/flatbot_test.go +++ b/parse_test.go @@ -14,22 +14,22 @@ func TestParse(t *testing.T) { } want := []flat{ flat{ - URL: "https://rightmove.co.uk/properties/156522206#", + ID: 156522206, Price: "£2,500", }, flat{ - URL: "https://rightmove.co.uk/properties/158462822#", + ID: 158462822, Price: "£3,000", }, flat{ - URL: "https://rightmove.co.uk/properties/157948184#", + ID: 157948184, Price: "£2,400", }} got, err := parse(data) if err != nil { t.Fatal(err) } - if !reflect.DeepEqual(want, got) { - t.Errorf("Parse failed: got: %v, want: %v", want, got) + if !reflect.DeepEqual(got, want) { + t.Errorf("Parse failed: got: %v, want: %v", got, want) } } @@ -0,0 +1,52 @@ +package main + +import ( + "encoding/json" + "errors" + "os" + "slices" +) + +func readSent(filename string) ([]flat, error) { + data, err := os.ReadFile(filename) + sent := make([]flat, 0) + if err != nil { + // This is fine, as we might just started and didn't dump + // anything into sent file yet. + if errors.Is(err, os.ErrNotExist) { + return sent, nil + } + return sent, err + } + err = json.Unmarshal(data, &sent) + if err != nil { + return sent, err + } + if !slices.IsSortedFunc(sent, compareID) { + return nil, errors.New("Invalid sent: not sorted") + } + return sent, nil +} + +func removeAlreadySent(fetched []flat, sent []flat) []flat { + if !slices.IsSortedFunc(sent, compareID) { + panic("Sent expected to be sorted") + } + recent := make([]flat, 0) + for _, f := range fetched { + _, found := slices.BinarySearchFunc(sent, f, compareID) + if found { + continue + } + recent = append(recent, f) + } + return recent +} + +func writeSent(sent []flat, filename string) error { + jsonData, err := json.Marshal(sent) + if err != nil { + return err + } + return os.WriteFile(filename, jsonData, 0644) +} diff --git a/sent_test.go b/sent_test.go new file mode 100644 index 0000000..64b13d6 --- /dev/null +++ b/sent_test.go @@ -0,0 +1,106 @@ +package main + +import ( + "os" + "path/filepath" + "reflect" + "testing" +) + +func TestReadSentNew(t *testing.T) { + tmp := t.TempDir() + filename := filepath.Join(tmp, "does-not-exist.json") + got, err := readSent(filename) + if err != nil { + t.Fatal(err) + } + if len(got) > 0 { + t.Errorf("Expect empty slice, but got: %v", got) + } +} + +func TestReadSent(t *testing.T) { + tmp := t.TempDir() + filename := filepath.Join(tmp, "sent.json") + data := []byte(`[{"ID":156522206,"Price":"£2,500"}]`) + os.WriteFile(filename, data, 0644) + + got, err := readSent(filename) + if err != nil { + t.Fatal(err) + } + want := []flat{ + flat{ID: 156522206, Price: "£2,500"}, + } + if !reflect.DeepEqual(got, want) { + t.Errorf("readSent failed: got: %v, want: %v", got, want) + } +} + +func TestRemoveAlreadySent(t *testing.T) { + flats := []flat{ + flat{ + ID: 156522206, + Price: "£2,500", + }, + flat{ + ID: 158462822, + Price: "£3,000", + }} + sent := []flat{ + flat{ + ID: 156522206, + Price: "£2,500", + }, + } + + got := removeAlreadySent(flats, sent) + want := []flat{flat{ID: 158462822, Price: "£3,000"}} + if !reflect.DeepEqual(got, want) { + t.Errorf("removeAlreadySent failed: got: %v, want: %v", + got, want) + } + +} + +func TestWriteSentNew(t *testing.T) { + tmp := t.TempDir() + filename := filepath.Join(tmp, "sent.json") + flats := []flat{flat{ID: 156522206, Price: "£2,500"}} + + err := writeSent(flats, filename) + if err != nil { + t.Fatal(err) + } + got, err := os.ReadFile(filename) + if err != nil { + t.Fatal(err) + } + want := []byte(`[{"ID":156522206,"Price":"£2,500"}]`) + if !reflect.DeepEqual(got, want) { + t.Errorf("writeSent failed: got: %v, want: %v", got, want) + } +} + +func TestWriteSentOverride(t *testing.T) { + tmp := t.TempDir() + filename := filepath.Join(tmp, "sent.json") + _, err := os.Create(filename) + if err != nil { + t.Fatal(err) + } + flats := []flat{flat{ID: 156522206, Price: "£2,500"}} + err = writeSent(flats, filename) + + if err != nil { + t.Fatal(err) + } + got, err := os.ReadFile(filename) + if err != nil { + t.Fatal(err) + } + want := []byte(`[{"ID":156522206,"Price":"£2,500"}]`) + if !reflect.DeepEqual(got, want) { + t.Errorf("writeSent failed: got: %v, want: %v", got, want) + } +} |