diff --git a/.env.example b/.env.example index cd4399a..7200d52 100644 --- a/.env.example +++ b/.env.example @@ -1,4 +1 @@ -PB_URL=https://www.alphabreed.com/api -PB_USER=your-username -PB_PASSWORD=your-password -PB_CATEGORY=ztvvxqod5s899fk +PB_URL=https://www.alphabreed.com/api/collections/storyurls/records diff --git a/.gitignore b/.gitignore index eb88fee..c8a5e3d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ .env -royalroadupdates +/royalroadupdates .idea/ .pi-subagents diff --git a/AGENTS.md b/AGENTS.md index ee663a0..1972928 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -14,9 +14,8 @@ PocketBase bookmarks collection and reports their last-update times. │ asserts JSON shape and entry count ├── internal/ │ ├── pocketbase/ -│ │ └── client.go - Authenticates and queries the -│ │ /api/collections/bookmarks/records endpoint. -│ │ Accepts filter, perPage, and field params. +│ │ └── client.go - Queries the public PocketBase collection endpoint. +│ │ Accepts perPage and field params. │ │ Returns a list of structs with URL fields. │ ├── royalroad/ │ │ └── scraper.go - Given a URL, fetches the page, confirms it @@ -33,13 +32,13 @@ PocketBase bookmarks collection and reports their last-update times. │ │ └── relative.go - Computes "X minutes/hours/days/months/years │ │ ago" strings from time.Time. │ └── app.go - Orchestrates the full pipeline: -│ 1. Load credentials (compile-time vars). +│ 1. Load endpoint URL from compile-time var. │ 2. Query PocketBase for URLs. │ 3. Iterate URLs sequentially. │ 4. Collect valid RoyalRoad results. │ 5. Sort by dateModified descending. │ 6. Dispatch to appropriate formatter. -├── .env - Real credentials (gitignored). +├── .env - Real endpoint URL (gitignored). ├── .env.example - Template showing required env vars. ├── .gitignore - Must include .env and the built binary. ├── Makefile - All build, test, and check targets. @@ -71,8 +70,8 @@ PocketBase bookmarks collection and reports their last-update times. - **No local Go installation.** Use `make` targets which wrap `docker run --rm -v $(pwd):/src -w /src golang:1.22`. -- **Credential injection:** The Makefile reads `.env` and passes values as - `-ldflags '-X main.PB_URL=... -X main.PB_USER=... -X main.PB_PASSWORD=...'`. +- **Endpoint injection:** The Makefile reads `.env` and passes the value as + `-ldflags '-X main.PBURL=...'`. - **Output:** The compiled binary is written to `./royalroadupdates` in the repository root. - **Targets:** diff --git a/Makefile b/Makefile index 9c98adb..3103494 100644 --- a/Makefile +++ b/Makefile @@ -9,13 +9,13 @@ build: $(DOCKER_RUN) sh -c '\ set -e; \ if [ -f .env ]; then . ./.env; fi; \ - if [ -z "$$PB_URL" ] || [ -z "$$PB_USER" ] || [ -z "$$PB_PASSWORD" ] || [ -z "$$PB_CATEGORY" ]; then \ - echo "error: .env is missing or one of PB_URL / PB_USER / PB_PASSWORD / PB_CATEGORY is not set"; \ + if [ -z "$$PB_URL" ]; then \ + echo "error: .env is missing or PB_URL is not set"; \ exit 1; \ fi; \ go build \ -buildvcs=false \ - -ldflags "-X main.PBURL=$${PB_URL} -X main.PBUser=$${PB_USER} -X main.PBPassword=$${PB_PASSWORD} -X main.PBCategory=$${PB_CATEGORY}" \ + -ldflags "-X main.PBURL=$${PB_URL}" \ -o royalroadupdates \ ./cmd/royalroadupdates' @@ -23,7 +23,7 @@ check: $(DOCKER_RUN) sh -c '\ set -e; \ if [ -f .env ]; then . ./.env; fi; \ - export PB_URL PB_USER PB_PASSWORD PB_CATEGORY; \ + export PB_URL; \ d=$$(gofmt -l .); \ if [ -n "$$d" ]; then echo "format errors:"; echo "$$d"; exit 1; fi; \ go vet -buildvcs=false ./...; \ diff --git a/PLAN.md b/PLAN.md index 59aafab..98db3e2 100644 --- a/PLAN.md +++ b/PLAN.md @@ -8,8 +8,8 @@ story names and how recently each was updated. ## Behavior -1. Authenticate to PocketBase using credentials baked in at compile time. -2. Query the `bookmarks` collection, filtering `category = "Geschichten"`. +1. Read the collection endpoint URL baked in at compile time. +2. Query the `storyurls` collection. All rows are valid; no filtering needed. 3. Extract the `url` field from each matching record. 4. For each URL, sequentially fetch the RoyalRoad fiction page. 5. Extract the fiction name and `dateModified` from the embedded JSON-LD @@ -34,7 +34,7 @@ story names and how recently each was updated. | Scenario | Behaviour | |----------|-----------| -| PocketBase unreachable or auth fails | Exit code ≠ 0, error message on stderr | +| PocketBase unreachable or fetch fails | Exit code ≠ 0, error message on stderr | | Zero valid RoyalRoad stories found after filtering | Exit code 0, empty table | | Individual RoyalRoad page fails to fetch or parse | Silent skip (no output, no error) | @@ -56,16 +56,13 @@ the standard library. ## Compile-Time Configuration -The three PocketBase connection values are compiled into the binary via `-ldflags`: +The collection endpoint URL is compiled into the binary via `-ldflags`: ``` -PB_URL=https://pocketbase.example.com -PB_USER=username -PB_PASSWORD=password +PBURL=https://example.com/api/collections/storyurls/records ``` -These are read from a `.env` file at build time. The resulting binary runs -standalone without any environment variables present. +This is read from the `.env` file at build time. The `.env` key is `PB_URL`; the Makefile passes its value via `-ldflags "-X main.PBURL=${PB_URL}"`. ## Output Formats @@ -98,8 +95,7 @@ Dates are always sortable ISO 8601 / RFC3339 strings in JSON output. ## Data Flow ``` -PocketBase (bookmarks collection) - └── Filter: category = "Geschichten" +PocketBase (storyurls collection) └── Extract: url field └── For each URL: ├── Skip if not royalroad.com diff --git a/cmd/royalroadupdates/e2e_test.go b/cmd/royalroadupdates/e2e_test.go new file mode 100644 index 0000000..18474f5 --- /dev/null +++ b/cmd/royalroadupdates/e2e_test.go @@ -0,0 +1,71 @@ +package main + +import ( + "bytes" + "encoding/json" + "os" + "os/exec" + "testing" + "time" +) + +// e2eEntry matches the JSON output shape. +type e2eEntry struct { + Name string `json:"name"` + Date string `json:"date"` +} + +func TestE2E(t *testing.T) { + pbURL := os.Getenv("PB_URL") + + if pbURL == "" { + t.Skip("PB_URL not set; skipping e2e test against live services") + } + + // Compile the binary with PB_URL. + ldflags := "-X main.PBURL=" + pbURL + + buildCmd := exec.Command("go", "build", "-ldflags", ldflags, "-o", "royalroadupdates", ".") + buildCmd.Dir = "." + if out, err := buildCmd.CombinedOutput(); err != nil { + t.Fatalf("build failed: %v\n%s", err, out) + } + defer os.Remove("royalroadupdates") + + // Run with --json. + runCmd := exec.Command("./royalroadupdates", "--json") + var stderr bytes.Buffer + runCmd.Stderr = &stderr + out, err := runCmd.Output() + if err != nil { + t.Fatalf("execution failed: %v\nstdout:\n%s\nstderr:\n%s", err, out, stderr.String()) + } + if stderr.Len() > 0 { + t.Fatalf("unexpected stderr output: %s", stderr.String()) + } + + var entries []e2eEntry + if err := json.Unmarshal(out, &entries); err != nil { + t.Fatalf("JSON parse failed: %v\noutput:\n%s", err, out) + } + + if len(entries) == 0 { + t.Fatalf("expected a non-empty JSON array, got empty") + } + + if len(entries) < 3 { + t.Fatalf("expected at least 3 entries, got %d", len(entries)) + } + + for i, e := range entries { + if e.Name == "" { + t.Errorf("entry %d: name is empty", i) + } + if e.Date == "" { + t.Fatalf("entry %d: date is empty", i) + } + if _, err := time.Parse(time.RFC3339, e.Date); err != nil { + t.Fatalf("entry %d: date %q not parseable: %v", i, e.Date, err) + } + } +} diff --git a/cmd/royalroadupdates/main.go b/cmd/royalroadupdates/main.go new file mode 100644 index 0000000..e55ca0b --- /dev/null +++ b/cmd/royalroadupdates/main.go @@ -0,0 +1,36 @@ +package main + +import ( + "context" + "flag" + "fmt" + "os" + "time" + + "royalroadupdates/internal" +) + +// This is set at compile time via -ldflags. +var PBURL = "" + +func main() { + jsonOutput := flag.Bool("json", false, "output JSON instead of table") + flag.Parse() + + if PBURL == "" { + fmt.Fprintln(os.Stderr, "error: PB_URL not configured") + os.Exit(1) + } + + cfg := internal.Config{PBURL: PBURL} + + app := internal.New(cfg) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + if err := app.Run(ctx, *jsonOutput); err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + os.Exit(1) + } +} diff --git a/internal/app.go b/internal/app.go index 0b2dbba..debd429 100644 --- a/internal/app.go +++ b/internal/app.go @@ -14,10 +14,7 @@ import ( // Config holds the compile-time settings for the application. type Config struct { - PBURL string - PBUser string - PBPassword string - PBCategory string + PBURL string } // App orchestrates the full pipeline. @@ -31,7 +28,7 @@ type App struct { // New creates a new App with the given configuration. func New(cfg Config) *App { return &App{ - pocketbase: pocketbase.New(cfg.PBURL, cfg.PBUser, cfg.PBPassword, cfg.PBCategory), + pocketbase: pocketbase.New(cfg.PBURL), scraper: royalroad.NewScraper(), outW: os.Stdout, errW: os.Stderr, diff --git a/internal/pocketbase/client.go b/internal/pocketbase/client.go index 4b9c362..9a10d5f 100644 --- a/internal/pocketbase/client.go +++ b/internal/pocketbase/client.go @@ -1,50 +1,37 @@ package pocketbase import ( - "bytes" "context" "encoding/json" "fmt" "net/http" "net/url" - "strings" "time" ) // Client handles interactions with the PocketBase API. type Client struct { - baseURL string - user string - password string - category string + endpoint *url.URL http *http.Client } // New creates a new PocketBase client. -func New(baseURL, user, password, category string) *Client { - // Normalise: ensure baseURL does not already end with /api. - baseURL = strings.TrimSuffix(baseURL, "/api") - baseURL = strings.TrimSuffix(baseURL, "/") +func New(endpoint string) *Client { + u, err := url.Parse(endpoint) + if err != nil || u == nil { + return &Client{ + endpoint: &url.URL{}, + http: &http.Client{Timeout: 30 * time.Second}, + } + } return &Client{ - baseURL: baseURL, - user: user, - password: password, - category: category, + endpoint: u, http: &http.Client{ Timeout: 30 * time.Second, }, } } -type authRequest struct { - Identity string `json:"identity"` - Password string `json:"password"` -} - -type authResponse struct { - Token string `json:"token"` -} - type record struct { URL string `json:"url"` } @@ -53,64 +40,11 @@ type recordsResponse struct { Items []record `json:"items"` } -// GetBookmarkURLs authenticates and retrieves URLs from the bookmarks collection. +// GetBookmarkURLs retrieves URLs from the collection. func (c *Client) GetBookmarkURLs(ctx context.Context) ([]string, error) { - token, err := c.authenticate(ctx) - if err != nil { - return nil, fmt.Errorf("pocketbase auth: %w", err) - } - - urls, err := c.fetchURLs(ctx, token) - if err != nil { - return nil, fmt.Errorf("pocketbase fetch URLs: %w", err) - } - - return urls, nil -} - -func (c *Client) authenticate(ctx context.Context) (string, error) { - reqBody, err := json.Marshal(authRequest{ - Identity: c.user, - Password: c.password, - }) - if err != nil { - return "", err - } - - authURL := c.baseURL + "/api/collections/users/auth-with-password" - req, err := http.NewRequestWithContext(ctx, "POST", authURL, bytes.NewBuffer(reqBody)) - if err != nil { - return "", err - } - req.Header.Set("Content-Type", "application/json") - - resp, err := c.http.Do(req) - if err != nil { - return "", err - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return "", fmt.Errorf("unexpected status: %d", resp.StatusCode) - } - - var authResp authResponse - if err := json.NewDecoder(resp.Body).Decode(&authResp); err != nil { - return "", err - } - - return authResp.Token, nil -} - -func (c *Client) fetchURLs(ctx context.Context, token string) ([]string, error) { - // Filter: category=, perPage=200, fields=url - filter := fmt.Sprintf(`category="%s"`, c.category) - u, err := url.Parse(c.baseURL + "/api/collections/bookmarks/records") - if err != nil { - return nil, fmt.Errorf("invalid base URL: %w", err) - } + // Clone the endpoint URL to avoid mutation on repeated calls. + u := *c.endpoint q := u.Query() - q.Set("filter", filter) q.Set("perPage", "200") q.Set("fields", "url") u.RawQuery = q.Encode() @@ -119,7 +53,6 @@ func (c *Client) fetchURLs(ctx context.Context, token string) ([]string, error) if err != nil { return nil, fmt.Errorf("creating request: %w", err) } - req.Header.Set("Authorization", token) resp, err := c.http.Do(req) if err != nil { diff --git a/internal/pocketbase/client_test.go b/internal/pocketbase/client_test.go index 07c26c5..ca1b861 100644 --- a/internal/pocketbase/client_test.go +++ b/internal/pocketbase/client_test.go @@ -10,29 +10,28 @@ import ( func TestGetBookmarkURLs_Success(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path == "/api/collections/users/auth-with-password" { - w.WriteHeader(http.StatusOK) - json.NewEncoder(w).Encode(authResponse{Token: "test-token"}) - return + // No auth needed - just return records. + if r.URL.Path != "/" { + t.Errorf("Expected path '/', got '%s'", r.URL.Path) } - if r.URL.Path == "/api/collections/bookmarks/records" { - if r.Header.Get("Authorization") != "test-token" { - t.Errorf("Expected Authorization header 'test-token', got '%s'", r.Header.Get("Authorization")) - } - w.WriteHeader(http.StatusOK) - json.NewEncoder(w).Encode(recordsResponse{ - Items: []record{ - {URL: "https://royalroad.com/fiction/1"}, - {URL: "https://royalroad.com/fiction/2"}, - }, - }) - return + // Verify query params. + if r.URL.Query().Get("perPage") != "200" { + t.Errorf("Expected perPage=200, got '%s'", r.URL.Query().Get("perPage")) } - t.Errorf("Unexpected request to %s", r.URL.Path) + if r.URL.Query().Get("fields") != "url" { + t.Errorf("Expected fields=url, got '%s'", r.URL.Query().Get("fields")) + } + w.WriteHeader(http.StatusOK) + json.NewEncoder(w).Encode(recordsResponse{ + Items: []record{ + {URL: "https://royalroad.com/fiction/1"}, + {URL: "https://royalroad.com/fiction/2"}, + }, + }) })) defer server.Close() - client := New(server.URL, "user", "pass", "Geschichten") + client := New(server.URL) urls, err := client.GetBookmarkURLs(context.Background()) if err != nil { @@ -46,20 +45,16 @@ func TestGetBookmarkURLs_Success(t *testing.T) { } } -func TestGetBookmarkURLs_AuthFailure(t *testing.T) { +func TestGetBookmarkURLs_Non200(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path == "/api/collections/users/auth-with-password" { - w.WriteHeader(http.StatusUnauthorized) - return - } - t.Errorf("Unexpected request to %s", r.URL.Path) + w.WriteHeader(http.StatusInternalServerError) })) defer server.Close() - client := New(server.URL, "user", "pass", "Geschichten") + client := New(server.URL) _, err := client.GetBookmarkURLs(context.Background()) if err == nil { - t.Error("Expected error on auth failure, got nil") + t.Error("Expected error on non-200 status, got nil") } }