package royalroad import ( "context" "encoding/json" "fmt" "io" "net/http" "net/url" "strings" "time" ) // Story represents the metadata extracted from a RoyalRoad fiction page. type Story struct { Name string DateModified time.Time } // jsonLD represents the structured data found in RoyalRoad fiction pages. type jsonLD struct { Name string `json:"name"` DateModified string `json:"dateModified"` } // Scraper handles fetching and parsing RoyalRoad fiction pages. type Scraper struct { client *http.Client } // NewScraper creates a new Scraper with a configured HTTP client. func NewScraper() *Scraper { return &Scraper{ client: &http.Client{ Timeout: 30 * time.Second, }, } } // FetchStory extracts the story name and modification date from a RoyalRoad fiction page. func (s *Scraper) FetchStory(ctx context.Context, rawURL string) (Story, error) { parsedURL, err := url.Parse(rawURL) if err != nil { return Story{}, fmt.Errorf("invalid URL %q: %w", rawURL, err) } host := strings.ToLower(parsedURL.Hostname()) if host != "royalroad.com" && host != "www.royalroad.com" { return Story{}, fmt.Errorf("URL host %q is not royalroad.com", host) } if strings.Contains(parsedURL.Path, "/chapter/") { return Story{}, fmt.Errorf("URL %q points to a chapter, not a main fiction page", rawURL) } req, err := http.NewRequestWithContext(ctx, "GET", rawURL, nil) if err != nil { return Story{}, fmt.Errorf("failed to create request: %w", err) } req.Header.Set("User-Agent", "royalroadupdates/1.0") resp, err := s.client.Do(req) if err != nil { return Story{}, fmt.Errorf("failed to fetch page: %w", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return Story{}, fmt.Errorf("unexpected response status: %s", resp.Status) } body, err := io.ReadAll(resp.Body) if err != nil { return Story{}, fmt.Errorf("failed to read body: %w", err) } story, err := s.parseJSONLD(body) if err != nil { return Story{}, fmt.Errorf("failed to parse JSON-LD: %w", err) } return story, nil } func (s *Scraper) parseJSONLD(body []byte) (Story, error) { startTag := `` bodyStr := string(body) startIndex := strings.Index(bodyStr, startTag) if startIndex == -1 { return Story{}, fmt.Errorf("JSON-LD script block not found") } startIndex += len(startTag) endIndex := strings.Index(bodyStr[startIndex:], endTag) if endIndex == -1 { return Story{}, fmt.Errorf("JSON-LD closing tag not found") } endIndex += startIndex jsonContent := body[startIndex:endIndex] var ld jsonLD if err := json.Unmarshal(jsonContent, &ld); err != nil { return Story{}, fmt.Errorf("failed to decode JSON: %w", err) } if ld.Name == "" { return Story{}, fmt.Errorf("story name missing in JSON-LD") } date, err := time.Parse(time.RFC3339, ld.DateModified) if err != nil { return Story{}, fmt.Errorf("invalid date format %q: %w", ld.DateModified, err) } return Story{ Name: ld.Name, DateModified: date, }, nil }