122 lines
3.0 KiB
Go
122 lines
3.0 KiB
Go
package royalroad
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"net/url"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// Story represents the metadata extracted from a RoyalRoad fiction page.
|
|
type Story struct {
|
|
Name string
|
|
DateModified time.Time
|
|
}
|
|
|
|
// jsonLD represents the structured data found in RoyalRoad fiction pages.
|
|
type jsonLD struct {
|
|
Name string `json:"name"`
|
|
DateModified string `json:"dateModified"`
|
|
}
|
|
|
|
// Scraper handles fetching and parsing RoyalRoad fiction pages.
|
|
type Scraper struct {
|
|
client *http.Client
|
|
}
|
|
|
|
// NewScraper creates a new Scraper with a configured HTTP client.
|
|
func NewScraper() *Scraper {
|
|
return &Scraper{
|
|
client: &http.Client{
|
|
Timeout: 30 * time.Second,
|
|
},
|
|
}
|
|
}
|
|
|
|
// FetchStory extracts the story name and modification date from a RoyalRoad fiction page.
|
|
func (s *Scraper) FetchStory(ctx context.Context, rawURL string) (Story, error) {
|
|
parsedURL, err := url.Parse(rawURL)
|
|
if err != nil {
|
|
return Story{}, fmt.Errorf("invalid URL %q: %w", rawURL, err)
|
|
}
|
|
|
|
host := strings.ToLower(parsedURL.Hostname())
|
|
if host != "royalroad.com" && host != "www.royalroad.com" {
|
|
return Story{}, fmt.Errorf("URL host %q is not royalroad.com", host)
|
|
}
|
|
|
|
if strings.Contains(parsedURL.Path, "/chapter/") {
|
|
return Story{}, fmt.Errorf("URL %q points to a chapter, not a main fiction page", rawURL)
|
|
}
|
|
|
|
req, err := http.NewRequestWithContext(ctx, "GET", rawURL, nil)
|
|
if err != nil {
|
|
return Story{}, fmt.Errorf("failed to create request: %w", err)
|
|
}
|
|
req.Header.Set("User-Agent", "royalroadupdates/1.0")
|
|
|
|
resp, err := s.client.Do(req)
|
|
if err != nil {
|
|
return Story{}, fmt.Errorf("failed to fetch page: %w", err)
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
if resp.StatusCode != http.StatusOK {
|
|
return Story{}, fmt.Errorf("unexpected response status: %s", resp.Status)
|
|
}
|
|
|
|
body, err := io.ReadAll(resp.Body)
|
|
if err != nil {
|
|
return Story{}, fmt.Errorf("failed to read body: %w", err)
|
|
}
|
|
|
|
story, err := s.parseJSONLD(body)
|
|
if err != nil {
|
|
return Story{}, fmt.Errorf("failed to parse JSON-LD: %w", err)
|
|
}
|
|
|
|
return story, nil
|
|
}
|
|
|
|
func (s *Scraper) parseJSONLD(body []byte) (Story, error) {
|
|
startTag := `<script type="application/ld+json">`
|
|
endTag := `</script>`
|
|
bodyStr := string(body)
|
|
|
|
startIndex := strings.Index(bodyStr, startTag)
|
|
if startIndex == -1 {
|
|
return Story{}, fmt.Errorf("JSON-LD script block not found")
|
|
}
|
|
startIndex += len(startTag)
|
|
|
|
endIndex := strings.Index(bodyStr[startIndex:], endTag)
|
|
if endIndex == -1 {
|
|
return Story{}, fmt.Errorf("JSON-LD closing tag not found")
|
|
}
|
|
endIndex += startIndex
|
|
|
|
jsonContent := body[startIndex:endIndex]
|
|
var ld jsonLD
|
|
if err := json.Unmarshal(jsonContent, &ld); err != nil {
|
|
return Story{}, fmt.Errorf("failed to decode JSON: %w", err)
|
|
}
|
|
|
|
if ld.Name == "" {
|
|
return Story{}, fmt.Errorf("story name missing in JSON-LD")
|
|
}
|
|
|
|
date, err := time.Parse(time.RFC3339, ld.DateModified)
|
|
if err != nil {
|
|
return Story{}, fmt.Errorf("invalid date format %q: %w", ld.DateModified, err)
|
|
}
|
|
|
|
return Story{
|
|
Name: ld.Name,
|
|
DateModified: date,
|
|
}, nil
|
|
}
|