Erstes Commit

2026-07-12 02:49:54 +02:00
commit 6dd456ca4a
15 changed files with 920 additions and 0 deletions
--- a/internal/royalroad/scraper.go
+++ b/internal/royalroad/scraper.go
@@ -0,0 +1,121 @@
+package royalroad
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+)
+
+// Story represents the metadata extracted from a RoyalRoad fiction page.
+type Story struct {
+	Name         string
+	DateModified time.Time
+}
+
+// jsonLD represents the structured data found in RoyalRoad fiction pages.
+type jsonLD struct {
+	Name         string `json:"name"`
+	DateModified string `json:"dateModified"`
+}
+
+// Scraper handles fetching and parsing RoyalRoad fiction pages.
+type Scraper struct {
+	client *http.Client
+}
+
+// NewScraper creates a new Scraper with a configured HTTP client.
+func NewScraper() *Scraper {
+	return &Scraper{
+		client: &http.Client{
+			Timeout: 30 * time.Second,
+		},
+	}
+}
+
+// FetchStory extracts the story name and modification date from a RoyalRoad fiction page.
+func (s *Scraper) FetchStory(ctx context.Context, rawURL string) (Story, error) {
+	parsedURL, err := url.Parse(rawURL)
+	if err != nil {
+		return Story{}, fmt.Errorf("invalid URL %q: %w", rawURL, err)
+	}
+
+	host := strings.ToLower(parsedURL.Hostname())
+	if host != "royalroad.com" && host != "www.royalroad.com" {
+		return Story{}, fmt.Errorf("URL host %q is not royalroad.com", host)
+	}
+
+	if strings.Contains(parsedURL.Path, "/chapter/") {
+		return Story{}, fmt.Errorf("URL %q points to a chapter, not a main fiction page", rawURL)
+	}
+
+	req, err := http.NewRequestWithContext(ctx, "GET", rawURL, nil)
+	if err != nil {
+		return Story{}, fmt.Errorf("failed to create request: %w", err)
+	}
+	req.Header.Set("User-Agent", "royalroadupdates/1.0")
+
+	resp, err := s.client.Do(req)
+	if err != nil {
+		return Story{}, fmt.Errorf("failed to fetch page: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return Story{}, fmt.Errorf("unexpected response status: %s", resp.Status)
+	}
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return Story{}, fmt.Errorf("failed to read body: %w", err)
+	}
+
+	story, err := s.parseJSONLD(body)
+	if err != nil {
+		return Story{}, fmt.Errorf("failed to parse JSON-LD: %w", err)
+	}
+
+	return story, nil
+}
+
+func (s *Scraper) parseJSONLD(body []byte) (Story, error) {
+	startTag := `<script type="application/ld+json">`
+	endTag := `</script>`
+	bodyStr := string(body)
+
+	startIndex := strings.Index(bodyStr, startTag)
+	if startIndex == -1 {
+		return Story{}, fmt.Errorf("JSON-LD script block not found")
+	}
+	startIndex += len(startTag)
+
+	endIndex := strings.Index(bodyStr[startIndex:], endTag)
+	if endIndex == -1 {
+		return Story{}, fmt.Errorf("JSON-LD closing tag not found")
+	}
+	endIndex += startIndex
+
+	jsonContent := body[startIndex:endIndex]
+	var ld jsonLD
+	if err := json.Unmarshal(jsonContent, &ld); err != nil {
+		return Story{}, fmt.Errorf("failed to decode JSON: %w", err)
+	}
+
+	if ld.Name == "" {
+		return Story{}, fmt.Errorf("story name missing in JSON-LD")
+	}
+
+	date, err := time.Parse(time.RFC3339, ld.DateModified)
+	if err != nil {
+		return Story{}, fmt.Errorf("invalid date format %q: %w", ld.DateModified, err)
+	}
+
+	return Story{
+		Name:         ld.Name,
+		DateModified: date,
+	}, nil
+}
--- a/internal/royalroad/scraper_test.go
+++ b/internal/royalroad/scraper_test.go
@@ -0,0 +1,94 @@
+package royalroad
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+)
+
+func TestParseJSONLD(t *testing.T) {
+	tests := []struct {
+		name    string
+		body    string
+		want    Story
+		wantErr bool
+	}{
+		{
+			name: "Success",
+			body: `<html><script type="application/ld+json">{"name":"Test Story","dateModified":"2023-07-06T17:47:41Z"}</script></html>`,
+			want: Story{
+				Name:         "Test Story",
+				DateModified: time.Date(2023, 7, 6, 17, 47, 41, 0, time.UTC),
+			},
+			wantErr: false,
+		},
+		{
+			name:    "Missing Script",
+			body:    `<html><body>No JSON-LD here</body></html>`,
+			wantErr: true,
+		},
+		{
+			name:    "Malformed JSON",
+			body:    `<script type="application/ld+json">{invalid json}</script>`,
+			wantErr: true,
+		},
+		{
+			name:    "Missing Name",
+			body:    `<script type="application/ld+json">{"dateModified":"2023-07-06T17:47:41Z"}</script>`,
+			wantErr: true,
+		},
+		{
+			name:    "Invalid Date",
+			body:    `<script type="application/ld+json">{"name":"Story","dateModified":"not-a-date"}</script>`,
+			wantErr: true,
+		},
+	}
+
+	s := NewScraper()
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := s.parseJSONLD([]byte(tt.body))
+			if (err != nil) != tt.wantErr {
+				t.Errorf("parseJSONLD() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !tt.wantErr && got != tt.want {
+				t.Errorf("parseJSONLD() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestFetchStory_NonRoyalRoadHost(t *testing.T) {
+	s := NewScraper()
+	_, err := s.FetchStory(context.Background(), "https://google.com/fiction/123")
+	if err == nil {
+		t.Error("Expected error for non-RoyalRoad host, got nil")
+	}
+}
+
+func TestFetchStory_ChapterURL(t *testing.T) {
+	s := NewScraper()
+	_, err := s.FetchStory(context.Background(), "https://www.royalroad.com/fiction/123/chapter/456")
+	if err == nil {
+		t.Error("Expected error for chapter URL, got nil")
+	}
+}
+
+func TestFetchStory_HTTPError(t *testing.T) {
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusNotFound)
+	}))
+	defer ts.Close()
+
+	s := NewScraper()
+	// We can't easily mock the host check for the server URL without custom logic,
+	// but we can verify that a non-RR URL (the test server) fails at the host check
+	// before it even hits the server.
+	_, err := s.FetchStory(context.Background(), ts.URL+"/fiction/123")
+	if err == nil {
+		t.Error("Expected error for non-RoyalRoad host, got nil")
+	}
+}