Add options to specify user agent and keep inner HTML

example_test.go

@@ -7,7 +7,7 @@ import (
 	"github.com/nfx/go-htmltable"
 )
 
-func ExampleNewSliceFromUrl() {
+func ExampleNewSliceFromURL() {
 	type Ticker struct {
 		Symbol   string `header:"Symbol"`
 		Security string `header:"Security"`
@@ -31,7 +31,7 @@ func ExampleNewSliceFromURL_rowspansAndColspans() {
 		MultiGpuCrossFire bool   `header:"Multi-GPU CrossFire"`
 		MultiGpuSLI       bool   `header:"Multi-GPU SLI"`
 		USBSupport        string `header:"USBsupport[b]"`
-		SATAPorts         int    `header:"Storage features SATAports"`
+		SATAPorts         string `header:"Storage features SATAports"`
 		RAID              string `header:"Storage features RAID"`
 		AMDStoreMI        bool   `header:"Storage features AMD StoreMI"`
 		Overclocking      string `header:"Processoroverclocking"`
@@ -41,15 +41,17 @@ func ExampleNewSliceFromURL_rowspansAndColspans() {
 		SupportZenPlus    string `header:"CPU support Zen+"`
 		SupportZen2       string `header:"CPU support Zen 2"`
 		SupportZen3       string `header:"CPU support Zen 3"`
+		ECCMemory         string `header:"ECC memory"`
 		Architecture      string `header:"Architecture"`
+		PartNumber        string `header:"Part number"`
 	}
 	am4Chipsets, _ := htmltable.NewSliceFromURL[AM4]("https://en.wikipedia.org/wiki/List_of_AMD_chipsets")
-	fmt.Println(am4Chipsets[2].Model)
-	fmt.Println(am4Chipsets[2].SupportZen2)
+	fmt.Println(am4Chipsets[5].Model)
+	fmt.Println(am4Chipsets[5].SupportZen2)
 
 	// Output:
 	// X370
-	// Varies[c]
+	// Varies[f]
 }
 
 func ExampleNewFromString() {
@@ -96,6 +98,7 @@ func ExampleLogger() {
 	_, _ = htmltable.NewFromURL("https://en.wikipedia.org/wiki/List_of_S%26P_500_companies")
 
 	// Output:
-	// [INFO] found table [columns [Symbol Security SEC filings GICSSector GICS Sub-Industry Headquarters Location Date first added CIK Founded] count 503]
-	// [INFO] found table [columns [Date Added Ticker Added Security Removed Ticker Removed Security Reason] count 316]
+	// [INFO] found table [columns [Symbol Security GICSSector GICS Sub-Industry Headquarters Location Date added CIK Founded] count 503]
+	// [INFO] found table [columns [Effective Date Added Ticker Added Security Removed Ticker Removed Security Reason] count 394]
+	// [INFO] found table [columns [vteS&P 500 companies Energy vteS&P 500 companies APA CorporationBaker HughesChevron CorporationConocoPhillipsCoterraDevon EnergyDiamondback EnergyEOG ResourcesEQT CorporationExpand EnergyExxonMobilHalliburtonKinder MorganMarathon PetroleumOccidental PetroleumOneokPhillips 66SLBTarga ResourcesTexas Pacific Land CorporationValero EnergyWilliams Companies] count 10]
 }

options.go

@@ -0,0 +1,38 @@
+package htmltable
+
+const (
+	baseUserAgent    = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
+	engine           = "AppleWebKit/537.36 (KHTML, like Gecko) "
+	browser          = "Chrome/121.0.0.0 Safari/537.36 "
+	packageInfo      = "nfx/go-htmltable (+https://github.com/nfx/go-htmltable)"
+	DefaultUserAgent = baseUserAgent + engine + browser + packageInfo
+)
+
+type options struct {
+	userAgent string
+	innerHTML bool
+}
+
+type Option func(*options)
+
+// WithUserAgent sets the User-Agent header used when fetching URLs
+func WithUserAgent(ua string) Option {
+	return func(o *options) {
+		o.userAgent = ua
+	}
+}
+
+// WithInnerHTML instructs the parser to keep the inner HTML of each cell
+func WithInnerHTML() Option {
+	return func(o *options) {
+		o.innerHTML = true
+	}
+}
+
+func applyOptions(opts []Option) options {
+	o := options{}
+	for _, opt := range opts {
+		opt(&o)
+	}
+	return o
+}

page.go

@@ -20,6 +20,7 @@ type Page struct {
 	Tables []*Table
 
 	ctx      context.Context
+	opts     options
 	rowSpans []int
 	colSpans []int
 	row      []string
@@ -35,21 +36,21 @@ type Page struct {
 }
 
 // New returns an instance of the page with possibly more than one table
-func New(ctx context.Context, r io.Reader) (*Page, error) {
-	p := &Page{ctx: ctx}
+func New(ctx context.Context, r io.Reader, opts ...Option) (*Page, error) {
+	p := &Page{ctx: ctx, opts: applyOptions(opts)}
 	return p, p.init(r)
 }
 
 // NewFromString is same as New(ctx.Context, io.Reader), but from string
-func NewFromString(r string) (*Page, error) {
-	return New(context.Background(), strings.NewReader(r))
+func NewFromString(r string, opts ...Option) (*Page, error) {
+	return New(context.Background(), strings.NewReader(r), opts...)
 }
 
 // NewFromResponse is same as New(ctx.Context, io.Reader), but from http.Response.
 //
 // In case of failure, returns `ResponseError`, that could be further inspected.
-func NewFromResponse(resp *http.Response) (*Page, error) {
-	p, err := New(resp.Request.Context(), resp.Body)
+func NewFromResponse(resp *http.Response, opts ...Option) (*Page, error) {
+	p, err := New(resp.Request.Context(), resp.Body, opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -59,15 +60,25 @@ func NewFromResponse(resp *http.Response) (*Page, error) {
 // NewFromURL is same as New(ctx.Context, io.Reader), but from URL.
 //
 // In case of failure, returns `ResponseError`, that could be further inspected.
-func NewFromURL(url string) (*Page, error) {
-	resp, err := http.Get(url)
+func NewFromURL(url string, opts ...Option) (*Page, error) {
+	o := applyOptions(opts)
+	req, err := http.NewRequest(http.MethodGet, url, nil)
+	if err != nil {
+		return nil, err
+	}
+	ua := o.userAgent
+	if ua == "" {
+		ua = DefaultUserAgent
+	}
+	req.Header.Set("User-Agent", ua)
+	resp, err := http.DefaultClient.Do(req)
 	if err != nil {
 		return nil, err
 	}
 	if resp.Body != nil {
 		defer resp.Body.Close()
 	}
-	return NewFromResponse(resp)
+	return NewFromResponse(resp, opts...)
 }
 
 // Len returns number of tables found on the page
@@ -203,8 +214,14 @@ func (p *Page) parse(n *html.Node) {
 		p.colSpan = append(p.colSpan, p.intAttrOr(n, "colspan", 1))
 		p.rowSpan = append(p.rowSpan, p.intAttrOr(n, "rowspan", 1))
 		var sb strings.Builder
-		p.innerText(n, &sb)
-		p.row = append(p.row, sb.String())
+		// Only retain inner HTML on td elements.  th elements need to be properly
+		// stripped for header struct reflection
+		if p.opts.innerHTML && n.Data == "td" {
+			p.innerHTML(n, &sb)
+		} else {
+			p.innerText(n, &sb)
+		}
+		p.row = append(p.row, strings.TrimSpace(sb.String()))
 		return
 	case "tr":
 		p.finishRow()
@@ -374,14 +391,24 @@ func (p *Page) innerText(n *html.Node, sb *strings.Builder) {
 		sb.WriteString(strings.TrimSpace(n.Data))
 		return
 	}
-	if n.FirstChild == nil {
+	if n.Type != html.ElementNode {
+		return
+	}
+	switch n.Data {
+	case "script", "style", "head":
 		return
 	}
 	for c := n.FirstChild; c != nil; c = c.NextSibling {
 		p.innerText(c, sb)
 	}
 }
 
+func (p *Page) innerHTML(n *html.Node, sb *strings.Builder) {
+	for c := n.FirstChild; c != nil; c = c.NextSibling {
+		html.Render(sb, c)
+	}
+}
+
 // Table is the low-level representation of raw header and rows.
 //
 // Every cell string value is truncated of its whitespace.

page_test.go

@@ -104,7 +104,10 @@ func TestNewFromHttpResponseError(t *testing.T) {
 }
 
 func TestRealPageFound(t *testing.T) {
-	wiki, err := http.Get("https://en.wikipedia.org/wiki/List_of_S%26P_500_companies")
+	req, err := http.NewRequest("GET", "https://en.wikipedia.org/wiki/List_of_S%26P_500_companies", nil)
+	assertNoError(t, err)
+	req.Header.Set("User-Agent", DefaultUserAgent)
+	wiki, err := http.DefaultClient.Do(req)
 	assertNoError(t, err)
 	p, err := NewFromResponse(wiki)
 	assertNoError(t, err)
@@ -114,11 +117,14 @@ func TestRealPageFound(t *testing.T) {
 }
 
 func TestRealPageFound_BasicRowColSpans(t *testing.T) {
-	wiki, err := http.Get("https://en.wikipedia.org/wiki/List_of_S%26P_500_companies")
+	req, err := http.NewRequest("GET", "https://en.wikipedia.org/wiki/List_of_S%26P_500_companies", nil)
+	assertNoError(t, err)
+	req.Header.Set("User-Agent", DefaultUserAgent)
+	wiki, err := http.DefaultClient.Do(req)
 	assertNoError(t, err)
 	p, err := NewFromResponse(wiki)
 	assertNoError(t, err)
-	snp, err := p.FindWithColumns("Date", "Added Ticker", "Removed Ticker")
+	snp, err := p.FindWithColumns("Effective Date", "Added Ticker", "Removed Ticker")
 	assertNoError(t, err)
 	assertGreaterOrEqual(t, len(snp.Rows), 250)
 }

slice.go

@@ -10,9 +10,9 @@ import (
 )
 
 // NewSlice returns slice of annotated struct types from io.Reader
-func NewSlice[T any](ctx context.Context, r io.Reader) ([]T, error) {
+func NewSlice[T any](ctx context.Context, r io.Reader, opts ...Option) ([]T, error) {
 	f := &feeder[T]{
-		Page: Page{ctx: ctx},
+		Page: Page{ctx: ctx, opts: applyOptions(opts)},
 	}
 	f.init(r)
 	return f.slice()
@@ -27,27 +27,37 @@ func NewSliceFromPage[T any](p *Page) ([]T, error) {
 
 // NewSliceFromString is same as NewSlice(context.Context, io.Reader),
 // but takes just a string.
-func NewSliceFromString[T any](in string) ([]T, error) {
-	return NewSlice[T](context.Background(), strings.NewReader(in))
+func NewSliceFromString[T any](in string, opts ...Option) ([]T, error) {
+	return NewSlice[T](context.Background(), strings.NewReader(in), opts...)
 }
 
 // NewSliceFromString is same as NewSlice(context.Context, io.Reader),
 // but takes just an http.Response
-func NewSliceFromResponse[T any](resp *http.Response) ([]T, error) {
-	return NewSlice[T](resp.Request.Context(), resp.Body)
+func NewSliceFromResponse[T any](resp *http.Response, opts ...Option) ([]T, error) {
+	return NewSlice[T](resp.Request.Context(), resp.Body, opts...)
 }
 
 // NewSliceFromString is same as NewSlice(context.Context, io.Reader),
 // but takes just an URL.
-func NewSliceFromURL[T any](url string) ([]T, error) {
-	resp, err := http.Get(url)
+func NewSliceFromURL[T any](url string, opts ...Option) ([]T, error) {
+	o := applyOptions(opts)
+	req, err := http.NewRequest(http.MethodGet, url, nil)
+	if err != nil {
+		return nil, err
+	}
+	ua := o.userAgent
+	if ua == "" {
+		ua = DefaultUserAgent
+	}
+	req.Header.Set("User-Agent", ua)
+	resp, err := http.DefaultClient.Do(req)
 	if err != nil {
 		return nil, err
 	}
 	if resp.Body != nil {
 		defer resp.Body.Close()
 	}
-	return NewSliceFromResponse[T](resp)
+	return NewSliceFromResponse[T](resp, opts...)
 }
 
 type feeder[T any] struct {