From e0cc5c9ddac8fbe2c57ba598eb58d74f54779e23 Mon Sep 17 00:00:00 2001 From: Jake Ireland Date: Fri, 10 Apr 2026 13:31:31 +1200 Subject: [PATCH 1/4] Add options stub for configurable settings We need a way to be able to pass optional settings to Page, in order to change its behaviour. --- options.go | 13 +++++++++++++ page.go | 17 +++++++++-------- slice.go | 16 ++++++++-------- 3 files changed, 30 insertions(+), 16 deletions(-) create mode 100644 options.go diff --git a/options.go b/options.go new file mode 100644 index 0000000..ba01221 --- /dev/null +++ b/options.go @@ -0,0 +1,13 @@ +package htmltable + +type options struct {} + +type Option func(*options) + +func applyOptions(opts []Option) options { + o := options{} + for _, opt := range opts { + opt(&o) + } + return o +} diff --git a/page.go b/page.go index 7fc450f..575e038 100644 --- a/page.go +++ b/page.go @@ -20,6 +20,7 @@ type Page struct { Tables []*Table ctx context.Context + opts options rowSpans []int colSpans []int row []string @@ -35,21 +36,21 @@ type Page struct { } // New returns an instance of the page with possibly more than one table -func New(ctx context.Context, r io.Reader) (*Page, error) { - p := &Page{ctx: ctx} +func New(ctx context.Context, r io.Reader, opts ...Option) (*Page, error) { + p := &Page{ctx: ctx, opts: applyOptions(opts)} return p, p.init(r) } // NewFromString is same as New(ctx.Context, io.Reader), but from string -func NewFromString(r string) (*Page, error) { - return New(context.Background(), strings.NewReader(r)) +func NewFromString(r string, opts ...Option) (*Page, error) { + return New(context.Background(), strings.NewReader(r), opts...) } // NewFromResponse is same as New(ctx.Context, io.Reader), but from http.Response. // // In case of failure, returns `ResponseError`, that could be further inspected. -func NewFromResponse(resp *http.Response) (*Page, error) { - p, err := New(resp.Request.Context(), resp.Body) +func NewFromResponse(resp *http.Response, opts ...Option) (*Page, error) { + p, err := New(resp.Request.Context(), resp.Body, opts...) if err != nil { return nil, err } @@ -59,7 +60,7 @@ func NewFromResponse(resp *http.Response) (*Page, error) { // NewFromURL is same as New(ctx.Context, io.Reader), but from URL. // // In case of failure, returns `ResponseError`, that could be further inspected. -func NewFromURL(url string) (*Page, error) { +func NewFromURL(url string, opts ...Option) (*Page, error) { resp, err := http.Get(url) if err != nil { return nil, err @@ -67,7 +68,7 @@ func NewFromURL(url string) (*Page, error) { if resp.Body != nil { defer resp.Body.Close() } - return NewFromResponse(resp) + return NewFromResponse(resp, opts...) } // Len returns number of tables found on the page diff --git a/slice.go b/slice.go index 64ea55e..bcd8b13 100644 --- a/slice.go +++ b/slice.go @@ -10,9 +10,9 @@ import ( ) // NewSlice returns slice of annotated struct types from io.Reader -func NewSlice[T any](ctx context.Context, r io.Reader) ([]T, error) { +func NewSlice[T any](ctx context.Context, r io.Reader, opts ...Option) ([]T, error) { f := &feeder[T]{ - Page: Page{ctx: ctx}, + Page: Page{ctx: ctx, opts: applyOptions(opts)}, } f.init(r) return f.slice() @@ -27,19 +27,19 @@ func NewSliceFromPage[T any](p *Page) ([]T, error) { // NewSliceFromString is same as NewSlice(context.Context, io.Reader), // but takes just a string. -func NewSliceFromString[T any](in string) ([]T, error) { - return NewSlice[T](context.Background(), strings.NewReader(in)) +func NewSliceFromString[T any](in string, opts ...Option) ([]T, error) { + return NewSlice[T](context.Background(), strings.NewReader(in), opts...) } // NewSliceFromString is same as NewSlice(context.Context, io.Reader), // but takes just an http.Response -func NewSliceFromResponse[T any](resp *http.Response) ([]T, error) { - return NewSlice[T](resp.Request.Context(), resp.Body) +func NewSliceFromResponse[T any](resp *http.Response, opts ...Option) ([]T, error) { + return NewSlice[T](resp.Request.Context(), resp.Body, opts...) } // NewSliceFromString is same as NewSlice(context.Context, io.Reader), // but takes just an URL. -func NewSliceFromURL[T any](url string) ([]T, error) { +func NewSliceFromURL[T any](url string, opts ...Option) ([]T, error) { resp, err := http.Get(url) if err != nil { return nil, err @@ -47,7 +47,7 @@ func NewSliceFromURL[T any](url string) ([]T, error) { if resp.Body != nil { defer resp.Body.Close() } - return NewSliceFromResponse[T](resp) + return NewSliceFromResponse[T](resp, opts...) } type feeder[T any] struct { From 0ffe54c9235bf1e9b556b8dcf08460da27b6e2cb Mon Sep 17 00:00:00 2001 From: Jake Ireland Date: Fri, 10 Apr 2026 13:51:31 +1200 Subject: [PATCH 2/4] Add option to set user agent Many websites will now require a user agent. Even Wikipedia is blocking our requests in our unit tests. We now allow a user agent to be set for HTTP queries. We also specify a default user agent to use if none is set by the user. --- options.go | 19 ++++++++++++++++++- page.go | 12 +++++++++++- slice.go | 12 +++++++++++- 3 files changed, 40 insertions(+), 3 deletions(-) diff --git a/options.go b/options.go index ba01221..aaf73bb 100644 --- a/options.go +++ b/options.go @@ -1,9 +1,26 @@ package htmltable -type options struct {} +const ( + baseUserAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " + engine = "AppleWebKit/537.36 (KHTML, like Gecko) " + browser = "Chrome/121.0.0.0 Safari/537.36 " + packageInfo = "nfx/go-htmltable (+https://github.com/nfx/go-htmltable)" + DefaultUserAgent = baseUserAgent + engine + browser + packageInfo +) + +type options struct { + userAgent string +} type Option func(*options) +// WithUserAgent sets the User-Agent header used when fetching URLs +func WithUserAgent(ua string) Option { + return func(o *options) { + o.userAgent = ua + } +} + func applyOptions(opts []Option) options { o := options{} for _, opt := range opts { diff --git a/page.go b/page.go index 575e038..4880725 100644 --- a/page.go +++ b/page.go @@ -61,7 +61,17 @@ func NewFromResponse(resp *http.Response, opts ...Option) (*Page, error) { // // In case of failure, returns `ResponseError`, that could be further inspected. func NewFromURL(url string, opts ...Option) (*Page, error) { - resp, err := http.Get(url) + o := applyOptions(opts) + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + return nil, err + } + ua := o.userAgent + if ua == "" { + ua = DefaultUserAgent + } + req.Header.Set("User-Agent", ua) + resp, err := http.DefaultClient.Do(req) if err != nil { return nil, err } diff --git a/slice.go b/slice.go index bcd8b13..0e6bcf8 100644 --- a/slice.go +++ b/slice.go @@ -40,7 +40,17 @@ func NewSliceFromResponse[T any](resp *http.Response, opts ...Option) ([]T, erro // NewSliceFromString is same as NewSlice(context.Context, io.Reader), // but takes just an URL. func NewSliceFromURL[T any](url string, opts ...Option) ([]T, error) { - resp, err := http.Get(url) + o := applyOptions(opts) + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + return nil, err + } + ua := o.userAgent + if ua == "" { + ua = DefaultUserAgent + } + req.Header.Set("User-Agent", ua) + resp, err := http.DefaultClient.Do(req) if err != nil { return nil, err } From e53ec05a04e3d27fbcb1ffa5bb60ba4f05314123 Mon Sep 17 00:00:00 2001 From: Jake Ireland Date: Fri, 10 Apr 2026 15:11:27 +1200 Subject: [PATCH 3/4] Fix unit tests --- example_test.go | 17 ++++++++++------- page.go | 6 +++++- page_test.go | 12 +++++++++--- 3 files changed, 24 insertions(+), 11 deletions(-) diff --git a/example_test.go b/example_test.go index 339bc05..2bdd0d3 100644 --- a/example_test.go +++ b/example_test.go @@ -7,7 +7,7 @@ import ( "github.com/nfx/go-htmltable" ) -func ExampleNewSliceFromUrl() { +func ExampleNewSliceFromURL() { type Ticker struct { Symbol string `header:"Symbol"` Security string `header:"Security"` @@ -31,7 +31,7 @@ func ExampleNewSliceFromURL_rowspansAndColspans() { MultiGpuCrossFire bool `header:"Multi-GPU CrossFire"` MultiGpuSLI bool `header:"Multi-GPU SLI"` USBSupport string `header:"USBsupport[b]"` - SATAPorts int `header:"Storage features SATAports"` + SATAPorts string `header:"Storage features SATAports"` RAID string `header:"Storage features RAID"` AMDStoreMI bool `header:"Storage features AMD StoreMI"` Overclocking string `header:"Processoroverclocking"` @@ -41,15 +41,17 @@ func ExampleNewSliceFromURL_rowspansAndColspans() { SupportZenPlus string `header:"CPU support Zen+"` SupportZen2 string `header:"CPU support Zen 2"` SupportZen3 string `header:"CPU support Zen 3"` + ECCMemory string `header:"ECC memory"` Architecture string `header:"Architecture"` + PartNumber string `header:"Part number"` } am4Chipsets, _ := htmltable.NewSliceFromURL[AM4]("https://en.wikipedia.org/wiki/List_of_AMD_chipsets") - fmt.Println(am4Chipsets[2].Model) - fmt.Println(am4Chipsets[2].SupportZen2) + fmt.Println(am4Chipsets[5].Model) + fmt.Println(am4Chipsets[5].SupportZen2) // Output: // X370 - // Varies[c] + // Varies[f] } func ExampleNewFromString() { @@ -96,6 +98,7 @@ func ExampleLogger() { _, _ = htmltable.NewFromURL("https://en.wikipedia.org/wiki/List_of_S%26P_500_companies") // Output: - // [INFO] found table [columns [Symbol Security SEC filings GICSSector GICS Sub-Industry Headquarters Location Date first added CIK Founded] count 503] - // [INFO] found table [columns [Date Added Ticker Added Security Removed Ticker Removed Security Reason] count 316] + // [INFO] found table [columns [Symbol Security GICSSector GICS Sub-Industry Headquarters Location Date added CIK Founded] count 503] + // [INFO] found table [columns [Effective Date Added Ticker Added Security Removed Ticker Removed Security Reason] count 394] + // [INFO] found table [columns [vteS&P 500 companies Energy vteS&P 500 companies APA CorporationBaker HughesChevron CorporationConocoPhillipsCoterraDevon EnergyDiamondback EnergyEOG ResourcesEQT CorporationExpand EnergyExxonMobilHalliburtonKinder MorganMarathon PetroleumOccidental PetroleumOneokPhillips 66SLBTarga ResourcesTexas Pacific Land CorporationValero EnergyWilliams Companies] count 10] } diff --git a/page.go b/page.go index 4880725..0677fb5 100644 --- a/page.go +++ b/page.go @@ -385,7 +385,11 @@ func (p *Page) innerText(n *html.Node, sb *strings.Builder) { sb.WriteString(strings.TrimSpace(n.Data)) return } - if n.FirstChild == nil { + if n.Type != html.ElementNode { + return + } + switch n.Data { + case "script", "style", "head": return } for c := n.FirstChild; c != nil; c = c.NextSibling { diff --git a/page_test.go b/page_test.go index 8da1c85..6417baa 100644 --- a/page_test.go +++ b/page_test.go @@ -104,7 +104,10 @@ func TestNewFromHttpResponseError(t *testing.T) { } func TestRealPageFound(t *testing.T) { - wiki, err := http.Get("https://en.wikipedia.org/wiki/List_of_S%26P_500_companies") + req, err := http.NewRequest("GET", "https://en.wikipedia.org/wiki/List_of_S%26P_500_companies", nil) + assertNoError(t, err) + req.Header.Set("User-Agent", DefaultUserAgent) + wiki, err := http.DefaultClient.Do(req) assertNoError(t, err) p, err := NewFromResponse(wiki) assertNoError(t, err) @@ -114,11 +117,14 @@ func TestRealPageFound(t *testing.T) { } func TestRealPageFound_BasicRowColSpans(t *testing.T) { - wiki, err := http.Get("https://en.wikipedia.org/wiki/List_of_S%26P_500_companies") + req, err := http.NewRequest("GET", "https://en.wikipedia.org/wiki/List_of_S%26P_500_companies", nil) + assertNoError(t, err) + req.Header.Set("User-Agent", DefaultUserAgent) + wiki, err := http.DefaultClient.Do(req) assertNoError(t, err) p, err := NewFromResponse(wiki) assertNoError(t, err) - snp, err := p.FindWithColumns("Date", "Added Ticker", "Removed Ticker") + snp, err := p.FindWithColumns("Effective Date", "Added Ticker", "Removed Ticker") assertNoError(t, err) assertGreaterOrEqual(t, len(snp.Rows), 250) } From 41e76133e5b1e725af2b4f03a1522d422baa71b3 Mon Sep 17 00:00:00 2001 From: Jake Ireland Date: Sat, 11 Apr 2026 00:01:35 +1200 Subject: [PATCH 4/4] Add option to retain inner HTML of table rows Addresses #37 --- options.go | 8 ++++++++ page.go | 16 ++++++++++++++-- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/options.go b/options.go index aaf73bb..a0534f3 100644 --- a/options.go +++ b/options.go @@ -10,6 +10,7 @@ const ( type options struct { userAgent string + innerHTML bool } type Option func(*options) @@ -21,6 +22,13 @@ func WithUserAgent(ua string) Option { } } +// WithInnerHTML instructs the parser to keep the inner HTML of each cell +func WithInnerHTML() Option { + return func(o *options) { + o.innerHTML = true + } +} + func applyOptions(opts []Option) options { o := options{} for _, opt := range opts { diff --git a/page.go b/page.go index 0677fb5..d50ef8a 100644 --- a/page.go +++ b/page.go @@ -214,8 +214,14 @@ func (p *Page) parse(n *html.Node) { p.colSpan = append(p.colSpan, p.intAttrOr(n, "colspan", 1)) p.rowSpan = append(p.rowSpan, p.intAttrOr(n, "rowspan", 1)) var sb strings.Builder - p.innerText(n, &sb) - p.row = append(p.row, sb.String()) + // Only retain inner HTML on td elements. th elements need to be properly + // stripped for header struct reflection + if p.opts.innerHTML && n.Data == "td" { + p.innerHTML(n, &sb) + } else { + p.innerText(n, &sb) + } + p.row = append(p.row, strings.TrimSpace(sb.String())) return case "tr": p.finishRow() @@ -397,6 +403,12 @@ func (p *Page) innerText(n *html.Node, sb *strings.Builder) { } } +func (p *Page) innerHTML(n *html.Node, sb *strings.Builder) { + for c := n.FirstChild; c != nil; c = c.NextSibling { + html.Render(sb, c) + } +} + // Table is the low-level representation of raw header and rows. // // Every cell string value is truncated of its whitespace.