Add transform.HTMLToMarkdown

Fixes #13946

Author: bep

go.mod

@@ -105,6 +105,8 @@ require (
 	github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.29.0 // indirect
 	github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.53.0 // indirect
 	github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.53.0 // indirect
+	github.com/JohannesKaufmann/dom v0.2.0 // indirect
+	github.com/JohannesKaufmann/html-to-markdown/v2 v2.4.0 // indirect
 	github.com/aws/aws-sdk-go v1.55.7 // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.11 // indirect
 	github.com/aws/aws-sdk-go-v2/config v1.29.17 // indirect
@@ -162,7 +164,7 @@ require (
 	github.com/perimeterx/marshmallow v1.1.5 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect
-	github.com/rivo/uniseg v0.2.0 // indirect
+	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/spiffe/go-spiffe/v2 v2.5.0 // indirect
 	github.com/woodsbury/decimal128 v1.3.0 // indirect

go.sum

@@ -91,6 +91,10 @@ github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0
 github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.53.0/go.mod h1:jUZ5LYlw40WMd07qxcQJD5M40aUxrfwqQX1g7zxYnrQ=
 github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.53.0 h1:Ron4zCA/yk6U7WOBXhTJcDpsUBG9npumK6xw2auFltQ=
 github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.53.0/go.mod h1:cSgYe11MCNYunTnRXrKiR/tHc0eoKjICUuWpNZoVCOo=
+github.com/JohannesKaufmann/dom v0.2.0 h1:1bragmEb19K8lHAqgFgqCpiPCFEZMTXzOIEjuxkUfLQ=
+github.com/JohannesKaufmann/dom v0.2.0/go.mod h1:57iSUl5RKric4bUkgos4zu6Xt5LMHUnw3TF1l5CbGZo=
+github.com/JohannesKaufmann/html-to-markdown/v2 v2.4.0 h1:C0/TerKdQX9Y9pbYi1EsLr5LDNANsqunyI/btpyfCg8=
+github.com/JohannesKaufmann/html-to-markdown/v2 v2.4.0/go.mod h1:OLaKh+giepO8j7teevrNwiy/fwf8LXgoc9g7rwaE1jk=
 github.com/alecthomas/assert/v2 v2.11.0 h1:2Q9r3ki8+JYXvGsDyBXwH3LcJ+WK5D0gc5E8vS6K3D0=
 github.com/alecthomas/assert/v2 v2.11.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k=
 github.com/alecthomas/chroma/v2 v2.20.0 h1:sfIHpxPyR07/Oylvmcai3X/exDlE8+FA820NTz+9sGw=
@@ -465,6 +469,8 @@ github.com/redis/go-redis/v9 v9.8.0 h1:q3nRvjrlge/6UD7eTu/DSg2uYiU2mCL0G/uzBWqhi
 github.com/redis/go-redis/v9 v9.8.0/go.mod h1:huWgSWd8mW6+m0VPhJjSSQ+d6Nh1VICQ6Q5lHuCH/Iw=
 github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
+github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
+github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
 github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=

tpl/transform/transform.go

@@ -27,6 +27,11 @@ import (
 	"strings"
 	"sync/atomic"
 
+	htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown/v2/converter"
+	"github.com/JohannesKaufmann/html-to-markdown/v2/plugin/base"
+	"github.com/JohannesKaufmann/html-to-markdown/v2/plugin/commonmark"
+	"github.com/JohannesKaufmann/html-to-markdown/v2/plugin/table"
+
 	bp "github.com/gohugoio/hugo/bufferpool"
 
 	"github.com/bep/goportabletext"
@@ -324,3 +329,32 @@ func (ns *Namespace) ToMath(ctx context.Context, args ...any) (template.HTML, er
 func (ns *Namespace) Reset() {
 	ns.cacheUnmarshal.Clear()
 }
+
+// This was added in Hugo v0.151.0 and should be considered experimental for now.
+// We need to test this out in the wild for a while before committing to this API,
+// and there will eventually be more options here.
+func (ns *Namespace) HTMLToMarkdown(ctx context.Context, args ...any) (string, error) {
+	if len(args) < 1 {
+		return "", errors.New("must provide at least one argument")
+	}
+	input, err := cast.ToStringE(args[0])
+	if err != nil {
+		return "", err
+	}
+
+	plugins := []htmltomarkdown.Plugin{
+		base.NewBasePlugin(),
+		commonmark.NewCommonmarkPlugin(),
+		table.NewTablePlugin(),
+	}
+
+	conv := htmltomarkdown.NewConverter(
+		htmltomarkdown.WithPlugins(plugins...),
+	)
+
+	markdown, err := conv.ConvertString(input)
+	if err != nil {
+		return "", err
+	}
+	return markdown, nil
+}

tpl/transform/transform_integration_test.go

@@ -535,3 +535,59 @@ disableKinds = ['page','rss','section','sitemap','taxonomy','term']
 	b, err = hugolib.TestE(t, f)
 	b.Assert(err.Error(), qt.Contains, "invalid strict mode")
 }
+
+func TestHTMLToMarkdown(t *testing.T) {
+	t.Parallel()
+
+	markdown := `
+# Heading
+
+Some **bold** text.
+
+A [link](https://example.com).
+
+An image:
+
+![alt text](https://example.com/image.jpg "Image Title")
+
+A list:
+
+- Item 1
+- Item 2
+  - Item 2a
+  - Item 2b
+
+A table:
+
+| Header 1 | Header 2 |
+|----------|----------|
+| Cell 1   | Cell 2   |
+| Cell 3   | Cell 4   |
+
+A blockquote:
+
+> This is a quote.
+	`
+	files := `
+-- hugo.toml --
+disableKinds = ['rss','section','sitemap','taxonomy','term']
+-- layouts/all.html --
+All html.
+-- layouts/all.markdown --
+{{ .Content | transform.HTMLToMarkdown | safeHTML }}
+-- content/p1.md --
+---
+title: p1
+outputs: ["html", "markdown"]
+---
+`
+
+	files += markdown
+
+	b := hugolib.Test(t, files)
+
+	b.AssertFileContent("public/p1/index.html", `All html.`)
+
+	// There are some white space differences, so we cannot do an exact match.
+	b.AssertFileContent("public/p1/index.md", markdown)
+}