Introduction

Installation

CLI

go install github.com/dotcommander/defuddle/cmd/defuddle@latest

Library

go get github.com/dotcommander/defuddle

Quick Start

CLI

defuddle parse https://example.com/article

defuddle parse https://example.com/article --markdown
defuddle parse https://example.com/article --json

Library — fetch and parse a URL

import (
 "context"
 "fmt"
 "github.com/dotcommander/defuddle"
)
result, err := defuddle.ParseFromURL(context.Background(), "https://example.com/article", nil)
if err != nil {
 log.Fatal(err)
}
fmt.Println(result.Title)
fmt.Println(result.Content) // clean HTML

Library — parse HTML you already have

result, err := defuddle.ParseFromString(ctx, htmlString, &defuddle.Options{
 URL: "https://example.com/article", // enables relative URL resolution
})

Lower-level API

d, err := defuddle.NewDefuddle(htmlString, &defuddle.Options{
 URL: "https://example.com/article",
 Markdown: true,
})
if err != nil {
 log.Fatal(err)
}
result, err := d.Parse(ctx)
fmt.Printf("Title: %s\n", result.Title)
fmt.Printf("Author: %s\n", result.Author)
fmt.Printf("Published: %s\n", result.Published)
fmt.Printf("Language: %s\n", result.Language)
fmt.Printf("Word Count: %d\n", result.WordCount)
fmt.Printf("Content: %s\n", result.Content) // Markdown when Markdown: true

Extracting Content

From a URL

result, err := defuddle.ParseFromURL(ctx, "https://example.com/article", &defuddle.Options{
 Markdown: true,
})

Multiple URLs (concurrent)

urls := []string{
 "https://example.com/article-1",
 "https://example.com/article-2",
}
results := defuddle.ParseFromURLs(ctx, urls, &defuddle.Options{
 MaxConcurrency: 10,
 Markdown: true,
})
for _, r := range results {
 if r.Err != nil {
 log.Printf("failed %s: %v", r.URL, r.Err)
 continue
 }
 fmt.Printf("%s (%d words)\n", r.Result.Title, r.Result.WordCount)
}

Markdown Output

result, err := defuddle.ParseFromURL(ctx, url, &defuddle.Options{Markdown: true})
fmt.Println(result.Content) // Markdown

result, err := defuddle.ParseFromURL(ctx, url, &defuddle.Options{SeparateMarkdown: true})
fmt.Println(result.Content) // HTML
fmt.Println(*result.ContentMarkdown) // Markdown

Site-Specific Extractors

Custom Extractors

type RecipeExtractor struct {
 *extractors.ExtractorBase
}
func NewRecipeExtractor(doc *goquery.Document, url string, schema any) extractors.BaseExtractor {
 return &RecipeExtractor{ExtractorBase: extractors.NewExtractorBase(doc, url, schema)}
}
func (e *RecipeExtractor) Name() string { return "RecipeExtractor" }
// CanExtract returns true only when the page has a recipe card — not every page on the host.
func (e *RecipeExtractor) CanExtract() bool {
 return e.GetDocument().Find("article.recipe-card").Length() > 0
}
func (e *RecipeExtractor) Extract() *extractors.ExtractorResult {
 doc := e.GetDocument()
 // ContentHTML is what becomes Result.Content.
 content, _ := doc.Find("article.recipe-card").Html()
 title := strings.TrimSpace(doc.Find("h1.recipe-title").Text())
 author := strings.TrimSpace(doc.Find(".recipe-author").Text())
 return &extractors.ExtractorResult{
 ContentHTML: content,
 Variables: map[string]string{
 "title": title,
 "author": author,
 "site": "Recipe Site",
 },
 }
}

extractors.Register(extractors.ExtractorMapping{
 Patterns: []any{"recipes.example.com"},
 Extractor: NewRecipeExtractor,
})

Configuration

Options

opts := &defuddle.Options{
 // Output
 Markdown: false, // Return content as Markdown
 SeparateMarkdown: false, // Return both HTML and Markdown
 // Content selection
 ContentSelector: "", // CSS selector override for main content
 URL: "", // Source URL (used for link resolution and domain detection)
 // Removal controls — pointer bools default to true when nil.
 // Use defuddle.PtrBool(false) to explicitly disable.
 RemoveExactSelectors: nil, // Remove known clutter (ads, nav, social buttons)
 RemovePartialSelectors: nil, // Remove probable clutter (class/id pattern matching)
 RemoveHiddenElements: nil, // Remove display:none and hidden elements
 RemoveContentPatterns: nil, // Remove boilerplate (breadcrumbs, related posts, etc.)
 RemoveLowScoring: nil, // Remove low-scoring non-content blocks
 RemoveImages: false, // Strip all images from output
 // Element processing
 ProcessCode: false, // Normalize code blocks with language detection
 ProcessImages: false, // Optimize images (lazy-load resolution, srcset)
 ProcessHeadings: false, // Clean heading hierarchy
 ProcessMath: false, // Normalize MathJax/KaTeX formulas
 ProcessFootnotes: false, // Standardize footnote format
 ProcessRoles: false, // Convert ARIA roles to semantic HTML
 // HTTP (for ParseFromURL / ParseFromURLs)
 Client: nil, // Custom *requests.Client; default uses 30s timeout
 MaxConcurrency: 5, // Parallel limit for ParseFromURLs
 Debug: false,
}

Content Selector

result, err := defuddle.ParseFromURL(ctx, url, &defuddle.Options{
 ContentSelector: "article.post-body",
})

The Extraction Pipeline

HTML Input
 |
 v
1. Schema.org -- Extract JSON-LD structured data
2. Site Detection -- Match URL to specialized extractor
3. Shadow DOM -- Flatten shadow roots and resolve React SSR
4. Selector Removal -- Strip known clutter by CSS selector
5. Content Scoring -- Score nodes and identify main content
6. Content Patterns -- Remove boilerplate (breadcrumbs, related posts, newsletters)
7. Standardization -- Normalize headings, footnotes, code blocks, images, math
8. Markdown -- Convert to Markdown (if requested)
 |
 v
Result

The Result Object

CLI Usage

Extracting Content

# From a URL
defuddle parse https://example.com/article
# From a local file
defuddle parse article.html
# From stdin (pipe HTML in)
curl -s https://example.com/article | defuddle parse
# As Markdown
defuddle parse https://example.com/article --markdown
# As JSON with all metadata
defuddle parse https://example.com/article --json
# Extract a single field
defuddle parse https://example.com/article --property title

Batch Processing

defuddle batch < urls.txt > articles.jsonl
# From a file, with markdown, 10 parallel fetches
defuddle batch --input urls.txt --markdown --concurrency 10 > articles.jsonl
# Bound total batch duration; --continue-on-error emits per-line error objects
defuddle batch --input urls.txt --timeout 2m --continue-on-error > articles.jsonl

Saving Output

defuddle parse https://example.com/article --markdown --output article.md

Authentication and Proxies

# Custom headers
defuddle parse https://example.com --header "Authorization: Bearer token123"
# Through a proxy
defuddle parse https://example.com --proxy http://localhost:8080
# Custom timeout
defuddle parse https://slow-site.com --timeout 120s

All CLI Options

Limitations

Examples

go run ./examples/basic # Simple extraction
go run ./examples/markdown # HTML to Markdown
go run ./examples/advanced # Full option usage
go run ./examples/extractors # Site-specific extraction
go run ./examples/custom_extractor # Building a custom extractor

Testing

# Run all tests
go test ./...
# With race detection
go test -race ./...
# Benchmarks
go test -bench=. -benchmem ./...

Credits

License