initial commit

This commit is contained in:
2026-08-16 21:18:45 -05:00
commit 1e05a01bcf
122 changed files with 29178 additions and 0 deletions
+262
View File
@@ -0,0 +1,262 @@
package blacklist
import (
"strings"
"testing"
"github.com/owen/vibedns/internal/models"
)
func TestMatchExactAndSubdomains(t *testing.T) {
b := NewBuilder(1, "Test", models.KindBlacklist, 4)
b.Add("example.com", true) // covers subdomains
b.Add("exact.example.net", false) // this name only
b.Add("*.wild.example", false) // wildcard syntax implies subdomains
set := b.Build()
tests := []struct {
name string
query string
want bool
}{
{"exact match on a subdomain entry", "example.com", true},
{"one level down", "www.example.com", true},
{"several levels down", "a.b.c.example.com", true},
{"trailing dot is ignored", "www.example.com.", true},
{"case is ignored", "WWW.Example.COM", true},
{"sibling is not matched", "notexample.com", false},
{"parent is not matched", "com", false},
{"exact-only entry matches itself", "exact.example.net", true},
{"exact-only entry does not cover subdomains", "www.exact.example.net", false},
{"wildcard entry matches the base", "wild.example", true},
{"wildcard entry covers subdomains", "anything.wild.example", true},
{"unlisted name", "example.org", false},
{"empty query", "", false},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
matched, ok := set.Match(tc.query)
if ok != tc.want {
t.Errorf("Match(%q) = %v (matched %q), want %v", tc.query, ok, matched, tc.want)
}
})
}
}
// TestSubdomainCoverageIsNotStored is the memory property the design depends
// on: covering every subdomain must not cost an entry per subdomain.
func TestSubdomainCoverageIsNotStored(t *testing.T) {
b := NewBuilder(1, "Test", models.KindBlacklist, 1)
b.Add("example.com", true)
set := b.Build()
if set.Len() != 1 {
t.Fatalf("stored %d entries, want exactly 1", set.Len())
}
for _, name := range []string{
"a.example.com", "b.a.example.com", "c.b.a.example.com",
"very.deeply.nested.name.example.com",
} {
if _, ok := set.Match(name); !ok {
t.Errorf("%s should be covered by the single stored entry", name)
}
}
}
func TestEmptySetMatchesNothing(t *testing.T) {
set := NewBuilder(1, "Empty", models.KindBlacklist, 0).Build()
if _, ok := set.Match("example.com"); ok {
t.Error("an empty set must not match")
}
var nilSet *Set
if _, ok := nilSet.Match("example.com"); ok {
t.Error("a nil set must not match")
}
}
func TestParsePlainList(t *testing.T) {
input := `# A comment
example.com
bad.example
tracker.example.net
; another comment style
! adblock comment
`
domains, summary := ParseString(input, ParseOptions{DefaultMatchSubdomains: true})
want := []string{"example.com", "bad.example", "tracker.example.net"}
if len(domains) != len(want) {
t.Fatalf("parsed %d domains, want %d: %v", len(domains), len(want), domains)
}
for i, d := range domains {
if d.Domain != want[i] {
t.Errorf("domain[%d] = %q, want %q", i, d.Domain, want[i])
}
}
if summary.Imported != 3 {
t.Errorf("imported = %d, want 3", summary.Imported)
}
if summary.Ignored != 4 {
t.Errorf("ignored = %d, want 4 comments and blanks", summary.Ignored)
}
}
func TestParseHostsFile(t *testing.T) {
input := `# Hosts-style blocklist
0.0.0.0 example.com
127.0.0.1 tracker.example.net
:: bad.example
0.0.0.0 multi-a.example multi-b.example
127.0.0.1 localhost
::1 ip6-localhost
0.0.0.0
192.168.1.1 printer.local
`
domains, summary := ParseString(input, ParseOptions{DefaultMatchSubdomains: true})
got := map[string]bool{}
for _, d := range domains {
got[d.Domain] = true
}
for _, want := range []string{
"example.com", "tracker.example.net", "bad.example",
"multi-a.example", "multi-b.example", "printer.local",
} {
if !got[want] {
t.Errorf("expected %q to be imported; got %v", want, keys(got))
}
}
// Loopback names are hosts-file boilerplate, not blockable domains.
for _, unwanted := range []string{"localhost", "ip6-localhost"} {
if got[unwanted] {
t.Errorf("%q should not have been imported", unwanted)
}
}
if summary.LinesProcessed != 9 {
t.Errorf("lines processed = %d, want 9", summary.LinesProcessed)
}
}
func TestParseAdblockRules(t *testing.T) {
input := `[Adblock Plus 2.0]
||ads.example.com^
||tracker.example.net^$third-party
@@||allowed.example.com^
||example.org/path/to/thing
##.banner-class
`
domains, _ := ParseString(input, ParseOptions{})
got := map[string]bool{}
for _, d := range domains {
got[d.Domain] = true
}
if !got["ads.example.com"] {
t.Error("a plain ||domain^ rule should be imported")
}
if !got["tracker.example.net"] {
t.Error("a ||domain^ rule with options should import the domain part")
}
if got["allowed.example.com"] {
t.Error("an @@ exception rule must not become a block entry")
}
// An Adblock host rule implies subdomain coverage.
for _, d := range domains {
if d.Domain == "ads.example.com" && !d.MatchSubdomains {
t.Error("an Adblock host rule should cover subdomains")
}
}
}
func TestParseDeduplicatesWithinFile(t *testing.T) {
input := "example.com\nexample.com\nEXAMPLE.COM\nexample.com.\n"
domains, summary := ParseString(input, ParseOptions{})
if len(domains) != 1 {
t.Errorf("parsed %d domains, want 1 after normalisation", len(domains))
}
if summary.Duplicates != 3 {
t.Errorf("duplicates = %d, want 3", summary.Duplicates)
}
}
func TestParseRejectsInvalidEntries(t *testing.T) {
input := "example.com\nnot a domain at all\n-bad-.example\nvalid.example\n"
domains, summary := ParseString(input, ParseOptions{})
got := map[string]bool{}
for _, d := range domains {
got[d.Domain] = true
}
if !got["example.com"] || !got["valid.example"] {
t.Errorf("valid entries were dropped: %v", keys(got))
}
if summary.Invalid == 0 {
t.Error("expected invalid entries to be counted")
}
if len(summary.InvalidSamples) == 0 {
t.Error("expected a sample of the rejected lines for the operator")
}
}
func TestParseIgnoresIPOnlyAndSingleLabel(t *testing.T) {
input := "192.0.2.1\nlocalhost\ncom\nvalid.example\n"
domains, _ := ParseString(input, ParseOptions{})
for _, d := range domains {
if d.Domain != "valid.example" {
t.Errorf("unexpected import %q", d.Domain)
}
}
}
func TestParseLargeInput(t *testing.T) {
// A realistic blocklist shape: confirm parsing scales and counts correctly.
var b strings.Builder
const n = 50000
for i := 0; i < n; i++ {
b.WriteString("0.0.0.0 host")
b.WriteString(itoa(i))
b.WriteString(".example.com\n")
}
domains, summary := ParseString(b.String(), ParseOptions{DefaultMatchSubdomains: true})
if len(domains) != n {
t.Errorf("parsed %d domains, want %d", len(domains), n)
}
if summary.Imported != n {
t.Errorf("imported = %d, want %d", summary.Imported, n)
}
if summary.Invalid != 0 {
t.Errorf("invalid = %d, want 0", summary.Invalid)
}
}
func keys(m map[string]bool) []string {
out := make([]string, 0, len(m))
for k := range m {
out = append(out, k)
}
return out
}
func itoa(i int) string {
if i == 0 {
return "0"
}
var buf [12]byte
pos := len(buf)
for i > 0 {
pos--
buf[pos] = byte('0' + i%10)
i /= 10
}
return string(buf[pos:])
}
+145
View File
@@ -0,0 +1,145 @@
// Package blacklist implements the domain lookup structure used by blacklists
// and allowlists, plus the parsers for bulk imports.
//
// The matcher is built for lists with hundreds of thousands of entries. It
// stores one map entry per configured domain and answers "is this name, or any
// parent of it, listed?" by walking the name's suffixes, which is bounded by
// the label count rather than by the size of the list. Subdomain coverage
// therefore costs nothing extra: blocking example.com automatically covers
// a.b.example.com without storing a single additional row.
package blacklist
import (
"strings"
)
// Match flags stored per domain. A single map keeps the memory footprint of a
// large list to one entry per domain rather than one per match mode.
const (
flagExact uint8 = 1 << 0 // matches the domain itself only
flagSuffix uint8 = 1 << 1 // matches the domain and every subdomain
)
// Set is an immutable compiled domain list.
type Set struct {
ID int64
Name string
Kind string
domains map[string]uint8
}
// Builder accumulates domains before freezing them into a Set.
type Builder struct {
id int64
name string
kind string
domains map[string]uint8
}
// NewBuilder starts building a list. sizeHint pre-sizes the map, which matters
// when loading a list with hundreds of thousands of domains.
func NewBuilder(id int64, name, kind string, sizeHint int) *Builder {
if sizeHint < 8 {
sizeHint = 8
}
return &Builder{id: id, name: name, kind: kind, domains: make(map[string]uint8, sizeHint)}
}
// Add records one domain. The domain must already be normalised: lowercase,
// no trailing dot. A leading "*." is understood as a subdomain wildcard.
func (b *Builder) Add(domain string, matchSubdomains bool) {
domain = strings.TrimSuffix(strings.ToLower(strings.TrimSpace(domain)), ".")
if domain == "" {
return
}
if strings.HasPrefix(domain, "*.") {
domain = domain[2:]
matchSubdomains = true
if domain == "" {
return
}
}
if matchSubdomains {
b.domains[domain] |= flagSuffix | flagExact
} else {
b.domains[domain] |= flagExact
}
}
// Len reports how many distinct domains have been added.
func (b *Builder) Len() int { return len(b.domains) }
// Build freezes the builder into a Set.
func (b *Builder) Build() *Set {
return &Set{ID: b.id, Name: b.name, Kind: b.kind, domains: b.domains}
}
// Len reports the number of domains in the set.
func (s *Set) Len() int {
if s == nil {
return 0
}
return len(s.domains)
}
// Match reports whether name is covered by this list, returning the listed
// domain that matched.
//
// name may be given with or without a trailing dot and in any case.
func (s *Set) Match(name string) (string, bool) {
if s == nil || len(s.domains) == 0 {
return "", false
}
n := normaliseQuery(name)
if n == "" {
return "", false
}
// Exact match on the full name.
if f, ok := s.domains[n]; ok && f&flagExact != 0 {
return n, true
}
// Walk up the parents; each one only matches if it was added as a
// subdomain-covering entry.
rest := n
for {
i := strings.IndexByte(rest, '.')
if i < 0 {
return "", false
}
rest = rest[i+1:]
if rest == "" {
return "", false
}
if f, ok := s.domains[rest]; ok && f&flagSuffix != 0 {
return rest, true
}
}
}
// Contains reports whether the exact domain is present in the list, ignoring
// subdomain coverage. It backs the "is this already in the list?" check.
func (s *Set) Contains(domain string) bool {
if s == nil {
return false
}
_, ok := s.domains[normaliseQuery(domain)]
return ok
}
// normaliseQuery lowercases a query name and removes the trailing dot.
func normaliseQuery(name string) string {
n := strings.TrimSpace(name)
if n == "" {
return ""
}
n = strings.TrimSuffix(n, ".")
// Fast path: most query names are already lowercase.
for i := 0; i < len(n); i++ {
if c := n[i]; c >= 'A' && c <= 'Z' {
return strings.ToLower(n)
}
}
return n
}
+228
View File
@@ -0,0 +1,228 @@
package blacklist
import (
"bufio"
"io"
"net/netip"
"strings"
"github.com/owen/vibedns/internal/models"
"github.com/owen/vibedns/internal/validate"
)
// ParsedDomain is one domain extracted from an import.
type ParsedDomain struct {
Domain string
MatchSubdomains bool
}
// ParseOptions tunes bulk import behaviour.
type ParseOptions struct {
// DefaultMatchSubdomains sets the match mode for entries that do not carry
// explicit wildcard syntax.
DefaultMatchSubdomains bool
// MaxInvalidSamples caps how many rejected lines are reported back.
MaxInvalidSamples int
}
// hostsPlaceholders are the addresses a hosts-file blocklist points at. Lines
// using any other address are still parsed, but these are the common ones.
var localhostNames = map[string]bool{
"localhost": true,
"localhost.localdomain": true,
"local": true,
"ip6-localhost": true,
"ip6-loopback": true,
"ip6-localnet": true,
"ip6-mcastprefix": true,
"ip6-allnodes": true,
"ip6-allrouters": true,
"ip6-allhosts": true,
"broadcasthost": true,
}
// Parse reads a domain list and returns the normalised, de-duplicated domains
// alongside a summary of what happened to every line.
//
// It accepts three shapes, mixed freely in one file:
//
// example.com plain list
// 0.0.0.0 ads.example.com hosts file
// ||tracker.example.net^ Adblock-style host rule
//
// Comments (#, ;, !), blank lines, IP-only lines and localhost entries are
// ignored. Everything that survives is normalised to lowercase without a
// trailing dot.
func Parse(r io.Reader, opts ParseOptions) ([]ParsedDomain, models.ImportSummary) {
if opts.MaxInvalidSamples <= 0 {
opts.MaxInvalidSamples = 10
}
var summary models.ImportSummary
// Pre-size for a large list; growth is amortised anyway.
out := make([]ParsedDomain, 0, 1024)
seen := make(map[string]struct{}, 1024)
sc := bufio.NewScanner(r)
// Blocklist lines are short, but a stray long line must not abort the scan.
sc.Buffer(make([]byte, 0, 64*1024), 1024*1024)
for sc.Scan() {
summary.LinesProcessed++
line := strings.TrimSpace(sc.Text())
if line == "" || isComment(line) {
summary.Ignored++
continue
}
line = stripTrailingComment(line)
if line == "" {
summary.Ignored++
continue
}
candidates, wildcard, ok := extractCandidates(line)
if !ok {
summary.Ignored++
continue
}
if len(candidates) == 0 {
summary.Invalid++
addSample(&summary, line, opts.MaxInvalidSamples)
continue
}
anyValid := false
for _, c := range candidates {
domain, err := validate.NormaliseDomain(c)
if err != nil {
continue
}
if localhostNames[domain] || !strings.Contains(domain, ".") {
// Single-label names are hosts-file noise, not blockable domains.
continue
}
if isIPLiteral(domain) {
continue
}
anyValid = true
if _, dup := seen[domain]; dup {
summary.Duplicates++
continue
}
seen[domain] = struct{}{}
out = append(out, ParsedDomain{
Domain: domain,
MatchSubdomains: wildcard || opts.DefaultMatchSubdomains,
})
summary.Imported++
}
if !anyValid {
summary.Invalid++
addSample(&summary, line, opts.MaxInvalidSamples)
}
}
if err := sc.Err(); err != nil {
// A read failure still returns what was parsed so far; the caller
// decides whether a partial import is acceptable.
return out, summary
}
return out, summary
}
func addSample(s *models.ImportSummary, line string, max int) {
if len(s.InvalidSamples) >= max {
return
}
if len(line) > 120 {
line = line[:120] + "..."
}
s.InvalidSamples = append(s.InvalidSamples, line)
}
func isComment(line string) bool {
switch line[0] {
case '#', ';':
return true
case '!':
// Adblock comment, but "!" never starts a host rule.
return true
case '[':
// Adblock header such as [Adblock Plus 2.0]
return true
}
return false
}
// stripTrailingComment removes an inline comment while leaving the rest.
func stripTrailingComment(line string) string {
if i := strings.IndexAny(line, "#;"); i >= 0 {
line = line[:i]
}
return strings.TrimSpace(line)
}
// extractCandidates pulls the domain-shaped tokens out of one line.
//
// The bool return reports whether the line should count as "ignored" rather
// than "invalid" — used for rules this importer deliberately does not support,
// such as Adblock element-hiding or exception rules.
func extractCandidates(line string) (domains []string, wildcard bool, supported bool) {
// Adblock-style rules.
if strings.HasPrefix(line, "@@") {
return nil, false, false // exception rule: not a block entry
}
if strings.HasPrefix(line, "||") {
rest := strings.TrimPrefix(line, "||")
rest = strings.TrimSuffix(rest, "^")
rest = strings.TrimSuffix(rest, "^$all")
if i := strings.IndexAny(rest, "/^$*"); i >= 0 {
// A path or option makes this a URL rule, which DNS cannot express.
if i == 0 {
return nil, false, false
}
rest = rest[:i]
}
return []string{rest}, true, true
}
if strings.ContainsAny(line, "/$") && !strings.HasPrefix(line, "0.0.0.0") {
// dnsmasq address=/example.com/0.0.0.0
if strings.HasPrefix(line, "address=/") || strings.HasPrefix(line, "server=/") {
parts := strings.Split(line, "/")
if len(parts) >= 2 && parts[1] != "" {
return []string{parts[1]}, true, true
}
return nil, false, false
}
return nil, false, false
}
fields := strings.Fields(line)
if len(fields) == 0 {
return nil, false, false
}
// Hosts-file syntax: the first field is an IP address, the rest are names.
if isIPLiteral(fields[0]) {
if len(fields) == 1 {
return nil, false, false // an address on its own carries no domain
}
return fields[1:], false, true
}
// Plain list: one domain per line. Extra fields are treated as noise.
first := fields[0]
if strings.HasPrefix(first, "*.") {
return []string{strings.TrimPrefix(first, "*.")}, true, true
}
return []string{first}, false, true
}
func isIPLiteral(s string) bool {
_, err := netip.ParseAddr(s)
return err == nil
}
// ParseString is a convenience wrapper for textarea input.
func ParseString(s string, opts ParseOptions) ([]ParsedDomain, models.ImportSummary) {
return Parse(strings.NewReader(s), opts)
}