initial commit
This commit is contained in:
@@ -0,0 +1,228 @@
|
||||
package blacklist
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"io"
|
||||
"net/netip"
|
||||
"strings"
|
||||
|
||||
"github.com/owen/vibedns/internal/models"
|
||||
"github.com/owen/vibedns/internal/validate"
|
||||
)
|
||||
|
||||
// ParsedDomain is one domain extracted from an import.
|
||||
type ParsedDomain struct {
|
||||
Domain string
|
||||
MatchSubdomains bool
|
||||
}
|
||||
|
||||
// ParseOptions tunes bulk import behaviour.
|
||||
type ParseOptions struct {
|
||||
// DefaultMatchSubdomains sets the match mode for entries that do not carry
|
||||
// explicit wildcard syntax.
|
||||
DefaultMatchSubdomains bool
|
||||
// MaxInvalidSamples caps how many rejected lines are reported back.
|
||||
MaxInvalidSamples int
|
||||
}
|
||||
|
||||
// hostsPlaceholders are the addresses a hosts-file blocklist points at. Lines
|
||||
// using any other address are still parsed, but these are the common ones.
|
||||
var localhostNames = map[string]bool{
|
||||
"localhost": true,
|
||||
"localhost.localdomain": true,
|
||||
"local": true,
|
||||
"ip6-localhost": true,
|
||||
"ip6-loopback": true,
|
||||
"ip6-localnet": true,
|
||||
"ip6-mcastprefix": true,
|
||||
"ip6-allnodes": true,
|
||||
"ip6-allrouters": true,
|
||||
"ip6-allhosts": true,
|
||||
"broadcasthost": true,
|
||||
}
|
||||
|
||||
// Parse reads a domain list and returns the normalised, de-duplicated domains
|
||||
// alongside a summary of what happened to every line.
|
||||
//
|
||||
// It accepts three shapes, mixed freely in one file:
|
||||
//
|
||||
// example.com plain list
|
||||
// 0.0.0.0 ads.example.com hosts file
|
||||
// ||tracker.example.net^ Adblock-style host rule
|
||||
//
|
||||
// Comments (#, ;, !), blank lines, IP-only lines and localhost entries are
|
||||
// ignored. Everything that survives is normalised to lowercase without a
|
||||
// trailing dot.
|
||||
func Parse(r io.Reader, opts ParseOptions) ([]ParsedDomain, models.ImportSummary) {
|
||||
if opts.MaxInvalidSamples <= 0 {
|
||||
opts.MaxInvalidSamples = 10
|
||||
}
|
||||
var summary models.ImportSummary
|
||||
// Pre-size for a large list; growth is amortised anyway.
|
||||
out := make([]ParsedDomain, 0, 1024)
|
||||
seen := make(map[string]struct{}, 1024)
|
||||
|
||||
sc := bufio.NewScanner(r)
|
||||
// Blocklist lines are short, but a stray long line must not abort the scan.
|
||||
sc.Buffer(make([]byte, 0, 64*1024), 1024*1024)
|
||||
|
||||
for sc.Scan() {
|
||||
summary.LinesProcessed++
|
||||
line := strings.TrimSpace(sc.Text())
|
||||
|
||||
if line == "" || isComment(line) {
|
||||
summary.Ignored++
|
||||
continue
|
||||
}
|
||||
line = stripTrailingComment(line)
|
||||
if line == "" {
|
||||
summary.Ignored++
|
||||
continue
|
||||
}
|
||||
|
||||
candidates, wildcard, ok := extractCandidates(line)
|
||||
if !ok {
|
||||
summary.Ignored++
|
||||
continue
|
||||
}
|
||||
if len(candidates) == 0 {
|
||||
summary.Invalid++
|
||||
addSample(&summary, line, opts.MaxInvalidSamples)
|
||||
continue
|
||||
}
|
||||
|
||||
anyValid := false
|
||||
for _, c := range candidates {
|
||||
domain, err := validate.NormaliseDomain(c)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if localhostNames[domain] || !strings.Contains(domain, ".") {
|
||||
// Single-label names are hosts-file noise, not blockable domains.
|
||||
continue
|
||||
}
|
||||
if isIPLiteral(domain) {
|
||||
continue
|
||||
}
|
||||
anyValid = true
|
||||
if _, dup := seen[domain]; dup {
|
||||
summary.Duplicates++
|
||||
continue
|
||||
}
|
||||
seen[domain] = struct{}{}
|
||||
out = append(out, ParsedDomain{
|
||||
Domain: domain,
|
||||
MatchSubdomains: wildcard || opts.DefaultMatchSubdomains,
|
||||
})
|
||||
summary.Imported++
|
||||
}
|
||||
if !anyValid {
|
||||
summary.Invalid++
|
||||
addSample(&summary, line, opts.MaxInvalidSamples)
|
||||
}
|
||||
}
|
||||
if err := sc.Err(); err != nil {
|
||||
// A read failure still returns what was parsed so far; the caller
|
||||
// decides whether a partial import is acceptable.
|
||||
return out, summary
|
||||
}
|
||||
return out, summary
|
||||
}
|
||||
|
||||
func addSample(s *models.ImportSummary, line string, max int) {
|
||||
if len(s.InvalidSamples) >= max {
|
||||
return
|
||||
}
|
||||
if len(line) > 120 {
|
||||
line = line[:120] + "..."
|
||||
}
|
||||
s.InvalidSamples = append(s.InvalidSamples, line)
|
||||
}
|
||||
|
||||
func isComment(line string) bool {
|
||||
switch line[0] {
|
||||
case '#', ';':
|
||||
return true
|
||||
case '!':
|
||||
// Adblock comment, but "!" never starts a host rule.
|
||||
return true
|
||||
case '[':
|
||||
// Adblock header such as [Adblock Plus 2.0]
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// stripTrailingComment removes an inline comment while leaving the rest.
|
||||
func stripTrailingComment(line string) string {
|
||||
if i := strings.IndexAny(line, "#;"); i >= 0 {
|
||||
line = line[:i]
|
||||
}
|
||||
return strings.TrimSpace(line)
|
||||
}
|
||||
|
||||
// extractCandidates pulls the domain-shaped tokens out of one line.
|
||||
//
|
||||
// The bool return reports whether the line should count as "ignored" rather
|
||||
// than "invalid" — used for rules this importer deliberately does not support,
|
||||
// such as Adblock element-hiding or exception rules.
|
||||
func extractCandidates(line string) (domains []string, wildcard bool, supported bool) {
|
||||
// Adblock-style rules.
|
||||
if strings.HasPrefix(line, "@@") {
|
||||
return nil, false, false // exception rule: not a block entry
|
||||
}
|
||||
if strings.HasPrefix(line, "||") {
|
||||
rest := strings.TrimPrefix(line, "||")
|
||||
rest = strings.TrimSuffix(rest, "^")
|
||||
rest = strings.TrimSuffix(rest, "^$all")
|
||||
if i := strings.IndexAny(rest, "/^$*"); i >= 0 {
|
||||
// A path or option makes this a URL rule, which DNS cannot express.
|
||||
if i == 0 {
|
||||
return nil, false, false
|
||||
}
|
||||
rest = rest[:i]
|
||||
}
|
||||
return []string{rest}, true, true
|
||||
}
|
||||
if strings.ContainsAny(line, "/$") && !strings.HasPrefix(line, "0.0.0.0") {
|
||||
// dnsmasq address=/example.com/0.0.0.0
|
||||
if strings.HasPrefix(line, "address=/") || strings.HasPrefix(line, "server=/") {
|
||||
parts := strings.Split(line, "/")
|
||||
if len(parts) >= 2 && parts[1] != "" {
|
||||
return []string{parts[1]}, true, true
|
||||
}
|
||||
return nil, false, false
|
||||
}
|
||||
return nil, false, false
|
||||
}
|
||||
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) == 0 {
|
||||
return nil, false, false
|
||||
}
|
||||
|
||||
// Hosts-file syntax: the first field is an IP address, the rest are names.
|
||||
if isIPLiteral(fields[0]) {
|
||||
if len(fields) == 1 {
|
||||
return nil, false, false // an address on its own carries no domain
|
||||
}
|
||||
return fields[1:], false, true
|
||||
}
|
||||
|
||||
// Plain list: one domain per line. Extra fields are treated as noise.
|
||||
first := fields[0]
|
||||
if strings.HasPrefix(first, "*.") {
|
||||
return []string{strings.TrimPrefix(first, "*.")}, true, true
|
||||
}
|
||||
return []string{first}, false, true
|
||||
}
|
||||
|
||||
func isIPLiteral(s string) bool {
|
||||
_, err := netip.ParseAddr(s)
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// ParseString is a convenience wrapper for textarea input.
|
||||
func ParseString(s string, opts ParseOptions) ([]ParsedDomain, models.ImportSummary) {
|
||||
return Parse(strings.NewReader(s), opts)
|
||||
}
|
||||
Reference in New Issue
Block a user