package search import ( "net/url" "strings" ) // CanonicalURL lowercases scheme+host, strips default ports and fragments, // and drops a trailing slash on the path. www. is left as-is. func CanonicalURL(raw string) string { raw = strings.TrimSpace(raw) if raw == "" { return "" } u, err := url.Parse(raw) if err != nil || u.Host == "" && u.Opaque == "" { return strings.TrimRight(raw, "/") } u.Fragment = "" u.Scheme = strings.ToLower(u.Scheme) host := strings.ToLower(u.Hostname()) port := u.Port() switch { case port == "80" && u.Scheme == "http": port = "" case port == "443" && u.Scheme == "https": port = "" } if port != "" { u.Host = host + ":" + port } else { u.Host = host } if u.Path == "/" { u.Path = "" } else if len(u.Path) > 1 && strings.HasSuffix(u.Path, "/") { u.Path = strings.TrimSuffix(u.Path, "/") } return u.String() }