done
This commit is contained in:
Dev
269
internal/mirror/fetcher.go
Normal file
269
internal/mirror/fetcher.go
Normal file
@@ -0,0 +1,269 @@
|
||||
package mirror
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// MirrorFetcher handles fetching mirror lists from official sources
|
||||
type MirrorFetcher struct {
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
// NewMirrorFetcher creates a new mirror fetcher
|
||||
func NewMirrorFetcher() *MirrorFetcher {
|
||||
return &MirrorFetcher{
|
||||
client: &http.Client{
|
||||
Timeout: 30 * time.Second,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// FetchMirrors gets the complete list of mirrors for a distribution family
|
||||
func (mf *MirrorFetcher) FetchMirrors(ctx context.Context, family string) ([]string, error) {
|
||||
switch family {
|
||||
case "arch":
|
||||
return mf.fetchArchMirrors(ctx)
|
||||
case "debian":
|
||||
return mf.fetchDebianMirrors(ctx)
|
||||
case "fedora":
|
||||
return mf.fetchFedoraMirrors(ctx)
|
||||
default:
|
||||
// Fallback to hardcoded mirrors for unsupported families
|
||||
return mf.getDefaultMirrors(family), nil
|
||||
}
|
||||
}
|
||||
|
||||
// fetchArchMirrors fetches Arch Linux mirrors from the official mirror status
|
||||
func (mf *MirrorFetcher) fetchArchMirrors(ctx context.Context) ([]string, error) {
|
||||
// Arch Linux provides a JSON API for mirror status
|
||||
url := "https://archlinux.org/mirrorlist/?protocol=https&use_mirror_status=on"
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create request: %w", err)
|
||||
}
|
||||
|
||||
resp, err := mf.client.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to fetch Arch mirrors: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("HTTP %d from Arch mirror API", resp.StatusCode)
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read response: %w", err)
|
||||
}
|
||||
|
||||
return mf.parseArchMirrorList(string(body))
|
||||
}
|
||||
|
||||
// parseArchMirrorList parses the Arch Linux mirror list format
|
||||
func (mf *MirrorFetcher) parseArchMirrorList(content string) ([]string, error) {
|
||||
var mirrors []string
|
||||
scanner := bufio.NewScanner(strings.NewReader(content))
|
||||
|
||||
// Look for lines like: #Server = https://mirror.example.com/archlinux/$repo/os/$arch
|
||||
serverRegex := regexp.MustCompile(`^#?Server\s*=\s*(.+)`)
|
||||
|
||||
for scanner.Scan() {
|
||||
line := strings.TrimSpace(scanner.Text())
|
||||
if matches := serverRegex.FindStringSubmatch(line); matches != nil {
|
||||
mirrorURL := strings.TrimSpace(matches[1])
|
||||
// Remove the $repo/os/$arch part for base URL
|
||||
if strings.Contains(mirrorURL, "$repo") {
|
||||
mirrorURL = strings.Replace(mirrorURL, "/$repo/os/$arch", "/", 1)
|
||||
}
|
||||
if mirrorURL != "" && !strings.Contains(mirrorURL, "$") {
|
||||
mirrors = append(mirrors, mirrorURL)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(mirrors) == 0 {
|
||||
return nil, fmt.Errorf("no mirrors found in Arch Linux mirror list")
|
||||
}
|
||||
|
||||
return mirrors, nil
|
||||
}
|
||||
|
||||
// fetchDebianMirrors fetches Debian mirrors from the official mirror list
|
||||
func (mf *MirrorFetcher) fetchDebianMirrors(ctx context.Context) ([]string, error) {
|
||||
// Debian maintains a list of mirrors
|
||||
url := "https://www.debian.org/mirror/list-full"
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create request: %w", err)
|
||||
}
|
||||
|
||||
resp, err := mf.client.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to fetch Debian mirrors: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("HTTP %d from Debian mirror list", resp.StatusCode)
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read response: %w", err)
|
||||
}
|
||||
|
||||
return mf.parseDebianMirrorList(string(body))
|
||||
}
|
||||
|
||||
// parseDebianMirrorList parses the Debian mirror list HTML
|
||||
func (mf *MirrorFetcher) parseDebianMirrorList(content string) ([]string, error) {
|
||||
var mirrors []string
|
||||
|
||||
// Look for HTTP/HTTPS URLs in the HTML
|
||||
urlRegex := regexp.MustCompile(`https?://[a-zA-Z0-9.-]+[a-zA-Z0-9.-/]*debian[a-zA-Z0-9.-/]*/?`)
|
||||
matches := urlRegex.FindAllString(content, -1)
|
||||
|
||||
seen := make(map[string]bool)
|
||||
for _, match := range matches {
|
||||
// Clean up and normalize the URL
|
||||
mirror := strings.TrimSpace(match)
|
||||
mirror = strings.TrimSuffix(mirror, "/") + "/"
|
||||
|
||||
// Avoid duplicates and ensure it's a proper mirror
|
||||
if !seen[mirror] && strings.Contains(mirror, "debian") {
|
||||
mirrors = append(mirrors, mirror)
|
||||
seen[mirror] = true
|
||||
}
|
||||
}
|
||||
|
||||
// Add some known good mirrors if we don't find enough
|
||||
knownMirrors := []string{
|
||||
"http://deb.debian.org/debian/",
|
||||
"http://ftp.us.debian.org/debian/",
|
||||
"http://ftp.uk.debian.org/debian/",
|
||||
"http://ftp.de.debian.org/debian/",
|
||||
"http://mirrors.kernel.org/debian/",
|
||||
}
|
||||
|
||||
for _, known := range knownMirrors {
|
||||
if !seen[known] {
|
||||
mirrors = append(mirrors, known)
|
||||
seen[known] = true
|
||||
}
|
||||
}
|
||||
|
||||
if len(mirrors) == 0 {
|
||||
return nil, fmt.Errorf("no mirrors found in Debian mirror list")
|
||||
}
|
||||
|
||||
return mirrors, nil
|
||||
}
|
||||
|
||||
// fetchFedoraMirrors fetches Fedora mirrors from the official mirror list
|
||||
func (mf *MirrorFetcher) fetchFedoraMirrors(ctx context.Context) ([]string, error) {
|
||||
// Fedora mirror list URL
|
||||
url := "https://mirrors.fedoraproject.org/mirrorlist?repo=fedora-39&arch=x86_64"
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create request: %w", err)
|
||||
}
|
||||
|
||||
resp, err := mf.client.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to fetch Fedora mirrors: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("HTTP %d from Fedora mirror list", resp.StatusCode)
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read response: %w", err)
|
||||
}
|
||||
|
||||
return mf.parseFedoraMirrorList(string(body))
|
||||
}
|
||||
|
||||
// parseFedoraMirrorList parses the Fedora mirror list format
|
||||
func (mf *MirrorFetcher) parseFedoraMirrorList(content string) ([]string, error) {
|
||||
var mirrors []string
|
||||
scanner := bufio.NewScanner(strings.NewReader(content))
|
||||
|
||||
for scanner.Scan() {
|
||||
line := strings.TrimSpace(scanner.Text())
|
||||
if strings.HasPrefix(line, "http") {
|
||||
// Extract base URL (remove specific paths)
|
||||
if idx := strings.Index(line, "/linux/"); idx != -1 {
|
||||
baseURL := line[:idx] + "/fedora/linux/"
|
||||
mirrors = append(mirrors, baseURL)
|
||||
} else {
|
||||
mirrors = append(mirrors, line)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(mirrors) == 0 {
|
||||
return nil, fmt.Errorf("no mirrors found in Fedora mirror list")
|
||||
}
|
||||
|
||||
return mirrors, nil
|
||||
}
|
||||
|
||||
// getDefaultMirrors returns hardcoded mirrors as fallback
|
||||
func (mf *MirrorFetcher) getDefaultMirrors(family string) []string {
|
||||
defaultMirrors := map[string][]string{
|
||||
"debian": {
|
||||
"http://deb.debian.org/debian/",
|
||||
"http://ftp.us.debian.org/debian/",
|
||||
"http://ftp.uk.debian.org/debian/",
|
||||
"http://ftp.de.debian.org/debian/",
|
||||
"http://mirrors.kernel.org/debian/",
|
||||
},
|
||||
"arch": {
|
||||
"https://mirror.rackspace.com/archlinux/",
|
||||
"https://mirrors.kernel.org/archlinux/",
|
||||
"https://mirror.math.princeton.edu/pub/archlinux/",
|
||||
"https://mirrors.mit.edu/archlinux/",
|
||||
"https://mirrors.liquidweb.com/archlinux/",
|
||||
},
|
||||
"fedora": {
|
||||
"https://download.fedoraproject.org/pub/fedora/linux/",
|
||||
"https://mirrors.kernel.org/fedora/",
|
||||
"https://mirror.math.princeton.edu/pub/fedora/linux/",
|
||||
"https://mirrors.mit.edu/fedora/linux/",
|
||||
},
|
||||
"opensuse": {
|
||||
"http://download.opensuse.org/distribution/",
|
||||
"http://ftp.gwdg.de/pub/linux/opensuse/distribution/",
|
||||
"http://ftp.halifax.rwth-aachen.de/opensuse/distribution/",
|
||||
},
|
||||
"gentoo": {
|
||||
"https://gentoo.osuosl.org/",
|
||||
"http://mirrors.kernel.org/gentoo/",
|
||||
"https://mirror.bytemark.co.uk/gentoo/",
|
||||
},
|
||||
"alpine": {
|
||||
"http://dl-cdn.alpinelinux.org/alpine/",
|
||||
"http://mirrors.dotsrc.org/alpine/",
|
||||
"http://mirror.fit.cvut.cz/alpine/",
|
||||
},
|
||||
}
|
||||
|
||||
if mirrors, exists := defaultMirrors[family]; exists {
|
||||
return mirrors
|
||||
}
|
||||
return []string{}
|
||||
}
|
||||
Reference in New Issue
Block a user