fastestmirror/internal/mirror/fetcher.go

package mirror

import (
	"bufio"
	"context"
	"fmt"
	"io"
	"net/http"
	"regexp"
	"strings"
	"time"
)

// MirrorFetcher handles fetching mirror lists from official sources
type MirrorFetcher struct {
	client *http.Client
}

// NewMirrorFetcher creates a new mirror fetcher
func NewMirrorFetcher() *MirrorFetcher {
	return &MirrorFetcher{
		client: &http.Client{
			Timeout: 30 * time.Second,
		},
	}
}

// FetchMirrors gets the complete list of mirrors for a distribution family
func (mf *MirrorFetcher) FetchMirrors(ctx context.Context, family string) ([]string, error) {
	switch family {
	case "arch":
		return mf.fetchArchMirrors(ctx)
	case "debian":
		return mf.fetchDebianMirrors(ctx)
	case "fedora":
		return mf.fetchFedoraMirrors(ctx)
	default:
		// Fallback to hardcoded mirrors for unsupported families
		return mf.getDefaultMirrors(family), nil
	}
}

// fetchArchMirrors fetches Arch Linux mirrors from the official mirror status
func (mf *MirrorFetcher) fetchArchMirrors(ctx context.Context) ([]string, error) {
	// Arch Linux provides a JSON API for mirror status
	url := "https://archlinux.org/mirrorlist/?protocol=https&use_mirror_status=on"

	req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
	if err != nil {
		return nil, fmt.Errorf("failed to create request: %w", err)
	}

	resp, err := mf.client.Do(req)
	if err != nil {
		return nil, fmt.Errorf("failed to fetch Arch mirrors: %w", err)
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("HTTP %d from Arch mirror API", resp.StatusCode)
	}

	body, err := io.ReadAll(resp.Body)
	if err != nil {
		return nil, fmt.Errorf("failed to read response: %w", err)
	}

	return mf.parseArchMirrorList(string(body))
}

// parseArchMirrorList parses the Arch Linux mirror list format
func (mf *MirrorFetcher) parseArchMirrorList(content string) ([]string, error) {
	var mirrors []string
	scanner := bufio.NewScanner(strings.NewReader(content))

	// Look for lines like: #Server = https://mirror.example.com/archlinux/$repo/os/$arch
	serverRegex := regexp.MustCompile(`^#?Server\s*=\s*(.+)`)

	for scanner.Scan() {
		line := strings.TrimSpace(scanner.Text())
		if matches := serverRegex.FindStringSubmatch(line); matches != nil {
			mirrorURL := strings.TrimSpace(matches[1])
			// Remove the $repo/os/$arch part for base URL
			if strings.Contains(mirrorURL, "$repo") {
				mirrorURL = strings.Replace(mirrorURL, "/$repo/os/$arch", "/", 1)
			}
			if mirrorURL != "" && !strings.Contains(mirrorURL, "$") {
				mirrors = append(mirrors, mirrorURL)
			}
		}
	}

	if len(mirrors) == 0 {
		return nil, fmt.Errorf("no mirrors found in Arch Linux mirror list")
	}

	return mirrors, nil
}

// fetchDebianMirrors fetches Debian mirrors from the official mirror list
func (mf *MirrorFetcher) fetchDebianMirrors(ctx context.Context) ([]string, error) {
	// Debian maintains a list of mirrors
	url := "https://www.debian.org/mirror/list-full"

	req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
	if err != nil {
		return nil, fmt.Errorf("failed to create request: %w", err)
	}

	resp, err := mf.client.Do(req)
	if err != nil {
		return nil, fmt.Errorf("failed to fetch Debian mirrors: %w", err)
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("HTTP %d from Debian mirror list", resp.StatusCode)
	}

	body, err := io.ReadAll(resp.Body)
	if err != nil {
		return nil, fmt.Errorf("failed to read response: %w", err)
	}

	return mf.parseDebianMirrorList(string(body))
}

// parseDebianMirrorList parses the Debian mirror list HTML
func (mf *MirrorFetcher) parseDebianMirrorList(content string) ([]string, error) {
	var mirrors []string

	// Look for HTTP/HTTPS URLs in the HTML
	urlRegex := regexp.MustCompile(`https?://[a-zA-Z0-9.-]+[a-zA-Z0-9.-/]*debian[a-zA-Z0-9.-/]*/?`)
	matches := urlRegex.FindAllString(content, -1)

	seen := make(map[string]bool)
	for _, match := range matches {
		// Clean up and normalize the URL
		mirror := strings.TrimSpace(match)
		mirror = strings.TrimSuffix(mirror, "/") + "/"

		// Avoid duplicates and ensure it's a proper mirror
		if !seen[mirror] && strings.Contains(mirror, "debian") {
			mirrors = append(mirrors, mirror)
			seen[mirror] = true
		}
	}

	// Add some known good mirrors if we don't find enough
	knownMirrors := []string{
		"http://deb.debian.org/debian/",
		"http://ftp.us.debian.org/debian/",
		"http://ftp.uk.debian.org/debian/",
		"http://ftp.de.debian.org/debian/",
		"http://mirrors.kernel.org/debian/",
	}

	for _, known := range knownMirrors {
		if !seen[known] {
			mirrors = append(mirrors, known)
			seen[known] = true
		}
	}

	if len(mirrors) == 0 {
		return nil, fmt.Errorf("no mirrors found in Debian mirror list")
	}

	return mirrors, nil
}

// fetchFedoraMirrors fetches Fedora mirrors from the official mirror list
func (mf *MirrorFetcher) fetchFedoraMirrors(ctx context.Context) ([]string, error) {
	// Fedora mirror list URL
	url := "https://mirrors.fedoraproject.org/mirrorlist?repo=fedora-39&arch=x86_64"

	req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
	if err != nil {
		return nil, fmt.Errorf("failed to create request: %w", err)
	}

	resp, err := mf.client.Do(req)
	if err != nil {
		return nil, fmt.Errorf("failed to fetch Fedora mirrors: %w", err)
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("HTTP %d from Fedora mirror list", resp.StatusCode)
	}

	body, err := io.ReadAll(resp.Body)
	if err != nil {
		return nil, fmt.Errorf("failed to read response: %w", err)
	}

	return mf.parseFedoraMirrorList(string(body))
}

// parseFedoraMirrorList parses the Fedora mirror list format
func (mf *MirrorFetcher) parseFedoraMirrorList(content string) ([]string, error) {
	var mirrors []string
	scanner := bufio.NewScanner(strings.NewReader(content))

	for scanner.Scan() {
		line := strings.TrimSpace(scanner.Text())
		if strings.HasPrefix(line, "http") {
			// Extract base URL (remove specific paths)
			if idx := strings.Index(line, "/linux/"); idx != -1 {
				baseURL := line[:idx] + "/fedora/linux/"
				mirrors = append(mirrors, baseURL)
			} else {
				mirrors = append(mirrors, line)
			}
		}
	}

	if len(mirrors) == 0 {
		return nil, fmt.Errorf("no mirrors found in Fedora mirror list")
	}

	return mirrors, nil
}

// getDefaultMirrors returns hardcoded mirrors as fallback
func (mf *MirrorFetcher) getDefaultMirrors(family string) []string {
	defaultMirrors := map[string][]string{
		"debian": {
			"http://deb.debian.org/debian/",
			"http://ftp.us.debian.org/debian/",
			"http://ftp.uk.debian.org/debian/",
			"http://ftp.de.debian.org/debian/",
			"http://mirrors.kernel.org/debian/",
		},
		"arch": {
			"https://mirror.rackspace.com/archlinux/",
			"https://mirrors.kernel.org/archlinux/",
			"https://mirror.math.princeton.edu/pub/archlinux/",
			"https://mirrors.mit.edu/archlinux/",
			"https://mirrors.liquidweb.com/archlinux/",
		},
		"fedora": {
			"https://download.fedoraproject.org/pub/fedora/linux/",
			"https://mirrors.kernel.org/fedora/",
			"https://mirror.math.princeton.edu/pub/fedora/linux/",
			"https://mirrors.mit.edu/fedora/linux/",
		},
		"opensuse": {
			"http://download.opensuse.org/distribution/",
			"http://ftp.gwdg.de/pub/linux/opensuse/distribution/",
			"http://ftp.halifax.rwth-aachen.de/opensuse/distribution/",
		},
		"gentoo": {
			"https://gentoo.osuosl.org/",
			"http://mirrors.kernel.org/gentoo/",
			"https://mirror.bytemark.co.uk/gentoo/",
		},
		"alpine": {
			"http://dl-cdn.alpinelinux.org/alpine/",
			"http://mirrors.dotsrc.org/alpine/",
			"http://mirror.fit.cvut.cz/alpine/",
		},
	}

	if mirrors, exists := defaultMirrors[family]; exists {
		return mirrors
	}
	return []string{}
}