GooGet/scraper/main.go

package main

import (
	"bufio"
	"fmt"
	"log"
	"net"
	"strings"
	"sync"
	"time"
)

// Function to scan a single port on a single IP address
func scanPort(ip string, port int, results chan<- string) {
	address := fmt.Sprintf("%s:%d", ip, port)
	conn, err := net.DialTimeout("tcp", address, 1*time.Second)
	if err != nil {
		results <- fmt.Sprintf("Closed: %s:%d", ip, port)
		return
	}
	conn.Close()
	results <- fmt.Sprintf("Open: %s:%d", ip, port)
}

// Function to parse the IP range and perform port scanning
func scanRange(startIP, endIP string, ports []int, results chan<- string, wg *sync.WaitGroup) {
	defer wg.Done()
	start := net.ParseIP(startIP)
	end := net.ParseIP(endIP)
	for ip := start; !ip.Equal(end); ip = nextIP(ip) {
		for _, port := range ports {
			go scanPort(ip.String(), port, results)
		}
	}
}

func nextIP(ip net.IP) net.IP {
	ip = ip.To4()
	for j := len(ip) - 1; j >= 0; j-- {
		ip[j]++
		if ip[j] != 0 {
			break
		}
	}
	return ip
}

// Function to process scan results
func processResults(results chan string, wg *sync.WaitGroup, scannedCount *int, totalCount int, progressCh chan<- int) {
	defer wg.Done()
	for result := range results {
		if strings.HasPrefix(result, "Open") {
			fmt.Println(result)
			parts := strings.Split(result, ":")
			ip := parts[1]
			port := parts[2]
			url := fmt.Sprintf("http://%s:%s", ip, port)
			data, err := scrapeData(strings.Replace(url, " ", "", -1))
			fmt.Println(data)
			if err == nil {
				err = postData(map[string]interface{}{"url": url, "keywords": data})
				if err != nil {
					log.Printf("Error posting data for %s: %v", url, err)
				}
			} else {
				log.Printf("Error scraping data for %s: %v", url, err)
			}
		}
		*scannedCount++
		if *scannedCount%100 == 0 || *scannedCount == totalCount {
			progressCh <- (*scannedCount * 100) / totalCount
		}
	}
}

func main() {
	//https://lite.ip2location.com
	ipRanges := `
96.125.176.0	96.125.177.255	512
96.125.180.0	96.125.223.255	11,264
96.125.240.0	96.125.255.255	4,096
96.126.70.0	96.126.70.255	256
96.127.192.0	96.127.255.255	16,384
96.20.0.0	96.23.255.255	262,144
96.30.128.0	96.30.191.255	16,384
96.43.224.0	96.43.239.255	4,096
96.44.192.0	96.45.15.255	20,480
96.44.32.0	96.44.127.255	24,576
96.45.192.0	96.45.207.255	4,096
96.45.43.0	96.45.43.255	256
96.45.68.0	96.45.68.255	256
96.46.192.0	96.46.207.255	4,096
96.46.32.0	96.46.63.255	8,192
96.47.176.0	96.47.191.255	4,096
96.47.74.0	96.47.74.255	256
96.48.0.0	96.55.255.255	524,288
96.6.145.0	96.6.145.255	256
96.6.152.0	96.6.157.255	1,536
96.63.0.0	96.63.63.255	16,384
96.63.128.0	96.63.175.255	12,288
96.7.231.0	96.7.231.255	256
96.7.68.0	96.7.71.255	1,024
96.9.226.0	96.9.226.255	256
96.9.96.0	96.9.127.255	8,192
97.107.176.0	97.107.191.255	4,096
`

	// Define the ports to be scanned
	ports := []int{80, 8080, 8880}
	results := make(chan string)
	progressCh := make(chan int)

	var totalCount, scannedCount int
	var wg sync.WaitGroup

	// Calculate total count of IP addresses
	scanner := bufio.NewScanner(strings.NewReader(ipRanges))
	for scanner.Scan() {
		line := scanner.Text()
		parts := strings.Fields(line)
		if len(parts) >= 2 {
			_, endIP := parts[0], parts[1]
			totalCount += countIPs(parts[0], endIP)
		}
	}

	// Start processing results
	wg.Add(1)
	go processResults(results, &wg, &scannedCount, totalCount, progressCh)

	// Parse IP ranges and start scanning
	scanner = bufio.NewScanner(strings.NewReader(ipRanges))
	for scanner.Scan() {
		line := scanner.Text()
		parts := strings.Fields(line)
		if len(parts) >= 2 {
			startIP := parts[0]
			endIP := parts[1]
			wg.Add(1)
			go scanRange(startIP, endIP, ports, results, &wg)
		}
	}

	// Progress reporting
	go func() {
		for progress := range progressCh {
			fmt.Printf("Progress: %d%%\n", progress)
		}
	}()

	// Allow some time for scans to complete
	wg.Wait()
	close(results)
	close(progressCh)
}

func countIPs(startIP, endIP string) int {
	start := net.ParseIP(startIP)
	end := net.ParseIP(endIP)
	count := 0
	for ip := start; !ip.Equal(end); ip = nextIP(ip) {
		count++
	}
	return count + 1 // Include endIP
}