You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
50 lines
1.0 KiB
Go
50 lines
1.0 KiB
Go
package spider
|
|
|
|
import (
|
|
"errors"
|
|
"github.com/snakem982/proxypool/tools"
|
|
"golang.org/x/net/html"
|
|
"strings"
|
|
"sync"
|
|
)
|
|
|
|
type Getter struct {
|
|
Id string `json:"id,omitempty" yaml:"id,omitempty"`
|
|
Type string `json:"type" yaml:"type"`
|
|
Url string `json:"url" yaml:"url"`
|
|
}
|
|
|
|
type Collect interface {
|
|
Get() []map[string]any
|
|
Get2ChanWG(pc chan []map[string]any, wg *sync.WaitGroup)
|
|
}
|
|
|
|
type collector func(getter Getter) Collect
|
|
|
|
var collectorMap = make(map[string]collector)
|
|
|
|
func Register(sourceType string, c collector) {
|
|
collectorMap[sourceType] = c
|
|
}
|
|
|
|
var ErrorCreateNotSupported = errors.New("type not supported")
|
|
|
|
func NewCollect(sourceType string, getter Getter) (Collect, error) {
|
|
if c, ok := collectorMap[sourceType]; ok {
|
|
return c(getter), nil
|
|
}
|
|
|
|
return nil, ErrorCreateNotSupported
|
|
}
|
|
|
|
func GetBytes(url string) []byte {
|
|
all := tools.ConcurrentHttpGet(url)
|
|
if all != nil {
|
|
temp := html.UnescapeString(string(all))
|
|
temp = strings.Replace(temp, "\"HOST\"", "\"Host\"", -1)
|
|
all = []byte(temp)
|
|
}
|
|
|
|
return all
|
|
}
|