mirror of
https://github.com/hashiromer/Upwork-Jobs-scraper-.git
synced 2025-12-29 16:16:01 +00:00
107 lines
2.4 KiB
Go
107 lines
2.4 KiB
Go
package upwork
|
|
|
|
import (
|
|
"bufio"
|
|
"fmt"
|
|
"log"
|
|
"os"
|
|
"scrapers/network"
|
|
"strings"
|
|
)
|
|
|
|
type Upwork struct {
|
|
UpworkHttpClient *network.Client
|
|
}
|
|
|
|
type UrlArgs struct {
|
|
Page int
|
|
Per_Page int
|
|
Query string
|
|
}
|
|
|
|
func (u Upwork) ConstructUrl(args UrlArgs) string {
|
|
url := "https://www.upwork.com/ab/jobs/search/url?per_page=%d&sort=recency&payment_verified=1&page=%d&q=%s"
|
|
return fmt.Sprintf(url, args.Per_Page, args.Page, args.Query)
|
|
}
|
|
|
|
func (u Upwork) SendRequest(url string) (string, error) {
|
|
Upclient := u.UpworkHttpClient
|
|
resp, err := Upclient.GetRequest(url)
|
|
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return resp.Body, nil
|
|
|
|
}
|
|
|
|
func mergeMaps(m1, m2 map[string]string) map[string]string {
|
|
// Iterate over m2 and add its key-value pairs to m1
|
|
for k, v := range m2 {
|
|
m1[k] = v
|
|
}
|
|
return m1
|
|
}
|
|
|
|
func readEnv(filename string) (map[string]string, error) {
|
|
|
|
// Open the .env file
|
|
file, err := os.Open(filename)
|
|
if err != nil {
|
|
fmt.Print()
|
|
return nil, err
|
|
}
|
|
defer file.Close()
|
|
|
|
// Create a map to store the key-value pairs
|
|
m := make(map[string]string)
|
|
|
|
// Read the file line by line
|
|
scanner := bufio.NewScanner(file)
|
|
for scanner.Scan() {
|
|
// Split the line on the "=" character
|
|
parts := strings.SplitN(scanner.Text(), "=", 2)
|
|
if len(parts) == 2 {
|
|
// Trim leading and trailing whitespace from the key and value
|
|
key := strings.TrimSpace(parts[0])
|
|
value := strings.TrimSpace(parts[1])
|
|
|
|
// Add the key-value pair to the map
|
|
m[key] = value
|
|
}
|
|
}
|
|
|
|
return m, nil
|
|
}
|
|
|
|
func InitUpwork() *Upwork {
|
|
headers := map[string]string{
|
|
"authority": "www.upwork.com",
|
|
"accept": "application/json, text/plain",
|
|
"accept-language": "en",
|
|
"cache-control": "no-cache",
|
|
"pragma": "no-cache",
|
|
"referer": "https://www.upwork.com/search/jobs/url?per_page=10&sort=recency",
|
|
"sec-fetch-site": "same-origin",
|
|
"sec-gpc": "1",
|
|
|
|
"vnd-eo-parent-span-id": "b8f3fe84-2aa4-4b9e-b750-6dc0e6e169a9",
|
|
"vnd-eo-span-id": "0d5bed38-342c-4b44-a885-b9c3de71a32a",
|
|
"x-odesk-user-agent": "oDesk LM",
|
|
"x-requested-with": "XMLHttpRequest",
|
|
}
|
|
auth_headers, err := readEnv("upwork/.env")
|
|
if err != nil {
|
|
log.Fatal("Could not read .env file, please add .env file in upwork folder")
|
|
}
|
|
headers = mergeMaps(auth_headers, headers)
|
|
|
|
client := network.InitClient(headers)
|
|
upwork := Upwork{
|
|
UpworkHttpClient: client,
|
|
}
|
|
|
|
return &upwork
|
|
|
|
}
|