mirror of
https://github.com/hashiromer/Upwork-Jobs-scraper-.git
synced 2025-12-17 09:46:01 +00:00
Updated Upwork API
This commit is contained in:
Hashir Omer
commit
c7c2998595
13
.gitignore
vendored
Normal file
13
.gitignore
vendored
Normal file
@ -0,0 +1,13 @@
|
|||||||
|
data/
|
||||||
|
|
||||||
|
|
||||||
|
all_jobs.json
|
||||||
|
|
||||||
|
*.json
|
||||||
|
*.csv
|
||||||
|
*.7z
|
||||||
|
oryxBuildBinary
|
||||||
|
*.ipynb
|
||||||
|
*.txt
|
||||||
|
*.env
|
||||||
|
!.env.example
|
||||||
132
JobSearchQuery.gql
Normal file
132
JobSearchQuery.gql
Normal file
@ -0,0 +1,132 @@
|
|||||||
|
query UserJobSearch($requestVariables: UserJobSearchV1Request!) {
|
||||||
|
search {
|
||||||
|
universalSearchNuxt {
|
||||||
|
userJobSearchV1(request: $requestVariables) {
|
||||||
|
paging {
|
||||||
|
total
|
||||||
|
offset
|
||||||
|
count
|
||||||
|
}
|
||||||
|
facets {
|
||||||
|
jobType {
|
||||||
|
key
|
||||||
|
value
|
||||||
|
}
|
||||||
|
workload {
|
||||||
|
key
|
||||||
|
value
|
||||||
|
}
|
||||||
|
clientHires {
|
||||||
|
key
|
||||||
|
value
|
||||||
|
}
|
||||||
|
durationV3 {
|
||||||
|
key
|
||||||
|
value
|
||||||
|
}
|
||||||
|
amount {
|
||||||
|
key
|
||||||
|
value
|
||||||
|
}
|
||||||
|
contractorTier {
|
||||||
|
key
|
||||||
|
value
|
||||||
|
}
|
||||||
|
contractToHire {
|
||||||
|
key
|
||||||
|
value
|
||||||
|
}
|
||||||
|
connects: connectPrice {
|
||||||
|
key
|
||||||
|
value
|
||||||
|
}
|
||||||
|
paymentVerified: payment {
|
||||||
|
key
|
||||||
|
value
|
||||||
|
}
|
||||||
|
proposals {
|
||||||
|
key
|
||||||
|
value
|
||||||
|
}
|
||||||
|
previousClients {
|
||||||
|
key
|
||||||
|
value
|
||||||
|
}
|
||||||
|
}
|
||||||
|
results {
|
||||||
|
id
|
||||||
|
title
|
||||||
|
description
|
||||||
|
relevanceEncoded
|
||||||
|
ontologySkills {
|
||||||
|
uid
|
||||||
|
parentSkillUid
|
||||||
|
prefLabel
|
||||||
|
prettyName: prefLabel
|
||||||
|
freeText
|
||||||
|
highlighted
|
||||||
|
}
|
||||||
|
isSTSVectorSearchResult
|
||||||
|
connectPrice
|
||||||
|
applied
|
||||||
|
upworkHistoryData {
|
||||||
|
client {
|
||||||
|
paymentVerificationStatus
|
||||||
|
country
|
||||||
|
totalReviews
|
||||||
|
totalFeedback
|
||||||
|
hasFinancialPrivacy
|
||||||
|
totalSpent {
|
||||||
|
isoCurrencyCode
|
||||||
|
amount
|
||||||
|
}
|
||||||
|
}
|
||||||
|
freelancerClientRelation {
|
||||||
|
lastContractRid
|
||||||
|
companyName
|
||||||
|
lastContractTitle
|
||||||
|
}
|
||||||
|
}
|
||||||
|
jobTile {
|
||||||
|
job {
|
||||||
|
id
|
||||||
|
ciphertext: cipherText
|
||||||
|
jobType
|
||||||
|
weeklyRetainerBudget
|
||||||
|
hourlyBudgetMax
|
||||||
|
hourlyBudgetMin
|
||||||
|
hourlyEngagementType
|
||||||
|
contractorTier
|
||||||
|
sourcingTimestamp
|
||||||
|
createTime
|
||||||
|
publishTime
|
||||||
|
enterpriseJob
|
||||||
|
personsToHire
|
||||||
|
premium
|
||||||
|
totalApplicants
|
||||||
|
hourlyEngagementDuration {
|
||||||
|
rid
|
||||||
|
label
|
||||||
|
weeks
|
||||||
|
mtime
|
||||||
|
ctime
|
||||||
|
}
|
||||||
|
fixedPriceAmount {
|
||||||
|
isoCurrencyCode
|
||||||
|
amount
|
||||||
|
}
|
||||||
|
fixedPriceEngagementDuration {
|
||||||
|
id
|
||||||
|
rid
|
||||||
|
label
|
||||||
|
weeks
|
||||||
|
ctime
|
||||||
|
mtime
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
24
README.md
Normal file
24
README.md
Normal file
@ -0,0 +1,24 @@
|
|||||||
|
# Upwork-Jobs-scraper
|
||||||
|
|
||||||
|
The code uses Upwork's internal Api to scrape new jobs posted on Upwork. I am not using headless browser due to two reasons.
|
||||||
|
|
||||||
|
1. Using headless bowsers is way more resource intensive compared to using an Api.
|
||||||
|
|
||||||
|
2. I don't have to deal with HTML parsing, the api returns json which can be directly passed to downstream systems.
|
||||||
|
|
||||||
|
# Note
|
||||||
|
|
||||||
|
The code uses Golang instead of Python because Upwork filters bots by checking TLS signatures of incoming requests. Unfortunately, I could not
|
||||||
|
find a way to do it in pure Python because Python is compiled with openssl and popular browsers do not use it. Chrome uses BoringSSl and firefox uses NSS.
|
||||||
|
These SSL libraries use different extensions and cipher suites which makes detection of TLS level configurations a more robust method to detect bot traffic.
|
||||||
|
|
||||||
|
Golang is a more lower level language compared to Python, so it allows changing network level configurations. I am using `cycletls` package in golang which makes spoofing TLS/JA3 fingerprints an easy task.
|
||||||
|
|
||||||
|
# How can you contribute?
|
||||||
|
|
||||||
|
These are some of the features I think could be useful.
|
||||||
|
|
||||||
|
- Better error handling with channels
|
||||||
|
- Add support for automatic proxy rotation. It can be extremely effective when used in conjunction with go routines.
|
||||||
|
- Add Api schema for Upwork Api.
|
||||||
|
- Add more scrapers, a lot of logic is platform agnostic which could be used to build scrapers for more platforms.
|
||||||
17
go.mod
Normal file
17
go.mod
Normal file
@ -0,0 +1,17 @@
|
|||||||
|
module scrapers
|
||||||
|
|
||||||
|
go 1.18
|
||||||
|
|
||||||
|
require github.com/Danny-Dasilva/CycleTLS/cycletls v0.0.0-20220620102923-c84d740b4757
|
||||||
|
|
||||||
|
require (
|
||||||
|
github.com/Danny-Dasilva/fhttp v0.0.0-20220524230104-f801520157d6 // indirect
|
||||||
|
github.com/Danny-Dasilva/utls v0.0.0-20220604023528-30cb107b834e // indirect
|
||||||
|
github.com/andybalholm/brotli v1.0.4 // indirect
|
||||||
|
github.com/dsnet/compress v0.0.1 // indirect
|
||||||
|
github.com/gorilla/websocket v1.5.0 // indirect
|
||||||
|
golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e // indirect
|
||||||
|
golang.org/x/net v0.0.0-20220615171555-694bf12d69de // indirect
|
||||||
|
golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c // indirect
|
||||||
|
golang.org/x/text v0.3.7 // indirect
|
||||||
|
)
|
||||||
53
go.sum
Normal file
53
go.sum
Normal file
@ -0,0 +1,53 @@
|
|||||||
|
github.com/Danny-Dasilva/CycleTLS/cycletls v0.0.0-20220620102923-c84d740b4757 h1:QH22vGS2DO07khPzKe3/CgFevznQkip5WNGEsQX7mFI=
|
||||||
|
github.com/Danny-Dasilva/CycleTLS/cycletls v0.0.0-20220620102923-c84d740b4757/go.mod h1:R4Hj85bdRH8zqymQ/oZUCmEsODgP3NpUvTEJtaVai7Y=
|
||||||
|
github.com/Danny-Dasilva/fhttp v0.0.0-20220418170016-5ea1c560e6a8/go.mod h1:t534vrahRNn9ax1tRiYSUvwJSa9jWaYYgETlfodBPm4=
|
||||||
|
github.com/Danny-Dasilva/fhttp v0.0.0-20220524230104-f801520157d6 h1:Wzbitazy0HugGNRACX7ZB1En21LT/TiVF6YbxoTTqN8=
|
||||||
|
github.com/Danny-Dasilva/fhttp v0.0.0-20220524230104-f801520157d6/go.mod h1:2IT2IFG+d+zzFuj3+ksGtVytcCBsF402zMNWHsWhD2U=
|
||||||
|
github.com/Danny-Dasilva/utls v0.0.0-20220418055514-7c61e0dbb504/go.mod h1:A2g8gPTJWDD3Y4iCTNon2vG3VcjdTBcgWBlZtopfNxU=
|
||||||
|
github.com/Danny-Dasilva/utls v0.0.0-20220418175931-f38e470e04f2/go.mod h1:A2g8gPTJWDD3Y4iCTNon2vG3VcjdTBcgWBlZtopfNxU=
|
||||||
|
github.com/Danny-Dasilva/utls v0.0.0-20220604023528-30cb107b834e h1:tqiguW0yAcIwQBQtD+d2rjBnboqB7CwG1OZ12F8avX8=
|
||||||
|
github.com/Danny-Dasilva/utls v0.0.0-20220604023528-30cb107b834e/go.mod h1:ssfbVNUfWJVRfW41RTpedOUlGXSq3J6aLmirUVkDgJk=
|
||||||
|
github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
|
||||||
|
github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY=
|
||||||
|
github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
|
||||||
|
github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
|
||||||
|
github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q=
|
||||||
|
github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo=
|
||||||
|
github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY=
|
||||||
|
github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
|
||||||
|
github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
||||||
|
github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
|
||||||
|
github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
|
||||||
|
github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8=
|
||||||
|
gitlab.com/yawning/bsaes.git v0.0.0-20190805113838-0a714cd429ec/go.mod h1:BZ1RAoRPbCxum9Grlv5aeksu2H8BiKehBYooU2LFiOQ=
|
||||||
|
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||||
|
golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||||
|
golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
|
||||||
|
golang.org/x/crypto v0.0.0-20220518034528-6f7dac969898/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
|
||||||
|
golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e h1:T8NU3HyQ8ClP4SEE+KbFlg6n0NhuTsN4MyznaarGsZM=
|
||||||
|
golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
|
||||||
|
golang.org/x/net v0.0.0-20190328230028-74de082e2cca/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||||
|
golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||||
|
golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||||
|
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||||
|
golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
|
||||||
|
golang.org/x/net v0.0.0-20220615171555-694bf12d69de h1:ogOG2+P6LjO2j55AkRScrkB2BFpd+Z8TY2wcM0Z3MGo=
|
||||||
|
golang.org/x/net v0.0.0-20220615171555-694bf12d69de/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||||
|
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||||
|
golang.org/x/sys v0.0.0-20190804053845-51ab0e2deafa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
|
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
|
golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
|
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
|
golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c h1:aFV+BgZ4svzjfabn8ERpuB4JI4N6/rdy1iusx77G3oU=
|
||||||
|
golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
|
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||||
|
golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||||
|
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||||
|
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||||
|
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||||
|
golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
|
||||||
|
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||||
|
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||||
13
main.go
Normal file
13
main.go
Normal file
@ -0,0 +1,13 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import "scrapers/upwork"
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
|
||||||
|
p := upwork.InitPipeline()
|
||||||
|
err := p.Run("")
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
35
network/httpClient.go
Normal file
35
network/httpClient.go
Normal file
@ -0,0 +1,35 @@
|
|||||||
|
package network
|
||||||
|
|
||||||
|
import "github.com/Danny-Dasilva/CycleTLS/cycletls"
|
||||||
|
|
||||||
|
type Client struct {
|
||||||
|
httpClient cycletls.CycleTLS
|
||||||
|
options cycletls.Options
|
||||||
|
}
|
||||||
|
|
||||||
|
func InitClient(headers map[string]string) *Client {
|
||||||
|
client := cycletls.Init()
|
||||||
|
options := cycletls.Options{
|
||||||
|
Body: "",
|
||||||
|
Ja3: "771,4865-4867-4866-49195-49199-52393-52392-49196-49200-49162-49161-49171-49172-51-57-47-53-10,0-23-65281-10-11-35-16-5-51-43-13-45-28-21,29-23-24-25-256-257,0",
|
||||||
|
UserAgent: "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:87.0) Gecko/20100101 Firefox/87.0",
|
||||||
|
Headers: headers,
|
||||||
|
}
|
||||||
|
|
||||||
|
httpClient := Client{
|
||||||
|
httpClient: client,
|
||||||
|
options: options,
|
||||||
|
}
|
||||||
|
|
||||||
|
return &httpClient
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c Client) GetRequest(url string) (cycletls.Response, error) {
|
||||||
|
return c.httpClient.Do(url, c.options, "GET")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c Client) PostRequest(url string, body string) (cycletls.Response, error) {
|
||||||
|
c.options.Body = body
|
||||||
|
c.options.Headers["Content-Type"] = "application/json"
|
||||||
|
return c.httpClient.Do(url, c.options, "POST")
|
||||||
|
}
|
||||||
2
upwork/.env.example
Normal file
2
upwork/.env.example
Normal file
@ -0,0 +1,2 @@
|
|||||||
|
authorization
|
||||||
|
cookie
|
||||||
828
upwork/types.go
Normal file
828
upwork/types.go
Normal file
@ -0,0 +1,828 @@
|
|||||||
|
package upwork
|
||||||
|
|
||||||
|
import "time"
|
||||||
|
|
||||||
|
type LoggedOutError struct {
|
||||||
|
Error struct {
|
||||||
|
Code int `json:"code"`
|
||||||
|
Message string `json:"message"`
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type UpworkApiResponse struct {
|
||||||
|
URL string `json:"url"`
|
||||||
|
SearchGUID string `json:"searchGuid"`
|
||||||
|
SearchResults struct {
|
||||||
|
Q string `json:"q"`
|
||||||
|
Paging struct {
|
||||||
|
Total int `json:"total"`
|
||||||
|
Offset int `json:"offset"`
|
||||||
|
Count int `json:"count"`
|
||||||
|
ResultSetTs int `json:"resultSetTs"`
|
||||||
|
} `json:"paging"`
|
||||||
|
Spellcheck struct {
|
||||||
|
CorrectedQueries []interface{} `json:"corrected_queries"`
|
||||||
|
} `json:"spellcheck"`
|
||||||
|
Jobs []struct {
|
||||||
|
Title string `json:"title"`
|
||||||
|
CreatedOn time.Time `json:"createdOn"`
|
||||||
|
Type int `json:"type"`
|
||||||
|
Ciphertext string `json:"ciphertext"`
|
||||||
|
Description string `json:"description"`
|
||||||
|
Category2 interface{} `json:"category2"`
|
||||||
|
Subcategory2 interface{} `json:"subcategory2"`
|
||||||
|
Skills []interface{} `json:"skills"`
|
||||||
|
Duration string `json:"duration"`
|
||||||
|
ShortDuration string `json:"shortDuration"`
|
||||||
|
DurationLabel string `json:"durationLabel"`
|
||||||
|
Engagement string `json:"engagement"`
|
||||||
|
ShortEngagement string `json:"shortEngagement"`
|
||||||
|
Amount struct {
|
||||||
|
CurrencyCode string `json:"currencyCode"`
|
||||||
|
Amount int `json:"amount"`
|
||||||
|
} `json:"amount"`
|
||||||
|
Recno int `json:"recno"`
|
||||||
|
UID string `json:"uid"`
|
||||||
|
Client struct {
|
||||||
|
PaymentVerificationStatus int `json:"paymentVerificationStatus"`
|
||||||
|
Location struct {
|
||||||
|
Country string `json:"country"`
|
||||||
|
} `json:"location"`
|
||||||
|
TotalSpent float64 `json:"totalSpent"`
|
||||||
|
TotalReviews int `json:"totalReviews"`
|
||||||
|
TotalFeedback float64 `json:"totalFeedback"`
|
||||||
|
CompanyRid int `json:"companyRid"`
|
||||||
|
CompanyName interface{} `json:"companyName"`
|
||||||
|
EdcUserID int `json:"edcUserId"`
|
||||||
|
LastContractPlatform interface{} `json:"lastContractPlatform"`
|
||||||
|
LastContractRid int `json:"lastContractRid"`
|
||||||
|
LastContractTitle interface{} `json:"lastContractTitle"`
|
||||||
|
FeedbackText string `json:"feedbackText"`
|
||||||
|
CompanyOrgUID string `json:"companyOrgUid"`
|
||||||
|
HasFinancialPrivacy bool `json:"hasFinancialPrivacy"`
|
||||||
|
} `json:"client"`
|
||||||
|
FreelancersToHire int `json:"freelancersToHire"`
|
||||||
|
RelevanceEncoded string `json:"relevanceEncoded"`
|
||||||
|
EnterpriseJob bool `json:"enterpriseJob"`
|
||||||
|
TierText string `json:"tierText"`
|
||||||
|
Tier string `json:"tier"`
|
||||||
|
TierLabel string `json:"tierLabel"`
|
||||||
|
IsSaved interface{} `json:"isSaved"`
|
||||||
|
Feedback string `json:"feedback"`
|
||||||
|
ProposalsTier string `json:"proposalsTier"`
|
||||||
|
IsApplied bool `json:"isApplied"`
|
||||||
|
Sticky bool `json:"sticky"`
|
||||||
|
StickyLabel string `json:"stickyLabel"`
|
||||||
|
JobTs string `json:"jobTs"`
|
||||||
|
PrefFreelancerLocationMandatory bool `json:"prefFreelancerLocationMandatory"`
|
||||||
|
PrefFreelancerLocation []interface{} `json:"prefFreelancerLocation"`
|
||||||
|
Premium bool `json:"premium"`
|
||||||
|
PlusBadge interface{} `json:"plusBadge"`
|
||||||
|
PublishedOn time.Time `json:"publishedOn"`
|
||||||
|
RenewedOn time.Time `json:"renewedOn"`
|
||||||
|
SandsService interface{} `json:"sandsService"`
|
||||||
|
SandsSpec interface{} `json:"sandsSpec"`
|
||||||
|
SandsAttrs interface{} `json:"sandsAttrs"`
|
||||||
|
Occupation interface{} `json:"occupation"`
|
||||||
|
Attrs []struct {
|
||||||
|
ParentSkillUID interface{} `json:"parentSkillUid"`
|
||||||
|
FreeText interface{} `json:"freeText"`
|
||||||
|
SkillType int `json:"skillType"`
|
||||||
|
UID string `json:"uid"`
|
||||||
|
Highlighted bool `json:"highlighted"`
|
||||||
|
PrettyName string `json:"prettyName"`
|
||||||
|
} `json:"attrs"`
|
||||||
|
IsLocal bool `json:"isLocal"`
|
||||||
|
WorkType interface{} `json:"workType"`
|
||||||
|
Locations []interface{} `json:"locations"`
|
||||||
|
Occupations struct {
|
||||||
|
Category struct {
|
||||||
|
UID string `json:"uid"`
|
||||||
|
PrefLabel string `json:"prefLabel"`
|
||||||
|
} `json:"category"`
|
||||||
|
Subcategories []struct {
|
||||||
|
UID string `json:"uid"`
|
||||||
|
PrefLabel string `json:"prefLabel"`
|
||||||
|
} `json:"subcategories"`
|
||||||
|
Oservice struct {
|
||||||
|
UID string `json:"uid"`
|
||||||
|
PrefLabel string `json:"prefLabel"`
|
||||||
|
} `json:"oservice"`
|
||||||
|
} `json:"occupations"`
|
||||||
|
WeeklyBudget interface{} `json:"weeklyBudget"`
|
||||||
|
HourlyBudgetText interface{} `json:"hourlyBudgetText"`
|
||||||
|
Tags []interface{} `json:"tags"`
|
||||||
|
ClientRelation interface{} `json:"clientRelation"`
|
||||||
|
TotalFreelancersToHire int `json:"totalFreelancersToHire"`
|
||||||
|
TeamUID interface{} `json:"teamUid"`
|
||||||
|
MultipleFreelancersToHirePredicted interface{} `json:"multipleFreelancersToHirePredicted"`
|
||||||
|
ConnectPrice int `json:"connectPrice"`
|
||||||
|
} `json:"jobs"`
|
||||||
|
Facets struct {
|
||||||
|
JobType struct {
|
||||||
|
Num0 int `json:"0"`
|
||||||
|
Num1 int `json:"1"`
|
||||||
|
WeeklyRetainer int `json:"weekly_retainer"`
|
||||||
|
} `json:"jobType"`
|
||||||
|
Workload struct {
|
||||||
|
None int `json:"none"`
|
||||||
|
FullTime int `json:"full_time"`
|
||||||
|
AsNeeded int `json:"as_needed"`
|
||||||
|
} `json:"workload"`
|
||||||
|
Duration struct {
|
||||||
|
Ongoing int `json:"ongoing"`
|
||||||
|
Week int `json:"week"`
|
||||||
|
Month int `json:"month"`
|
||||||
|
Semester int `json:"semester"`
|
||||||
|
Quarter int `json:"quarter"`
|
||||||
|
} `json:"duration"`
|
||||||
|
ClientHires struct {
|
||||||
|
Num0 int `json:"0"`
|
||||||
|
One9 int `json:"1-9"`
|
||||||
|
One0 int `json:"10-"`
|
||||||
|
} `json:"clientHires"`
|
||||||
|
Budget struct {
|
||||||
|
Num0 int `json:"0"`
|
||||||
|
Num50 int `json:"50"`
|
||||||
|
Num100 int `json:"100"`
|
||||||
|
Num250 int `json:"250"`
|
||||||
|
Num500 int `json:"500"`
|
||||||
|
Num1000 int `json:"1000"`
|
||||||
|
Num2000 int `json:"2000"`
|
||||||
|
Num5000 int `json:"5000"`
|
||||||
|
Num10000 int `json:"10000"`
|
||||||
|
Num20000 int `json:"20000"`
|
||||||
|
Num100000 int `json:"100000"`
|
||||||
|
} `json:"budget"`
|
||||||
|
ContractorTier struct {
|
||||||
|
Num1 int `json:"1"`
|
||||||
|
Num2 int `json:"2"`
|
||||||
|
Num3 int `json:"3"`
|
||||||
|
} `json:"contractorTier"`
|
||||||
|
Categories2 []struct {
|
||||||
|
Category struct {
|
||||||
|
Slug string `json:"slug"`
|
||||||
|
Name string `json:"name"`
|
||||||
|
ResultCount int `json:"resultCount"`
|
||||||
|
} `json:"category"`
|
||||||
|
Subcategories []struct {
|
||||||
|
Slug string `json:"slug"`
|
||||||
|
Name string `json:"name"`
|
||||||
|
ResultCount int `json:"resultCount"`
|
||||||
|
} `json:"subcategories"`
|
||||||
|
} `json:"categories2"`
|
||||||
|
PreviousClients struct {
|
||||||
|
All int `json:"all"`
|
||||||
|
} `json:"previousClients"`
|
||||||
|
Proposals struct {
|
||||||
|
Five9 int `json:"5-9"`
|
||||||
|
Two049 int `json:"20-49"`
|
||||||
|
Zero4 int `json:"0-4"`
|
||||||
|
One014 int `json:"10-14"`
|
||||||
|
NAMING_FAILED int `json:"*"`
|
||||||
|
One519 int `json:"15-19"`
|
||||||
|
} `json:"proposals"`
|
||||||
|
Amount struct {
|
||||||
|
FiveThousandPlus int `json:"5000-"`
|
||||||
|
ANY int `json:"*"`
|
||||||
|
Hundred_to_Five_hundred int `json:"100-499"`
|
||||||
|
Thousand_to_five_thousand int `json:"1000-4999"`
|
||||||
|
ZeroTo_Ninty_Nine int `json:"0-99"`
|
||||||
|
FiveHundred_to_thousand int `json:"500-999"`
|
||||||
|
} `json:"amount"`
|
||||||
|
UserLocationMatch struct {
|
||||||
|
Num1 int `json:"1"`
|
||||||
|
} `json:"userLocationMatch"`
|
||||||
|
Services interface{} `json:"services"`
|
||||||
|
Specs interface{} `json:"specs"`
|
||||||
|
Attrs interface{} `json:"attrs"`
|
||||||
|
DurationV2 struct {
|
||||||
|
Week int `json:"week"`
|
||||||
|
Ongoing int `json:"ongoing"`
|
||||||
|
Month int `json:"month"`
|
||||||
|
} `json:"durationV2"`
|
||||||
|
Enterprise struct {
|
||||||
|
True int `json:"true"`
|
||||||
|
False int `json:"false"`
|
||||||
|
} `json:"enterprise"`
|
||||||
|
UserDomesticJobsLastDays struct {
|
||||||
|
Num7 int `json:"7"`
|
||||||
|
} `json:"userDomesticJobsLastDays"`
|
||||||
|
DurationV3 struct {
|
||||||
|
Weeks int `json:"weeks"`
|
||||||
|
Months int `json:"months"`
|
||||||
|
Semester int `json:"semester"`
|
||||||
|
Ongoing int `json:"ongoing"`
|
||||||
|
} `json:"durationV3"`
|
||||||
|
Occupations []struct {
|
||||||
|
TaxonomyLevel string `json:"taxonomyLevel"`
|
||||||
|
UID string `json:"uid"`
|
||||||
|
Label string `json:"label"`
|
||||||
|
Count int `json:"count"`
|
||||||
|
Occupations []struct {
|
||||||
|
TaxonomyLevel string `json:"taxonomyLevel"`
|
||||||
|
UID string `json:"uid"`
|
||||||
|
Label string `json:"label"`
|
||||||
|
Count int `json:"count"`
|
||||||
|
Occupations []struct {
|
||||||
|
TaxonomyLevel string `json:"taxonomyLevel"`
|
||||||
|
UID string `json:"uid"`
|
||||||
|
Label string `json:"label"`
|
||||||
|
Count int `json:"count"`
|
||||||
|
Occupations interface{} `json:"occupations"`
|
||||||
|
} `json:"occupations"`
|
||||||
|
} `json:"occupations"`
|
||||||
|
} `json:"occupations"`
|
||||||
|
EngagementDuration interface{} `json:"engagementDuration"`
|
||||||
|
FreelancersNeeded struct {
|
||||||
|
Zero1 int `json:"0-1"`
|
||||||
|
Six int `json:"6-"`
|
||||||
|
Two5 int `json:"2-5"`
|
||||||
|
Two int `json:"2-"`
|
||||||
|
Zero int `json:"0-"`
|
||||||
|
} `json:"freelancersNeeded"`
|
||||||
|
Location struct {
|
||||||
|
SouthAmerica int `json:"South America"`
|
||||||
|
PapuaNewGuinea int `json:"Papua New Guinea"`
|
||||||
|
Cambodia int `json:"Cambodia"`
|
||||||
|
Paraguay int `json:"Paraguay"`
|
||||||
|
Kazakhstan int `json:"Kazakhstan"`
|
||||||
|
Bahamas int `json:"Bahamas"`
|
||||||
|
MarshallIslands int `json:"Marshall Islands"`
|
||||||
|
Mali int `json:"Mali"`
|
||||||
|
Panama int `json:"Panama"`
|
||||||
|
Guadeloupe int `json:"Guadeloupe"`
|
||||||
|
EasternAsia int `json:"Eastern Asia"`
|
||||||
|
Laos int `json:"Laos"`
|
||||||
|
Argentina int `json:"Argentina"`
|
||||||
|
Seychelles int `json:"Seychelles"`
|
||||||
|
Africa int `json:"Africa"`
|
||||||
|
Zambia int `json:"Zambia"`
|
||||||
|
Belize int `json:"Belize"`
|
||||||
|
Bahrain int `json:"Bahrain"`
|
||||||
|
Congo int `json:"Congo"`
|
||||||
|
GuineaBissau int `json:"Guinea-Bissau"`
|
||||||
|
SaintBarthelemy int `json:"Saint Barthelemy"`
|
||||||
|
Namibia int `json:"Namibia"`
|
||||||
|
PalestinianTerritories int `json:"Palestinian Territories"`
|
||||||
|
Finland int `json:"Finland"`
|
||||||
|
FaroeIslands int `json:"Faroe Islands"`
|
||||||
|
NetherlandsAntilles int `json:"Netherlands Antilles"`
|
||||||
|
Georgia int `json:"Georgia"`
|
||||||
|
SaintKittsAndNevis int `json:"Saint Kitts and Nevis"`
|
||||||
|
Yemen int `json:"Yemen"`
|
||||||
|
PuertoRico int `json:"Puerto Rico"`
|
||||||
|
Madagascar int `json:"Madagascar"`
|
||||||
|
Aruba int `json:"Aruba"`
|
||||||
|
Sweden int `json:"Sweden"`
|
||||||
|
Malawi int `json:"Malawi"`
|
||||||
|
Andorra int `json:"Andorra"`
|
||||||
|
Liechtenstein int `json:"Liechtenstein"`
|
||||||
|
Poland int `json:"Poland"`
|
||||||
|
UnitedStatesVirginIslands int `json:"United States Virgin Islands"`
|
||||||
|
Jordan int `json:"Jordan"`
|
||||||
|
Bulgaria int `json:"Bulgaria"`
|
||||||
|
Tunisia int `json:"Tunisia"`
|
||||||
|
UnitedArabEmirates int `json:"United Arab Emirates"`
|
||||||
|
Kenya int `json:"Kenya"`
|
||||||
|
FrenchPolynesia int `json:"French Polynesia"`
|
||||||
|
Lebanon int `json:"Lebanon"`
|
||||||
|
Djibouti int `json:"Djibouti"`
|
||||||
|
Azerbaijan int `json:"Azerbaijan"`
|
||||||
|
CzechRepublic int `json:"Czech Republic"`
|
||||||
|
SaintLucia int `json:"Saint Lucia"`
|
||||||
|
Mauritania int `json:"Mauritania"`
|
||||||
|
Guernsey int `json:"Guernsey"`
|
||||||
|
Mayotte int `json:"Mayotte"`
|
||||||
|
SanMarino int `json:"San Marino"`
|
||||||
|
Israel int `json:"Israel"`
|
||||||
|
Australia int `json:"Australia"`
|
||||||
|
Tajikistan int `json:"Tajikistan"`
|
||||||
|
Myanmar int `json:"Myanmar"`
|
||||||
|
CentralAmerica int `json:"Central America"`
|
||||||
|
EasternAfrica int `json:"Eastern Africa"`
|
||||||
|
Cameroon int `json:"Cameroon"`
|
||||||
|
Gibraltar int `json:"Gibraltar"`
|
||||||
|
Cyprus int `json:"Cyprus"`
|
||||||
|
Malaysia int `json:"Malaysia"`
|
||||||
|
Oman int `json:"Oman"`
|
||||||
|
Iceland int `json:"Iceland"`
|
||||||
|
Armenia int `json:"Armenia"`
|
||||||
|
Gabon int `json:"Gabon"`
|
||||||
|
WesternAsia int `json:"Western Asia"`
|
||||||
|
Polynesia int `json:"Polynesia"`
|
||||||
|
Luxembourg int `json:"Luxembourg"`
|
||||||
|
Brazil int `json:"Brazil"`
|
||||||
|
TurksAndCaicosIslands int `json:"Turks and Caicos Islands"`
|
||||||
|
Algeria int `json:"Algeria"`
|
||||||
|
Slovenia int `json:"Slovenia"`
|
||||||
|
Jersey int `json:"Jersey"`
|
||||||
|
AntiguaAndBarbuda int `json:"Antigua and Barbuda"`
|
||||||
|
Ecuador int `json:"Ecuador"`
|
||||||
|
Colombia int `json:"Colombia"`
|
||||||
|
Moldova int `json:"Moldova"`
|
||||||
|
Vanuatu int `json:"Vanuatu"`
|
||||||
|
UnitedStatesMinorOutlyingIslands int `json:"United States Minor Outlying Islands"`
|
||||||
|
Italy int `json:"Italy"`
|
||||||
|
Honduras int `json:"Honduras"`
|
||||||
|
Haiti int `json:"Haiti"`
|
||||||
|
Burundi int `json:"Burundi"`
|
||||||
|
Singapore int `json:"Singapore"`
|
||||||
|
FrenchGuiana int `json:"French Guiana"`
|
||||||
|
AmericanSamoa int `json:"American Samoa"`
|
||||||
|
Russia int `json:"Russia"`
|
||||||
|
Netherlands int `json:"Netherlands"`
|
||||||
|
China int `json:"China"`
|
||||||
|
Martinique int `json:"Martinique"`
|
||||||
|
SaintPierreAndMiquelon int `json:"Saint Pierre and Miquelon"`
|
||||||
|
Kyrgyzstan int `json:"Kyrgyzstan"`
|
||||||
|
Reunion int `json:"Reunion"`
|
||||||
|
Bhutan int `json:"Bhutan"`
|
||||||
|
Romania int `json:"Romania"`
|
||||||
|
MiddleAfrica int `json:"Middle Africa"`
|
||||||
|
Togo int `json:"Togo"`
|
||||||
|
SouthernAsia int `json:"Southern Asia"`
|
||||||
|
Philippines int `json:"Philippines"`
|
||||||
|
CoteDIvoire int `json:"Cote d\'Ivoire"`
|
||||||
|
Uzbekistan int `json:"Uzbekistan"`
|
||||||
|
Asia int `json:"Asia"`
|
||||||
|
BritishVirginIslands int `json:"British Virgin Islands"`
|
||||||
|
Zimbabwe int `json:"Zimbabwe"`
|
||||||
|
BritishIndianOceanTerritory int `json:"British Indian Ocean Territory"`
|
||||||
|
Montenegro int `json:"Montenegro"`
|
||||||
|
Indonesia int `json:"Indonesia"`
|
||||||
|
Dominica int `json:"Dominica"`
|
||||||
|
Benin int `json:"Benin"`
|
||||||
|
Angola int `json:"Angola"`
|
||||||
|
EasternEurope int `json:"Eastern Europe"`
|
||||||
|
Portugal int `json:"Portugal"`
|
||||||
|
BruneiDarussalam int `json:"Brunei Darussalam"`
|
||||||
|
NewCaledonia int `json:"New Caledonia"`
|
||||||
|
Grenada int `json:"Grenada"`
|
||||||
|
Greece int `json:"Greece"`
|
||||||
|
CaymanIslands int `json:"Cayman Islands"`
|
||||||
|
Mongolia int `json:"Mongolia"`
|
||||||
|
Latvia int `json:"Latvia"`
|
||||||
|
Morocco int `json:"Morocco"`
|
||||||
|
Guyana int `json:"Guyana"`
|
||||||
|
Guatemala int `json:"Guatemala"`
|
||||||
|
Chile int `json:"Chile"`
|
||||||
|
Nepal int `json:"Nepal"`
|
||||||
|
NorthernEurope int `json:"Northern Europe"`
|
||||||
|
IsleOfMan int `json:"Isle of Man"`
|
||||||
|
Ukraine int `json:"Ukraine"`
|
||||||
|
Tanzania int `json:"Tanzania"`
|
||||||
|
Ghana int `json:"Ghana"`
|
||||||
|
HolySee int `json:"Holy See"`
|
||||||
|
CentralAsia int `json:"Central Asia"`
|
||||||
|
Anguilla int `json:"Anguilla"`
|
||||||
|
SouthEasternAsia int `json:"South-Eastern Asia"`
|
||||||
|
India int `json:"India"`
|
||||||
|
Canada int `json:"Canada"`
|
||||||
|
Maldives int `json:"Maldives"`
|
||||||
|
Turkey int `json:"Turkey"`
|
||||||
|
Belgium int `json:"Belgium"`
|
||||||
|
Taiwan int `json:"Taiwan"`
|
||||||
|
TrinidadAndTobago int `json:"Trinidad and Tobago"`
|
||||||
|
SouthernEurope int `json:"Southern Europe"`
|
||||||
|
SouthAfrica int `json:"South Africa"`
|
||||||
|
Bermuda int `json:"Bermuda"`
|
||||||
|
AlandIslands int `json:"Aland Islands"`
|
||||||
|
Jamaica int `json:"Jamaica"`
|
||||||
|
Turkmenistan int `json:"Turkmenistan"`
|
||||||
|
Peru int `json:"Peru"`
|
||||||
|
Germany int `json:"Germany"`
|
||||||
|
Americas int `json:"Americas"`
|
||||||
|
Fiji int `json:"Fiji"`
|
||||||
|
HongKong int `json:"Hong Kong"`
|
||||||
|
UnitedStates int `json:"United States"`
|
||||||
|
Guinea int `json:"Guinea"`
|
||||||
|
MicronesiaFederatedStatesOf int `json:"Micronesia, Federated States of"`
|
||||||
|
Somalia int `json:"Somalia"`
|
||||||
|
Chad int `json:"Chad"`
|
||||||
|
Thailand int `json:"Thailand"`
|
||||||
|
Kiribati int `json:"Kiribati"`
|
||||||
|
EquatorialGuinea int `json:"Equatorial Guinea"`
|
||||||
|
CostaRica int `json:"Costa Rica"`
|
||||||
|
SaintMartinFrenchPart int `json:"Saint Martin (French part)"`
|
||||||
|
Vietnam int `json:"Vietnam"`
|
||||||
|
Nigeria int `json:"Nigeria"`
|
||||||
|
Kuwait int `json:"Kuwait"`
|
||||||
|
Croatia int `json:"Croatia"`
|
||||||
|
Uruguay int `json:"Uruguay"`
|
||||||
|
SriLanka int `json:"Sri Lanka"`
|
||||||
|
CookIslands int `json:"Cook Islands"`
|
||||||
|
TimorLeste int `json:"Timor-Leste"`
|
||||||
|
UnitedKingdom int `json:"United Kingdom"`
|
||||||
|
Switzerland int `json:"Switzerland"`
|
||||||
|
Samoa int `json:"Samoa"`
|
||||||
|
Spain int `json:"Spain"`
|
||||||
|
WesternAfrica int `json:"Western Africa"`
|
||||||
|
Venezuela int `json:"Venezuela"`
|
||||||
|
BurkinaFaso int `json:"Burkina Faso"`
|
||||||
|
Swaziland int `json:"Swaziland"`
|
||||||
|
Caribbean int `json:"Caribbean"`
|
||||||
|
Estonia int `json:"Estonia"`
|
||||||
|
Niue int `json:"Niue"`
|
||||||
|
SouthKorea int `json:"South Korea"`
|
||||||
|
Austria int `json:"Austria"`
|
||||||
|
Mozambique int `json:"Mozambique"`
|
||||||
|
ElSalvador int `json:"El Salvador"`
|
||||||
|
Monaco int `json:"Monaco"`
|
||||||
|
Guam int `json:"Guam"`
|
||||||
|
Lesotho int `json:"Lesotho"`
|
||||||
|
Tonga int `json:"Tonga"`
|
||||||
|
NorthernAfrica int `json:"Northern Africa"`
|
||||||
|
WesternSahara int `json:"Western Sahara"`
|
||||||
|
Hungary int `json:"Hungary"`
|
||||||
|
Japan int `json:"Japan"`
|
||||||
|
Europe int `json:"Europe"`
|
||||||
|
Curacao int `json:"Curacao"`
|
||||||
|
Belarus int `json:"Belarus"`
|
||||||
|
Mauritius int `json:"Mauritius"`
|
||||||
|
WesternEurope int `json:"Western Europe"`
|
||||||
|
Albania int `json:"Albania"`
|
||||||
|
NewZealand int `json:"New Zealand"`
|
||||||
|
NorthernAmerica int `json:"Northern America"`
|
||||||
|
SintMaartenDutchPart int `json:"Sint Maarten (Dutch part)"`
|
||||||
|
Senegal int `json:"Senegal"`
|
||||||
|
Macedonia int `json:"Macedonia"`
|
||||||
|
Ethiopia int `json:"Ethiopia"`
|
||||||
|
Egypt int `json:"Egypt"`
|
||||||
|
SierraLeone int `json:"Sierra Leone"`
|
||||||
|
Bolivia int `json:"Bolivia"`
|
||||||
|
Oceania int `json:"Oceania"`
|
||||||
|
Malta int `json:"Malta"`
|
||||||
|
SaudiArabia int `json:"Saudi Arabia"`
|
||||||
|
CapeVerde int `json:"Cape Verde"`
|
||||||
|
Pakistan int `json:"Pakistan"`
|
||||||
|
Gambia int `json:"Gambia"`
|
||||||
|
Qatar int `json:"Qatar"`
|
||||||
|
Ireland int `json:"Ireland"`
|
||||||
|
Slovakia int `json:"Slovakia"`
|
||||||
|
Serbia int `json:"Serbia"`
|
||||||
|
Lithuania int `json:"Lithuania"`
|
||||||
|
France int `json:"France"`
|
||||||
|
BosniaAndHerzegovina int `json:"Bosnia and Herzegovina"`
|
||||||
|
AustraliaAndNewZealand int `json:"Australia and New Zealand"`
|
||||||
|
Rwanda int `json:"Rwanda"`
|
||||||
|
Bangladesh int `json:"Bangladesh"`
|
||||||
|
Nicaragua int `json:"Nicaragua"`
|
||||||
|
Barbados int `json:"Barbados"`
|
||||||
|
Norway int `json:"Norway"`
|
||||||
|
SouthernAfrica int `json:"Southern Africa"`
|
||||||
|
SaintVincentAndTheGrenadines int `json:"Saint Vincent and the Grenadines"`
|
||||||
|
Botswana int `json:"Botswana"`
|
||||||
|
Melanesia int `json:"Melanesia"`
|
||||||
|
Macao int `json:"Macao"`
|
||||||
|
DominicanRepublic int `json:"Dominican Republic"`
|
||||||
|
Denmark int `json:"Denmark"`
|
||||||
|
Uganda int `json:"Uganda"`
|
||||||
|
Mexico int `json:"Mexico"`
|
||||||
|
Suriname int `json:"Suriname"`
|
||||||
|
Greenland int `json:"Greenland"`
|
||||||
|
} `json:"location"`
|
||||||
|
Timezone struct {
|
||||||
|
int `json:""`
|
||||||
|
AmericaSaoPaulo int `json:"America/Sao_Paulo"`
|
||||||
|
AsiaVladivostok int `json:"Asia/Vladivostok"`
|
||||||
|
EuropeBerlin int `json:"Europe/Berlin"`
|
||||||
|
AfricaCairo int `json:"Africa/Cairo"`
|
||||||
|
EuropeMoscow int `json:"Europe/Moscow"`
|
||||||
|
PacificHonolulu int `json:"Pacific/Honolulu"`
|
||||||
|
AustraliaHobart int `json:"Australia/Hobart"`
|
||||||
|
EuropeLondon int `json:"Europe/London"`
|
||||||
|
AsiaBaghdad int `json:"Asia/Baghdad"`
|
||||||
|
AsiaShanghai int `json:"Asia/Shanghai"`
|
||||||
|
AmericaTijuana int `json:"America/Tijuana"`
|
||||||
|
AmericaManagua int `json:"America/Managua"`
|
||||||
|
AsiaYerevan int `json:"Asia/Yerevan"`
|
||||||
|
AsiaKamchatka int `json:"Asia/Kamchatka"`
|
||||||
|
AfricaHarare int `json:"Africa/Harare"`
|
||||||
|
AmericaNome int `json:"America/Nome"`
|
||||||
|
AsiaYakutsk int `json:"Asia/Yakutsk"`
|
||||||
|
AmericaChicago int `json:"America/Chicago"`
|
||||||
|
AmericaHalifax int `json:"America/Halifax"`
|
||||||
|
AmericaIndianaIndianapolis int `json:"America/Indiana/Indianapolis"`
|
||||||
|
EuropeParis int `json:"Europe/Paris"`
|
||||||
|
PacificFiji int `json:"Pacific/Fiji"`
|
||||||
|
AsiaTehran int `json:"Asia/Tehran"`
|
||||||
|
AmericaLaPaz int `json:"America/La_Paz"`
|
||||||
|
AsiaTashkent int `json:"Asia/Tashkent"`
|
||||||
|
AsiaBangkok int `json:"Asia/Bangkok"`
|
||||||
|
PacificMidway int `json:"Pacific/Midway"`
|
||||||
|
AmericaRecife int `json:"America/Recife"`
|
||||||
|
AmericaBuenosAires int `json:"America/Buenos_Aires"`
|
||||||
|
AustraliaAdelaide int `json:"Australia/Adelaide"`
|
||||||
|
AsiaYangon int `json:"Asia/Yangon"`
|
||||||
|
AsiaKatmandu int `json:"Asia/Katmandu"`
|
||||||
|
AsiaAlmaty int `json:"Asia/Almaty"`
|
||||||
|
AmericaPhoenix int `json:"America/Phoenix"`
|
||||||
|
EuropePrague int `json:"Europe/Prague"`
|
||||||
|
AmericaMexicoCity int `json:"America/Mexico_City"`
|
||||||
|
AsiaTbilisi int `json:"Asia/Tbilisi"`
|
||||||
|
AsiaJerusalem int `json:"Asia/Jerusalem"`
|
||||||
|
EuropeLisbon int `json:"Europe/Lisbon"`
|
||||||
|
AtlanticSouthGeorgia int `json:"Atlantic/South_Georgia"`
|
||||||
|
AsiaKarachi int `json:"Asia/Karachi"`
|
||||||
|
AustraliaPerth int `json:"Australia/Perth"`
|
||||||
|
AustraliaDarwin int `json:"Australia/Darwin"`
|
||||||
|
AsiaCalcutta int `json:"Asia/Calcutta"`
|
||||||
|
AmericaBogota int `json:"America/Bogota"`
|
||||||
|
AsiaKabul int `json:"Asia/Kabul"`
|
||||||
|
AmericaNewYork int `json:"America/New_York"`
|
||||||
|
AtlanticAzores int `json:"Atlantic/Azores"`
|
||||||
|
AsiaKrasnoyarsk int `json:"Asia/Krasnoyarsk"`
|
||||||
|
EET int `json:"EET"`
|
||||||
|
PacificAuckland int `json:"Pacific/Auckland"`
|
||||||
|
EuropeMinsk int `json:"Europe/Minsk"`
|
||||||
|
AfricaCasablanca int `json:"Africa/Casablanca"`
|
||||||
|
AmericaCaracas int `json:"America/Caracas"`
|
||||||
|
EuropeKiev int `json:"Europe/Kiev"`
|
||||||
|
AsiaMagadan int `json:"Asia/Magadan"`
|
||||||
|
AmericaRegina int `json:"America/Regina"`
|
||||||
|
PacificGuam int `json:"Pacific/Guam"`
|
||||||
|
AsiaIrkutsk int `json:"Asia/Irkutsk"`
|
||||||
|
AfricaAlgiers int `json:"Africa/Algiers"`
|
||||||
|
AmericaStJohns int `json:"America/St_Johns"`
|
||||||
|
AmericaFortaleza int `json:"America/Fortaleza"`
|
||||||
|
AmericaDenver int `json:"America/Denver"`
|
||||||
|
AmericaIndianaKnox int `json:"America/Indiana/Knox"`
|
||||||
|
EtcUTC int `json:"Etc/UTC"`
|
||||||
|
AustraliaSydney int `json:"Australia/Sydney"`
|
||||||
|
PacificApia int `json:"Pacific/Apia"`
|
||||||
|
AsiaTokyo int `json:"Asia/Tokyo"`
|
||||||
|
AsiaIstanbul int `json:"Asia/Istanbul"`
|
||||||
|
AsiaOmsk int `json:"Asia/Omsk"`
|
||||||
|
AustraliaBrisbane int `json:"Australia/Brisbane"`
|
||||||
|
AmericaLosAngeles int `json:"America/Los_Angeles"`
|
||||||
|
AsiaYekaterinburg int `json:"Asia/Yekaterinburg"`
|
||||||
|
EuropeAthens int `json:"Europe/Athens"`
|
||||||
|
} `json:"timezone"`
|
||||||
|
ConnectPrice struct {
|
||||||
|
Num4 int `json:"4"`
|
||||||
|
Num6 int `json:"6"`
|
||||||
|
Zero2 int `json:"0-2"`
|
||||||
|
} `json:"connectPrice"`
|
||||||
|
ContractToHire struct {
|
||||||
|
True int `json:"true"`
|
||||||
|
False int `json:"false"`
|
||||||
|
} `json:"contractToHire"`
|
||||||
|
Categories []interface{} `json:"categories"`
|
||||||
|
Subcategories []interface{} `json:"subcategories"`
|
||||||
|
PaymentVerified struct {
|
||||||
|
Num1 int `json:"1"`
|
||||||
|
} `json:"paymentVerified"`
|
||||||
|
} `json:"facets"`
|
||||||
|
IsSearchWithEmptyParams bool `json:"isSearchWithEmptyParams"`
|
||||||
|
CurrentQuery struct {
|
||||||
|
Sort string `json:"sort"`
|
||||||
|
} `json:"currentQuery"`
|
||||||
|
QueryParsedParams struct {
|
||||||
|
Sort string `json:"sort"`
|
||||||
|
Paging string `json:"paging"`
|
||||||
|
} `json:"queryParsedParams"`
|
||||||
|
PageTitle string `json:"pageTitle"`
|
||||||
|
JobSearchError bool `json:"jobSearchError"`
|
||||||
|
RssLink string `json:"rssLink"`
|
||||||
|
AtomLink string `json:"atomLink"`
|
||||||
|
SmartSearch struct {
|
||||||
|
DownloadTeamApplication bool `json:"downloadTeamApplication"`
|
||||||
|
} `json:"smartSearch"`
|
||||||
|
} `json:"searchResults"`
|
||||||
|
Filters struct {
|
||||||
|
Q string `json:"q"`
|
||||||
|
Sort string `json:"sort"`
|
||||||
|
Skills struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
Label string `json:"label"`
|
||||||
|
Options []interface{} `json:"options"`
|
||||||
|
} `json:"skills"`
|
||||||
|
Categories []struct {
|
||||||
|
URLName string `json:"urlName"`
|
||||||
|
Value string `json:"value"`
|
||||||
|
Label string `json:"label"`
|
||||||
|
ActiveLabel string `json:"activeLabel"`
|
||||||
|
Checked bool `json:"checked"`
|
||||||
|
Count int `json:"count"`
|
||||||
|
Subcategories []struct {
|
||||||
|
URLName string `json:"urlName"`
|
||||||
|
Value string `json:"value"`
|
||||||
|
Label string `json:"label"`
|
||||||
|
ActiveLabel string `json:"activeLabel"`
|
||||||
|
Checked bool `json:"checked"`
|
||||||
|
Count int `json:"count"`
|
||||||
|
} `json:"subcategories"`
|
||||||
|
} `json:"categories"`
|
||||||
|
JobType struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
Label string `json:"label"`
|
||||||
|
LoggingSublocation string `json:"loggingSublocation"`
|
||||||
|
LoggingLabel string `json:"loggingLabel"`
|
||||||
|
Options []struct {
|
||||||
|
Value string `json:"value"`
|
||||||
|
Label string `json:"label"`
|
||||||
|
Checked bool `json:"checked"`
|
||||||
|
Count int `json:"count"`
|
||||||
|
} `json:"options"`
|
||||||
|
} `json:"jobType"`
|
||||||
|
ContractorTier struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
Label string `json:"label"`
|
||||||
|
LoggingSublocation string `json:"loggingSublocation"`
|
||||||
|
LoggingLabel string `json:"loggingLabel"`
|
||||||
|
Options []struct {
|
||||||
|
Value string `json:"value"`
|
||||||
|
Label string `json:"label"`
|
||||||
|
Checked bool `json:"checked"`
|
||||||
|
Count int `json:"count"`
|
||||||
|
} `json:"options"`
|
||||||
|
} `json:"contractorTier"`
|
||||||
|
ClientHires struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
Label string `json:"label"`
|
||||||
|
LoggingSublocation string `json:"loggingSublocation"`
|
||||||
|
LoggingLabel string `json:"loggingLabel"`
|
||||||
|
Options []struct {
|
||||||
|
Value string `json:"value"`
|
||||||
|
Label string `json:"label"`
|
||||||
|
Checked bool `json:"checked"`
|
||||||
|
Count int `json:"count"`
|
||||||
|
} `json:"options"`
|
||||||
|
} `json:"clientHires"`
|
||||||
|
Proposals struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
Label string `json:"label"`
|
||||||
|
LoggingSublocation string `json:"loggingSublocation"`
|
||||||
|
LoggingLabel string `json:"loggingLabel"`
|
||||||
|
Options []struct {
|
||||||
|
Value string `json:"value"`
|
||||||
|
Label string `json:"label"`
|
||||||
|
Checked bool `json:"checked"`
|
||||||
|
Count int `json:"count"`
|
||||||
|
ActiveLabel string `json:"activeLabel"`
|
||||||
|
} `json:"options"`
|
||||||
|
} `json:"proposals"`
|
||||||
|
Amount struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
Label string `json:"label"`
|
||||||
|
LoggingSublocation string `json:"loggingSublocation"`
|
||||||
|
DisabledJobTypeValue string `json:"disabledJobTypeValue"`
|
||||||
|
DisabledMessage string `json:"disabledMessage"`
|
||||||
|
Options []struct {
|
||||||
|
Value string `json:"value"`
|
||||||
|
Label string `json:"label"`
|
||||||
|
Checked bool `json:"checked"`
|
||||||
|
Count int `json:"count"`
|
||||||
|
} `json:"options"`
|
||||||
|
CustomAmount struct {
|
||||||
|
Value interface{} `json:"value"`
|
||||||
|
Label string `json:"label"`
|
||||||
|
Checked bool `json:"checked"`
|
||||||
|
CustomMin interface{} `json:"customMin"`
|
||||||
|
CustomMax interface{} `json:"customMax"`
|
||||||
|
} `json:"customAmount"`
|
||||||
|
} `json:"amount"`
|
||||||
|
Workload struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
Label string `json:"label"`
|
||||||
|
LoggingSublocation string `json:"loggingSublocation"`
|
||||||
|
LoggingLabel string `json:"loggingLabel"`
|
||||||
|
Options []struct {
|
||||||
|
Value string `json:"value"`
|
||||||
|
Label string `json:"label"`
|
||||||
|
Checked bool `json:"checked"`
|
||||||
|
Count int `json:"count"`
|
||||||
|
} `json:"options"`
|
||||||
|
DisabledJobTypeValue string `json:"disabledJobTypeValue"`
|
||||||
|
DisabledMessage string `json:"disabledMessage"`
|
||||||
|
} `json:"workload"`
|
||||||
|
DurationV3 struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
Label string `json:"label"`
|
||||||
|
LoggingSublocation string `json:"loggingSublocation"`
|
||||||
|
LoggingLabel string `json:"loggingLabel"`
|
||||||
|
Options []struct {
|
||||||
|
Value string `json:"value"`
|
||||||
|
Label string `json:"label"`
|
||||||
|
Checked bool `json:"checked"`
|
||||||
|
Count int `json:"count"`
|
||||||
|
} `json:"options"`
|
||||||
|
DisabledJobTypeValue string `json:"disabledJobTypeValue"`
|
||||||
|
DisabledMessage string `json:"disabledMessage"`
|
||||||
|
} `json:"duration_v3"`
|
||||||
|
PreviousClients struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
Label string `json:"label"`
|
||||||
|
LoggingSublocation string `json:"loggingSublocation"`
|
||||||
|
Value string `json:"value"`
|
||||||
|
Checked bool `json:"checked"`
|
||||||
|
Count int `json:"count"`
|
||||||
|
} `json:"previousClients"`
|
||||||
|
PaymentVerified struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
Label string `json:"label"`
|
||||||
|
LoggingSublocation string `json:"loggingSublocation"`
|
||||||
|
Value string `json:"value"`
|
||||||
|
Checked bool `json:"checked"`
|
||||||
|
Count int `json:"count"`
|
||||||
|
} `json:"paymentVerified"`
|
||||||
|
TalentClouds struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
Label string `json:"label"`
|
||||||
|
LoggingSublocation string `json:"loggingSublocation"`
|
||||||
|
Options []interface{} `json:"options"`
|
||||||
|
} `json:"talentClouds"`
|
||||||
|
UserLocationMatch struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
Label string `json:"label"`
|
||||||
|
LoggingSublocation string `json:"loggingSublocation"`
|
||||||
|
Value string `json:"value"`
|
||||||
|
Checked bool `json:"checked"`
|
||||||
|
Count int `json:"count"`
|
||||||
|
} `json:"userLocationMatch"`
|
||||||
|
Occupation struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
Label string `json:"label"`
|
||||||
|
LoggingSublocation string `json:"loggingSublocation"`
|
||||||
|
Options []interface{} `json:"options"`
|
||||||
|
} `json:"occupation"`
|
||||||
|
OntologySkills struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
Label string `json:"label"`
|
||||||
|
LoggingSublocation string `json:"loggingSublocation"`
|
||||||
|
Options []interface{} `json:"options"`
|
||||||
|
} `json:"ontologySkills"`
|
||||||
|
HourlyRate struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
Label string `json:"label"`
|
||||||
|
LoggingSublocation string `json:"loggingSublocation"`
|
||||||
|
Options []interface{} `json:"options"`
|
||||||
|
CustomHourlyRate struct {
|
||||||
|
Value interface{} `json:"value"`
|
||||||
|
Label string `json:"label"`
|
||||||
|
Checked bool `json:"checked"`
|
||||||
|
CustomMin interface{} `json:"customMin"`
|
||||||
|
CustomMax interface{} `json:"customMax"`
|
||||||
|
} `json:"customHourlyRate"`
|
||||||
|
} `json:"hourly_rate"`
|
||||||
|
FreelancersNeeded struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
Label string `json:"label"`
|
||||||
|
LoggingSublocation string `json:"loggingSublocation"`
|
||||||
|
LoggingLabel string `json:"loggingLabel"`
|
||||||
|
Options []struct {
|
||||||
|
Value string `json:"value"`
|
||||||
|
Label string `json:"label"`
|
||||||
|
Checked bool `json:"checked"`
|
||||||
|
Count int `json:"count"`
|
||||||
|
} `json:"options"`
|
||||||
|
} `json:"freelancers_needed"`
|
||||||
|
Location struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
Label string `json:"label"`
|
||||||
|
LoggingSublocation string `json:"loggingSublocation"`
|
||||||
|
Options []struct {
|
||||||
|
UID string `json:"uid"`
|
||||||
|
Label string `json:"label"`
|
||||||
|
SubRegionUID string `json:"subRegionUid,omitempty"`
|
||||||
|
RegionUID string `json:"regionUid,omitempty"`
|
||||||
|
Value string `json:"value"`
|
||||||
|
Type string `json:"type"`
|
||||||
|
Checked bool `json:"checked"`
|
||||||
|
Count int `json:"count"`
|
||||||
|
} `json:"options"`
|
||||||
|
} `json:"location"`
|
||||||
|
Timezones struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
Label string `json:"label"`
|
||||||
|
LoggingSublocation string `json:"loggingSublocation"`
|
||||||
|
Options []struct {
|
||||||
|
Value string `json:"value"`
|
||||||
|
Label string `json:"label"`
|
||||||
|
Checked bool `json:"checked"`
|
||||||
|
Count int `json:"count"`
|
||||||
|
} `json:"options"`
|
||||||
|
} `json:"timezones"`
|
||||||
|
Connects struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
Label string `json:"label"`
|
||||||
|
LoggingSublocation string `json:"loggingSublocation"`
|
||||||
|
LoggingLabel string `json:"loggingLabel"`
|
||||||
|
Options []struct {
|
||||||
|
Value string `json:"value"`
|
||||||
|
Label string `json:"label"`
|
||||||
|
Checked bool `json:"checked"`
|
||||||
|
Count int `json:"count"`
|
||||||
|
} `json:"options"`
|
||||||
|
} `json:"connects"`
|
||||||
|
ContractToHire struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
Label string `json:"label"`
|
||||||
|
LoggingSublocation string `json:"loggingSublocation"`
|
||||||
|
Value string `json:"value"`
|
||||||
|
Checked bool `json:"checked"`
|
||||||
|
Count int `json:"count"`
|
||||||
|
} `json:"contractToHire"`
|
||||||
|
} `json:"filters"`
|
||||||
|
}
|
||||||
104
upwork/upworkClient.go
Normal file
104
upwork/upworkClient.go
Normal file
@ -0,0 +1,104 @@
|
|||||||
|
package upwork
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bufio"
|
||||||
|
"encoding/json"
|
||||||
|
"log"
|
||||||
|
"os"
|
||||||
|
"scrapers/network"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
type Upwork struct {
|
||||||
|
UpworkHttpClient *network.Client
|
||||||
|
}
|
||||||
|
|
||||||
|
type GraphQLRequest struct {
|
||||||
|
Query string `json:"query"`
|
||||||
|
Variables map[string]interface{} `json:"variables"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (u Upwork) SendRequest(query string, variables map[string]interface{}) (string, error) {
|
||||||
|
url := "https://www.upwork.com/api/graphql/v1"
|
||||||
|
|
||||||
|
requestBody := GraphQLRequest{
|
||||||
|
Query: query,
|
||||||
|
Variables: variables,
|
||||||
|
}
|
||||||
|
|
||||||
|
jsonBody, err := json.Marshal(requestBody)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
resp, err := u.UpworkHttpClient.PostRequest(url, string(jsonBody))
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
return resp.Body, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func readGraphQLQuery(filename string) (string, error) {
|
||||||
|
content, err := os.ReadFile(filename)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return string(content), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func mergeMaps(m1, m2 map[string]string) map[string]string {
|
||||||
|
for k, v := range m2 {
|
||||||
|
m1[k] = v
|
||||||
|
}
|
||||||
|
return m1
|
||||||
|
}
|
||||||
|
|
||||||
|
func readEnv(filename string) (map[string]string, error) {
|
||||||
|
file, err := os.Open(filename)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer file.Close()
|
||||||
|
|
||||||
|
m := make(map[string]string)
|
||||||
|
scanner := bufio.NewScanner(file)
|
||||||
|
for scanner.Scan() {
|
||||||
|
parts := strings.SplitN(scanner.Text(), "=", 2)
|
||||||
|
if len(parts) == 2 {
|
||||||
|
key := strings.TrimSpace(parts[0])
|
||||||
|
value := strings.TrimSpace(parts[1])
|
||||||
|
m[key] = value
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return m, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func InitUpwork() *Upwork {
|
||||||
|
headers := map[string]string{
|
||||||
|
"authority": "www.upwork.com",
|
||||||
|
"accept": "application/json, text/plain",
|
||||||
|
"accept-language": "en",
|
||||||
|
"content-type": "application/json",
|
||||||
|
"cache-control": "no-cache",
|
||||||
|
"pragma": "no-cache",
|
||||||
|
"referer": "https://www.upwork.com/search/jobs/",
|
||||||
|
"sec-fetch-site": "same-origin",
|
||||||
|
"sec-gpc": "1",
|
||||||
|
"x-requested-with": "XMLHttpRequest",
|
||||||
|
}
|
||||||
|
|
||||||
|
auth_headers, err := readEnv("upwork/.env")
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal("Could not read .env file, please add .env file in upwork folder")
|
||||||
|
}
|
||||||
|
headers = mergeMaps(auth_headers, headers)
|
||||||
|
|
||||||
|
client := network.InitClient(headers)
|
||||||
|
upwork := Upwork{
|
||||||
|
UpworkHttpClient: client,
|
||||||
|
}
|
||||||
|
|
||||||
|
return &upwork
|
||||||
|
}
|
||||||
154
upwork/upworkPipeline.go
Normal file
154
upwork/upworkPipeline.go
Normal file
@ -0,0 +1,154 @@
|
|||||||
|
package upwork
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"log"
|
||||||
|
"os"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
type UpworkPipeLine struct {
|
||||||
|
upworkClient *Upwork
|
||||||
|
}
|
||||||
|
|
||||||
|
func InitPipeline() *UpworkPipeLine {
|
||||||
|
return &UpworkPipeLine{
|
||||||
|
upworkClient: InitUpwork(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (u *UpworkPipeLine) appendToJSONL(data []interface{}, filename string) error {
|
||||||
|
file, err := os.OpenFile(filename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer file.Close()
|
||||||
|
|
||||||
|
for _, item := range data {
|
||||||
|
jsonLine, err := json.Marshal(item)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if _, err := file.Write(append(jsonLine, '\n')); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (u *UpworkPipeLine) isResponseValid(data string) bool {
|
||||||
|
var resp map[string]interface{}
|
||||||
|
err := json.Unmarshal([]byte(data), &resp)
|
||||||
|
if err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
_, hasErrors := resp["errors"]
|
||||||
|
return !hasErrors
|
||||||
|
}
|
||||||
|
|
||||||
|
func (u *UpworkPipeLine) getTotalDocuments(query string, variables map[string]interface{}) (int, error) {
|
||||||
|
resp, err := u.upworkClient.SendRequest(query, variables)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
|
||||||
|
if !u.isResponseValid(resp) {
|
||||||
|
return 0, fmt.Errorf("invalid response")
|
||||||
|
}
|
||||||
|
|
||||||
|
var graphQLResp map[string]interface{}
|
||||||
|
err = json.Unmarshal([]byte(resp), &graphQLResp)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
|
||||||
|
paging, ok := graphQLResp["data"].(map[string]interface{})["search"].(map[string]interface{})["universalSearchNuxt"].(map[string]interface{})["userJobSearchV1"].(map[string]interface{})["paging"].(map[string]interface{})
|
||||||
|
if !ok {
|
||||||
|
return 0, fmt.Errorf("unexpected response structure")
|
||||||
|
}
|
||||||
|
|
||||||
|
total, ok := paging["total"].(float64)
|
||||||
|
if !ok {
|
||||||
|
return 0, fmt.Errorf("total is not a number")
|
||||||
|
}
|
||||||
|
|
||||||
|
return int(total), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (u *UpworkPipeLine) handleRequest(query string, variables map[string]interface{}, filename string) error {
|
||||||
|
resp, err := u.upworkClient.SendRequest(query, variables)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if !u.isResponseValid(resp) {
|
||||||
|
//print the response
|
||||||
|
fmt.Println(resp)
|
||||||
|
return fmt.Errorf("invalid response returned")
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
var graphQLResp map[string]interface{}
|
||||||
|
err = json.Unmarshal([]byte(resp), &graphQLResp)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
results, ok := graphQLResp["data"].(map[string]interface{})["search"].(map[string]interface{})["universalSearchNuxt"].(map[string]interface{})["userJobSearchV1"].(map[string]interface{})["results"].([]interface{})
|
||||||
|
if !ok {
|
||||||
|
return fmt.Errorf("unexpected response structure")
|
||||||
|
}
|
||||||
|
|
||||||
|
return u.appendToJSONL(results, filename)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (u *UpworkPipeLine) Run(userQuery string) error {
|
||||||
|
query, err := readGraphQLQuery("JobSearchQuery.gql")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
variables := map[string]interface{}{
|
||||||
|
"requestVariables": map[string]interface{}{
|
||||||
|
"userQuery": userQuery,
|
||||||
|
"sort": "recency",
|
||||||
|
"highlight": true,
|
||||||
|
"paging": map[string]interface{}{
|
||||||
|
"offset": 0,
|
||||||
|
"count": 50,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
total_docs, err := u.getTotalDocuments(query, variables)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Printf("%s has a total of %d jobs\n", userQuery, total_docs)
|
||||||
|
|
||||||
|
iterations := total_docs / 50
|
||||||
|
// if iterations > 100 {
|
||||||
|
// iterations = 100
|
||||||
|
// }
|
||||||
|
|
||||||
|
fmt.Printf("A total of %d iterations will be performed\n", iterations)
|
||||||
|
|
||||||
|
now := time.Now()
|
||||||
|
dateString := now.Format("2006-01-02")
|
||||||
|
filename := fmt.Sprintf("upwork_jobs_%s.jsonl", dateString)
|
||||||
|
|
||||||
|
for i := 0; i < iterations; i++ {
|
||||||
|
fmt.Printf("Processing iteration %d of %d\n", i+1, iterations)
|
||||||
|
variables["requestVariables"].(map[string]interface{})["paging"].(map[string]interface{})["offset"] = i * 50
|
||||||
|
err := u.handleRequest(query, variables, filename)
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("Error in iteration %d: %v", i+1, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Printf("Job data has been written to %s\n", filename)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
Loading…
x
Reference in New Issue
Block a user