mirror of
https://github.com/hashiromer/Upwork-Jobs-scraper-.git
synced 2025-12-29 16:16:01 +00:00
Added types for upwork api respone
This commit is contained in:
Hashir Omer
3
.gitignore
vendored
3
.gitignore
vendored
@@ -8,4 +8,5 @@ all_jobs.json
|
||||
*.7z
|
||||
oryxBuildBinary
|
||||
*.ipynb
|
||||
*.txt
|
||||
*.txt
|
||||
*.env
|
||||
823
upwork/types.go
Normal file
823
upwork/types.go
Normal file
@@ -0,0 +1,823 @@
|
||||
package upwork
|
||||
|
||||
import "time"
|
||||
|
||||
type UpworkApiResponse struct {
|
||||
URL string `json:"url"`
|
||||
SearchGUID string `json:"searchGuid"`
|
||||
SearchResults struct {
|
||||
Q string `json:"q"`
|
||||
Paging struct {
|
||||
Total int `json:"total"`
|
||||
Offset int `json:"offset"`
|
||||
Count int `json:"count"`
|
||||
ResultSetTs int `json:"resultSetTs"`
|
||||
} `json:"paging"`
|
||||
Spellcheck struct {
|
||||
CorrectedQueries []interface{} `json:"corrected_queries"`
|
||||
} `json:"spellcheck"`
|
||||
Jobs []struct {
|
||||
Title string `json:"title"`
|
||||
CreatedOn time.Time `json:"createdOn"`
|
||||
Type int `json:"type"`
|
||||
Ciphertext string `json:"ciphertext"`
|
||||
Description string `json:"description"`
|
||||
Category2 interface{} `json:"category2"`
|
||||
Subcategory2 interface{} `json:"subcategory2"`
|
||||
Skills []interface{} `json:"skills"`
|
||||
Duration string `json:"duration"`
|
||||
ShortDuration string `json:"shortDuration"`
|
||||
DurationLabel string `json:"durationLabel"`
|
||||
Engagement string `json:"engagement"`
|
||||
ShortEngagement string `json:"shortEngagement"`
|
||||
Amount struct {
|
||||
CurrencyCode string `json:"currencyCode"`
|
||||
Amount int `json:"amount"`
|
||||
} `json:"amount"`
|
||||
Recno int `json:"recno"`
|
||||
UID string `json:"uid"`
|
||||
Client struct {
|
||||
PaymentVerificationStatus int `json:"paymentVerificationStatus"`
|
||||
Location struct {
|
||||
Country string `json:"country"`
|
||||
} `json:"location"`
|
||||
TotalSpent float64 `json:"totalSpent"`
|
||||
TotalReviews int `json:"totalReviews"`
|
||||
TotalFeedback float64 `json:"totalFeedback"`
|
||||
CompanyRid int `json:"companyRid"`
|
||||
CompanyName interface{} `json:"companyName"`
|
||||
EdcUserID int `json:"edcUserId"`
|
||||
LastContractPlatform interface{} `json:"lastContractPlatform"`
|
||||
LastContractRid int `json:"lastContractRid"`
|
||||
LastContractTitle interface{} `json:"lastContractTitle"`
|
||||
FeedbackText string `json:"feedbackText"`
|
||||
CompanyOrgUID string `json:"companyOrgUid"`
|
||||
HasFinancialPrivacy bool `json:"hasFinancialPrivacy"`
|
||||
} `json:"client"`
|
||||
FreelancersToHire int `json:"freelancersToHire"`
|
||||
RelevanceEncoded string `json:"relevanceEncoded"`
|
||||
EnterpriseJob bool `json:"enterpriseJob"`
|
||||
TierText string `json:"tierText"`
|
||||
Tier string `json:"tier"`
|
||||
TierLabel string `json:"tierLabel"`
|
||||
IsSaved interface{} `json:"isSaved"`
|
||||
Feedback string `json:"feedback"`
|
||||
ProposalsTier string `json:"proposalsTier"`
|
||||
IsApplied bool `json:"isApplied"`
|
||||
Sticky bool `json:"sticky"`
|
||||
StickyLabel string `json:"stickyLabel"`
|
||||
JobTs string `json:"jobTs"`
|
||||
PrefFreelancerLocationMandatory bool `json:"prefFreelancerLocationMandatory"`
|
||||
PrefFreelancerLocation []interface{} `json:"prefFreelancerLocation"`
|
||||
Premium bool `json:"premium"`
|
||||
PlusBadge interface{} `json:"plusBadge"`
|
||||
PublishedOn time.Time `json:"publishedOn"`
|
||||
RenewedOn time.Time `json:"renewedOn"`
|
||||
SandsService interface{} `json:"sandsService"`
|
||||
SandsSpec interface{} `json:"sandsSpec"`
|
||||
SandsAttrs interface{} `json:"sandsAttrs"`
|
||||
Occupation interface{} `json:"occupation"`
|
||||
Attrs []struct {
|
||||
ParentSkillUID interface{} `json:"parentSkillUid"`
|
||||
FreeText interface{} `json:"freeText"`
|
||||
SkillType int `json:"skillType"`
|
||||
UID string `json:"uid"`
|
||||
Highlighted bool `json:"highlighted"`
|
||||
PrettyName string `json:"prettyName"`
|
||||
} `json:"attrs"`
|
||||
IsLocal bool `json:"isLocal"`
|
||||
WorkType interface{} `json:"workType"`
|
||||
Locations []interface{} `json:"locations"`
|
||||
Occupations struct {
|
||||
Category struct {
|
||||
UID string `json:"uid"`
|
||||
PrefLabel string `json:"prefLabel"`
|
||||
} `json:"category"`
|
||||
Subcategories []struct {
|
||||
UID string `json:"uid"`
|
||||
PrefLabel string `json:"prefLabel"`
|
||||
} `json:"subcategories"`
|
||||
Oservice struct {
|
||||
UID string `json:"uid"`
|
||||
PrefLabel string `json:"prefLabel"`
|
||||
} `json:"oservice"`
|
||||
} `json:"occupations"`
|
||||
WeeklyBudget interface{} `json:"weeklyBudget"`
|
||||
HourlyBudgetText interface{} `json:"hourlyBudgetText"`
|
||||
Tags []interface{} `json:"tags"`
|
||||
ClientRelation interface{} `json:"clientRelation"`
|
||||
TotalFreelancersToHire int `json:"totalFreelancersToHire"`
|
||||
TeamUID interface{} `json:"teamUid"`
|
||||
MultipleFreelancersToHirePredicted interface{} `json:"multipleFreelancersToHirePredicted"`
|
||||
ConnectPrice int `json:"connectPrice"`
|
||||
} `json:"jobs"`
|
||||
Facets struct {
|
||||
JobType struct {
|
||||
Num0 int `json:"0"`
|
||||
Num1 int `json:"1"`
|
||||
WeeklyRetainer int `json:"weekly_retainer"`
|
||||
} `json:"jobType"`
|
||||
Workload struct {
|
||||
None int `json:"none"`
|
||||
FullTime int `json:"full_time"`
|
||||
AsNeeded int `json:"as_needed"`
|
||||
} `json:"workload"`
|
||||
Duration struct {
|
||||
Ongoing int `json:"ongoing"`
|
||||
Week int `json:"week"`
|
||||
Month int `json:"month"`
|
||||
Semester int `json:"semester"`
|
||||
Quarter int `json:"quarter"`
|
||||
} `json:"duration"`
|
||||
ClientHires struct {
|
||||
Num0 int `json:"0"`
|
||||
One9 int `json:"1-9"`
|
||||
One0 int `json:"10-"`
|
||||
} `json:"clientHires"`
|
||||
Budget struct {
|
||||
Num0 int `json:"0"`
|
||||
Num50 int `json:"50"`
|
||||
Num100 int `json:"100"`
|
||||
Num250 int `json:"250"`
|
||||
Num500 int `json:"500"`
|
||||
Num1000 int `json:"1000"`
|
||||
Num2000 int `json:"2000"`
|
||||
Num5000 int `json:"5000"`
|
||||
Num10000 int `json:"10000"`
|
||||
Num20000 int `json:"20000"`
|
||||
Num100000 int `json:"100000"`
|
||||
} `json:"budget"`
|
||||
ContractorTier struct {
|
||||
Num1 int `json:"1"`
|
||||
Num2 int `json:"2"`
|
||||
Num3 int `json:"3"`
|
||||
} `json:"contractorTier"`
|
||||
Categories2 []struct {
|
||||
Category struct {
|
||||
Slug string `json:"slug"`
|
||||
Name string `json:"name"`
|
||||
ResultCount int `json:"resultCount"`
|
||||
} `json:"category"`
|
||||
Subcategories []struct {
|
||||
Slug string `json:"slug"`
|
||||
Name string `json:"name"`
|
||||
ResultCount int `json:"resultCount"`
|
||||
} `json:"subcategories"`
|
||||
} `json:"categories2"`
|
||||
PreviousClients struct {
|
||||
All int `json:"all"`
|
||||
} `json:"previousClients"`
|
||||
Proposals struct {
|
||||
Five9 int `json:"5-9"`
|
||||
Two049 int `json:"20-49"`
|
||||
Zero4 int `json:"0-4"`
|
||||
One014 int `json:"10-14"`
|
||||
NAMING_FAILED int `json:"*"`
|
||||
One519 int `json:"15-19"`
|
||||
} `json:"proposals"`
|
||||
Amount struct {
|
||||
FiveThousandPlus int `json:"5000-"`
|
||||
ANY int `json:"*"`
|
||||
Hundred_to_Five_hundred int `json:"100-499"`
|
||||
Thousand_to_five_thousand int `json:"1000-4999"`
|
||||
ZeroTo_Ninty_Nine int `json:"0-99"`
|
||||
FiveHundred_to_thousand int `json:"500-999"`
|
||||
} `json:"amount"`
|
||||
UserLocationMatch struct {
|
||||
Num1 int `json:"1"`
|
||||
} `json:"userLocationMatch"`
|
||||
Services interface{} `json:"services"`
|
||||
Specs interface{} `json:"specs"`
|
||||
Attrs interface{} `json:"attrs"`
|
||||
DurationV2 struct {
|
||||
Week int `json:"week"`
|
||||
Ongoing int `json:"ongoing"`
|
||||
Month int `json:"month"`
|
||||
} `json:"durationV2"`
|
||||
Enterprise struct {
|
||||
True int `json:"true"`
|
||||
False int `json:"false"`
|
||||
} `json:"enterprise"`
|
||||
UserDomesticJobsLastDays struct {
|
||||
Num7 int `json:"7"`
|
||||
} `json:"userDomesticJobsLastDays"`
|
||||
DurationV3 struct {
|
||||
Weeks int `json:"weeks"`
|
||||
Months int `json:"months"`
|
||||
Semester int `json:"semester"`
|
||||
Ongoing int `json:"ongoing"`
|
||||
} `json:"durationV3"`
|
||||
Occupations []struct {
|
||||
TaxonomyLevel string `json:"taxonomyLevel"`
|
||||
UID string `json:"uid"`
|
||||
Label string `json:"label"`
|
||||
Count int `json:"count"`
|
||||
Occupations []struct {
|
||||
TaxonomyLevel string `json:"taxonomyLevel"`
|
||||
UID string `json:"uid"`
|
||||
Label string `json:"label"`
|
||||
Count int `json:"count"`
|
||||
Occupations []struct {
|
||||
TaxonomyLevel string `json:"taxonomyLevel"`
|
||||
UID string `json:"uid"`
|
||||
Label string `json:"label"`
|
||||
Count int `json:"count"`
|
||||
Occupations interface{} `json:"occupations"`
|
||||
} `json:"occupations"`
|
||||
} `json:"occupations"`
|
||||
} `json:"occupations"`
|
||||
EngagementDuration interface{} `json:"engagementDuration"`
|
||||
FreelancersNeeded struct {
|
||||
Zero1 int `json:"0-1"`
|
||||
Six int `json:"6-"`
|
||||
Two5 int `json:"2-5"`
|
||||
Two int `json:"2-"`
|
||||
Zero int `json:"0-"`
|
||||
} `json:"freelancersNeeded"`
|
||||
Location struct {
|
||||
SouthAmerica int `json:"South America"`
|
||||
PapuaNewGuinea int `json:"Papua New Guinea"`
|
||||
Cambodia int `json:"Cambodia"`
|
||||
Paraguay int `json:"Paraguay"`
|
||||
Kazakhstan int `json:"Kazakhstan"`
|
||||
Bahamas int `json:"Bahamas"`
|
||||
MarshallIslands int `json:"Marshall Islands"`
|
||||
Mali int `json:"Mali"`
|
||||
Panama int `json:"Panama"`
|
||||
Guadeloupe int `json:"Guadeloupe"`
|
||||
EasternAsia int `json:"Eastern Asia"`
|
||||
Laos int `json:"Laos"`
|
||||
Argentina int `json:"Argentina"`
|
||||
Seychelles int `json:"Seychelles"`
|
||||
Africa int `json:"Africa"`
|
||||
Zambia int `json:"Zambia"`
|
||||
Belize int `json:"Belize"`
|
||||
Bahrain int `json:"Bahrain"`
|
||||
Congo int `json:"Congo"`
|
||||
GuineaBissau int `json:"Guinea-Bissau"`
|
||||
SaintBarthelemy int `json:"Saint Barthelemy"`
|
||||
Namibia int `json:"Namibia"`
|
||||
PalestinianTerritories int `json:"Palestinian Territories"`
|
||||
Finland int `json:"Finland"`
|
||||
FaroeIslands int `json:"Faroe Islands"`
|
||||
NetherlandsAntilles int `json:"Netherlands Antilles"`
|
||||
Georgia int `json:"Georgia"`
|
||||
SaintKittsAndNevis int `json:"Saint Kitts and Nevis"`
|
||||
Yemen int `json:"Yemen"`
|
||||
PuertoRico int `json:"Puerto Rico"`
|
||||
Madagascar int `json:"Madagascar"`
|
||||
Aruba int `json:"Aruba"`
|
||||
Sweden int `json:"Sweden"`
|
||||
Malawi int `json:"Malawi"`
|
||||
Andorra int `json:"Andorra"`
|
||||
Liechtenstein int `json:"Liechtenstein"`
|
||||
Poland int `json:"Poland"`
|
||||
UnitedStatesVirginIslands int `json:"United States Virgin Islands"`
|
||||
Jordan int `json:"Jordan"`
|
||||
Bulgaria int `json:"Bulgaria"`
|
||||
Tunisia int `json:"Tunisia"`
|
||||
CongoTheDemocraticRepublicOfThe int `json:"Congo, the Democratic Republic of the"`
|
||||
UnitedArabEmirates int `json:"United Arab Emirates"`
|
||||
Kenya int `json:"Kenya"`
|
||||
FrenchPolynesia int `json:"French Polynesia"`
|
||||
Lebanon int `json:"Lebanon"`
|
||||
Djibouti int `json:"Djibouti"`
|
||||
Azerbaijan int `json:"Azerbaijan"`
|
||||
CzechRepublic int `json:"Czech Republic"`
|
||||
SaintLucia int `json:"Saint Lucia"`
|
||||
Mauritania int `json:"Mauritania"`
|
||||
Guernsey int `json:"Guernsey"`
|
||||
Mayotte int `json:"Mayotte"`
|
||||
SanMarino int `json:"San Marino"`
|
||||
Israel int `json:"Israel"`
|
||||
Australia int `json:"Australia"`
|
||||
Tajikistan int `json:"Tajikistan"`
|
||||
Myanmar int `json:"Myanmar"`
|
||||
CentralAmerica int `json:"Central America"`
|
||||
EasternAfrica int `json:"Eastern Africa"`
|
||||
Cameroon int `json:"Cameroon"`
|
||||
Gibraltar int `json:"Gibraltar"`
|
||||
Cyprus int `json:"Cyprus"`
|
||||
Malaysia int `json:"Malaysia"`
|
||||
Oman int `json:"Oman"`
|
||||
Iceland int `json:"Iceland"`
|
||||
Armenia int `json:"Armenia"`
|
||||
Gabon int `json:"Gabon"`
|
||||
WesternAsia int `json:"Western Asia"`
|
||||
Polynesia int `json:"Polynesia"`
|
||||
Luxembourg int `json:"Luxembourg"`
|
||||
Brazil int `json:"Brazil"`
|
||||
TurksAndCaicosIslands int `json:"Turks and Caicos Islands"`
|
||||
Algeria int `json:"Algeria"`
|
||||
Slovenia int `json:"Slovenia"`
|
||||
Jersey int `json:"Jersey"`
|
||||
AntiguaAndBarbuda int `json:"Antigua and Barbuda"`
|
||||
Ecuador int `json:"Ecuador"`
|
||||
Colombia int `json:"Colombia"`
|
||||
Moldova int `json:"Moldova"`
|
||||
Vanuatu int `json:"Vanuatu"`
|
||||
UnitedStatesMinorOutlyingIslands int `json:"United States Minor Outlying Islands"`
|
||||
Italy int `json:"Italy"`
|
||||
Honduras int `json:"Honduras"`
|
||||
Haiti int `json:"Haiti"`
|
||||
Burundi int `json:"Burundi"`
|
||||
Singapore int `json:"Singapore"`
|
||||
FrenchGuiana int `json:"French Guiana"`
|
||||
AmericanSamoa int `json:"American Samoa"`
|
||||
Russia int `json:"Russia"`
|
||||
Netherlands int `json:"Netherlands"`
|
||||
China int `json:"China"`
|
||||
Martinique int `json:"Martinique"`
|
||||
SaintPierreAndMiquelon int `json:"Saint Pierre and Miquelon"`
|
||||
Kyrgyzstan int `json:"Kyrgyzstan"`
|
||||
Reunion int `json:"Reunion"`
|
||||
Bhutan int `json:"Bhutan"`
|
||||
Romania int `json:"Romania"`
|
||||
MiddleAfrica int `json:"Middle Africa"`
|
||||
Togo int `json:"Togo"`
|
||||
SouthernAsia int `json:"Southern Asia"`
|
||||
Philippines int `json:"Philippines"`
|
||||
CoteDIvoire int `json:"Cote d'Ivoire"`
|
||||
Uzbekistan int `json:"Uzbekistan"`
|
||||
Asia int `json:"Asia"`
|
||||
BritishVirginIslands int `json:"British Virgin Islands"`
|
||||
Zimbabwe int `json:"Zimbabwe"`
|
||||
BritishIndianOceanTerritory int `json:"British Indian Ocean Territory"`
|
||||
Montenegro int `json:"Montenegro"`
|
||||
Indonesia int `json:"Indonesia"`
|
||||
Dominica int `json:"Dominica"`
|
||||
Benin int `json:"Benin"`
|
||||
Angola int `json:"Angola"`
|
||||
EasternEurope int `json:"Eastern Europe"`
|
||||
Portugal int `json:"Portugal"`
|
||||
BruneiDarussalam int `json:"Brunei Darussalam"`
|
||||
NewCaledonia int `json:"New Caledonia"`
|
||||
Grenada int `json:"Grenada"`
|
||||
Greece int `json:"Greece"`
|
||||
CaymanIslands int `json:"Cayman Islands"`
|
||||
Mongolia int `json:"Mongolia"`
|
||||
Latvia int `json:"Latvia"`
|
||||
Morocco int `json:"Morocco"`
|
||||
Guyana int `json:"Guyana"`
|
||||
Guatemala int `json:"Guatemala"`
|
||||
Chile int `json:"Chile"`
|
||||
Nepal int `json:"Nepal"`
|
||||
NorthernEurope int `json:"Northern Europe"`
|
||||
IsleOfMan int `json:"Isle of Man"`
|
||||
Ukraine int `json:"Ukraine"`
|
||||
Tanzania int `json:"Tanzania"`
|
||||
Ghana int `json:"Ghana"`
|
||||
HolySee int `json:"Holy See"`
|
||||
CentralAsia int `json:"Central Asia"`
|
||||
Anguilla int `json:"Anguilla"`
|
||||
SouthEasternAsia int `json:"South-Eastern Asia"`
|
||||
India int `json:"India"`
|
||||
Canada int `json:"Canada"`
|
||||
Maldives int `json:"Maldives"`
|
||||
Turkey int `json:"Turkey"`
|
||||
Belgium int `json:"Belgium"`
|
||||
Taiwan int `json:"Taiwan"`
|
||||
TrinidadAndTobago int `json:"Trinidad and Tobago"`
|
||||
SouthernEurope int `json:"Southern Europe"`
|
||||
SouthAfrica int `json:"South Africa"`
|
||||
Bermuda int `json:"Bermuda"`
|
||||
AlandIslands int `json:"Aland Islands"`
|
||||
Jamaica int `json:"Jamaica"`
|
||||
Turkmenistan int `json:"Turkmenistan"`
|
||||
Peru int `json:"Peru"`
|
||||
Germany int `json:"Germany"`
|
||||
Americas int `json:"Americas"`
|
||||
Fiji int `json:"Fiji"`
|
||||
HongKong int `json:"Hong Kong"`
|
||||
UnitedStates int `json:"United States"`
|
||||
Guinea int `json:"Guinea"`
|
||||
MicronesiaFederatedStatesOf int `json:"Micronesia, Federated States of"`
|
||||
Somalia int `json:"Somalia"`
|
||||
Chad int `json:"Chad"`
|
||||
Thailand int `json:"Thailand"`
|
||||
Kiribati int `json:"Kiribati"`
|
||||
EquatorialGuinea int `json:"Equatorial Guinea"`
|
||||
CostaRica int `json:"Costa Rica"`
|
||||
SaintMartinFrenchPart int `json:"Saint Martin (French part)"`
|
||||
Vietnam int `json:"Vietnam"`
|
||||
Nigeria int `json:"Nigeria"`
|
||||
Kuwait int `json:"Kuwait"`
|
||||
Croatia int `json:"Croatia"`
|
||||
Uruguay int `json:"Uruguay"`
|
||||
SriLanka int `json:"Sri Lanka"`
|
||||
CookIslands int `json:"Cook Islands"`
|
||||
TimorLeste int `json:"Timor-Leste"`
|
||||
UnitedKingdom int `json:"United Kingdom"`
|
||||
Switzerland int `json:"Switzerland"`
|
||||
Samoa int `json:"Samoa"`
|
||||
Spain int `json:"Spain"`
|
||||
WesternAfrica int `json:"Western Africa"`
|
||||
Venezuela int `json:"Venezuela"`
|
||||
BurkinaFaso int `json:"Burkina Faso"`
|
||||
Swaziland int `json:"Swaziland"`
|
||||
Caribbean int `json:"Caribbean"`
|
||||
Estonia int `json:"Estonia"`
|
||||
Niue int `json:"Niue"`
|
||||
SouthKorea int `json:"South Korea"`
|
||||
Austria int `json:"Austria"`
|
||||
Mozambique int `json:"Mozambique"`
|
||||
ElSalvador int `json:"El Salvador"`
|
||||
Monaco int `json:"Monaco"`
|
||||
Guam int `json:"Guam"`
|
||||
Lesotho int `json:"Lesotho"`
|
||||
Tonga int `json:"Tonga"`
|
||||
NorthernAfrica int `json:"Northern Africa"`
|
||||
WesternSahara int `json:"Western Sahara"`
|
||||
Hungary int `json:"Hungary"`
|
||||
Japan int `json:"Japan"`
|
||||
Europe int `json:"Europe"`
|
||||
Curacao int `json:"Curacao"`
|
||||
Belarus int `json:"Belarus"`
|
||||
Mauritius int `json:"Mauritius"`
|
||||
WesternEurope int `json:"Western Europe"`
|
||||
Albania int `json:"Albania"`
|
||||
NewZealand int `json:"New Zealand"`
|
||||
NorthernAmerica int `json:"Northern America"`
|
||||
SintMaartenDutchPart int `json:"Sint Maarten (Dutch part)"`
|
||||
Senegal int `json:"Senegal"`
|
||||
Macedonia int `json:"Macedonia"`
|
||||
Ethiopia int `json:"Ethiopia"`
|
||||
Egypt int `json:"Egypt"`
|
||||
SierraLeone int `json:"Sierra Leone"`
|
||||
Bolivia int `json:"Bolivia"`
|
||||
Oceania int `json:"Oceania"`
|
||||
Malta int `json:"Malta"`
|
||||
SaudiArabia int `json:"Saudi Arabia"`
|
||||
CapeVerde int `json:"Cape Verde"`
|
||||
Pakistan int `json:"Pakistan"`
|
||||
Gambia int `json:"Gambia"`
|
||||
Qatar int `json:"Qatar"`
|
||||
Ireland int `json:"Ireland"`
|
||||
Slovakia int `json:"Slovakia"`
|
||||
Serbia int `json:"Serbia"`
|
||||
Lithuania int `json:"Lithuania"`
|
||||
France int `json:"France"`
|
||||
BosniaAndHerzegovina int `json:"Bosnia and Herzegovina"`
|
||||
AustraliaAndNewZealand int `json:"Australia and New Zealand"`
|
||||
Rwanda int `json:"Rwanda"`
|
||||
Bangladesh int `json:"Bangladesh"`
|
||||
Nicaragua int `json:"Nicaragua"`
|
||||
Barbados int `json:"Barbados"`
|
||||
Norway int `json:"Norway"`
|
||||
SouthernAfrica int `json:"Southern Africa"`
|
||||
SaintVincentAndTheGrenadines int `json:"Saint Vincent and the Grenadines"`
|
||||
Botswana int `json:"Botswana"`
|
||||
Melanesia int `json:"Melanesia"`
|
||||
Macao int `json:"Macao"`
|
||||
DominicanRepublic int `json:"Dominican Republic"`
|
||||
Denmark int `json:"Denmark"`
|
||||
Uganda int `json:"Uganda"`
|
||||
Mexico int `json:"Mexico"`
|
||||
Suriname int `json:"Suriname"`
|
||||
Micronesia int `json:"Micronesia"`
|
||||
Greenland int `json:"Greenland"`
|
||||
} `json:"location"`
|
||||
Timezone struct {
|
||||
int `json:""`
|
||||
AmericaSaoPaulo int `json:"America/Sao_Paulo"`
|
||||
AsiaVladivostok int `json:"Asia/Vladivostok"`
|
||||
EuropeBerlin int `json:"Europe/Berlin"`
|
||||
AfricaCairo int `json:"Africa/Cairo"`
|
||||
EuropeMoscow int `json:"Europe/Moscow"`
|
||||
PacificHonolulu int `json:"Pacific/Honolulu"`
|
||||
AustraliaHobart int `json:"Australia/Hobart"`
|
||||
EuropeLondon int `json:"Europe/London"`
|
||||
AsiaBaghdad int `json:"Asia/Baghdad"`
|
||||
AsiaShanghai int `json:"Asia/Shanghai"`
|
||||
AmericaTijuana int `json:"America/Tijuana"`
|
||||
AmericaManagua int `json:"America/Managua"`
|
||||
AsiaYerevan int `json:"Asia/Yerevan"`
|
||||
AsiaKamchatka int `json:"Asia/Kamchatka"`
|
||||
AfricaHarare int `json:"Africa/Harare"`
|
||||
AmericaNome int `json:"America/Nome"`
|
||||
AsiaYakutsk int `json:"Asia/Yakutsk"`
|
||||
AmericaChicago int `json:"America/Chicago"`
|
||||
AmericaHalifax int `json:"America/Halifax"`
|
||||
AmericaIndianaIndianapolis int `json:"America/Indiana/Indianapolis"`
|
||||
EuropeParis int `json:"Europe/Paris"`
|
||||
PacificFiji int `json:"Pacific/Fiji"`
|
||||
AsiaTehran int `json:"Asia/Tehran"`
|
||||
AmericaLaPaz int `json:"America/La_Paz"`
|
||||
AsiaTashkent int `json:"Asia/Tashkent"`
|
||||
AsiaBangkok int `json:"Asia/Bangkok"`
|
||||
PacificMidway int `json:"Pacific/Midway"`
|
||||
AmericaRecife int `json:"America/Recife"`
|
||||
AmericaBuenosAires int `json:"America/Buenos_Aires"`
|
||||
AustraliaAdelaide int `json:"Australia/Adelaide"`
|
||||
AsiaYangon int `json:"Asia/Yangon"`
|
||||
AsiaKatmandu int `json:"Asia/Katmandu"`
|
||||
AsiaAlmaty int `json:"Asia/Almaty"`
|
||||
AmericaPhoenix int `json:"America/Phoenix"`
|
||||
EuropePrague int `json:"Europe/Prague"`
|
||||
AmericaMexicoCity int `json:"America/Mexico_City"`
|
||||
AsiaTbilisi int `json:"Asia/Tbilisi"`
|
||||
AsiaJerusalem int `json:"Asia/Jerusalem"`
|
||||
EuropeLisbon int `json:"Europe/Lisbon"`
|
||||
AtlanticSouthGeorgia int `json:"Atlantic/South_Georgia"`
|
||||
AsiaKarachi int `json:"Asia/Karachi"`
|
||||
AustraliaPerth int `json:"Australia/Perth"`
|
||||
AustraliaDarwin int `json:"Australia/Darwin"`
|
||||
AsiaCalcutta int `json:"Asia/Calcutta"`
|
||||
AmericaBogota int `json:"America/Bogota"`
|
||||
AsiaKabul int `json:"Asia/Kabul"`
|
||||
AmericaNewYork int `json:"America/New_York"`
|
||||
AtlanticAzores int `json:"Atlantic/Azores"`
|
||||
AsiaKrasnoyarsk int `json:"Asia/Krasnoyarsk"`
|
||||
EET int `json:"EET"`
|
||||
PacificAuckland int `json:"Pacific/Auckland"`
|
||||
EuropeMinsk int `json:"Europe/Minsk"`
|
||||
AfricaCasablanca int `json:"Africa/Casablanca"`
|
||||
AmericaCaracas int `json:"America/Caracas"`
|
||||
EuropeKiev int `json:"Europe/Kiev"`
|
||||
AsiaMagadan int `json:"Asia/Magadan"`
|
||||
AmericaRegina int `json:"America/Regina"`
|
||||
PacificGuam int `json:"Pacific/Guam"`
|
||||
AsiaIrkutsk int `json:"Asia/Irkutsk"`
|
||||
AfricaAlgiers int `json:"Africa/Algiers"`
|
||||
AmericaStJohns int `json:"America/St_Johns"`
|
||||
AmericaFortaleza int `json:"America/Fortaleza"`
|
||||
AmericaDenver int `json:"America/Denver"`
|
||||
AmericaIndianaKnox int `json:"America/Indiana/Knox"`
|
||||
EtcUTC int `json:"Etc/UTC"`
|
||||
AustraliaSydney int `json:"Australia/Sydney"`
|
||||
PacificApia int `json:"Pacific/Apia"`
|
||||
AsiaTokyo int `json:"Asia/Tokyo"`
|
||||
AsiaIstanbul int `json:"Asia/Istanbul"`
|
||||
AsiaOmsk int `json:"Asia/Omsk"`
|
||||
AustraliaBrisbane int `json:"Australia/Brisbane"`
|
||||
AmericaLosAngeles int `json:"America/Los_Angeles"`
|
||||
AsiaYekaterinburg int `json:"Asia/Yekaterinburg"`
|
||||
EuropeAthens int `json:"Europe/Athens"`
|
||||
} `json:"timezone"`
|
||||
ConnectPrice struct {
|
||||
Num4 int `json:"4"`
|
||||
Num6 int `json:"6"`
|
||||
Zero2 int `json:"0-2"`
|
||||
} `json:"connectPrice"`
|
||||
ContractToHire struct {
|
||||
True int `json:"true"`
|
||||
False int `json:"false"`
|
||||
} `json:"contractToHire"`
|
||||
Categories []interface{} `json:"categories"`
|
||||
Subcategories []interface{} `json:"subcategories"`
|
||||
PaymentVerified struct {
|
||||
Num1 int `json:"1"`
|
||||
} `json:"paymentVerified"`
|
||||
} `json:"facets"`
|
||||
IsSearchWithEmptyParams bool `json:"isSearchWithEmptyParams"`
|
||||
CurrentQuery struct {
|
||||
Sort string `json:"sort"`
|
||||
} `json:"currentQuery"`
|
||||
QueryParsedParams struct {
|
||||
Sort string `json:"sort"`
|
||||
Paging string `json:"paging"`
|
||||
} `json:"queryParsedParams"`
|
||||
PageTitle string `json:"pageTitle"`
|
||||
JobSearchError bool `json:"jobSearchError"`
|
||||
RssLink string `json:"rssLink"`
|
||||
AtomLink string `json:"atomLink"`
|
||||
SmartSearch struct {
|
||||
DownloadTeamApplication bool `json:"downloadTeamApplication"`
|
||||
} `json:"smartSearch"`
|
||||
} `json:"searchResults"`
|
||||
Filters struct {
|
||||
Q string `json:"q"`
|
||||
Sort string `json:"sort"`
|
||||
Skills struct {
|
||||
Name string `json:"name"`
|
||||
Label string `json:"label"`
|
||||
Options []interface{} `json:"options"`
|
||||
} `json:"skills"`
|
||||
Categories []struct {
|
||||
URLName string `json:"urlName"`
|
||||
Value string `json:"value"`
|
||||
Label string `json:"label"`
|
||||
ActiveLabel string `json:"activeLabel"`
|
||||
Checked bool `json:"checked"`
|
||||
Count int `json:"count"`
|
||||
Subcategories []struct {
|
||||
URLName string `json:"urlName"`
|
||||
Value string `json:"value"`
|
||||
Label string `json:"label"`
|
||||
ActiveLabel string `json:"activeLabel"`
|
||||
Checked bool `json:"checked"`
|
||||
Count int `json:"count"`
|
||||
} `json:"subcategories"`
|
||||
} `json:"categories"`
|
||||
JobType struct {
|
||||
Name string `json:"name"`
|
||||
Label string `json:"label"`
|
||||
LoggingSublocation string `json:"loggingSublocation"`
|
||||
LoggingLabel string `json:"loggingLabel"`
|
||||
Options []struct {
|
||||
Value string `json:"value"`
|
||||
Label string `json:"label"`
|
||||
Checked bool `json:"checked"`
|
||||
Count int `json:"count"`
|
||||
} `json:"options"`
|
||||
} `json:"jobType"`
|
||||
ContractorTier struct {
|
||||
Name string `json:"name"`
|
||||
Label string `json:"label"`
|
||||
LoggingSublocation string `json:"loggingSublocation"`
|
||||
LoggingLabel string `json:"loggingLabel"`
|
||||
Options []struct {
|
||||
Value string `json:"value"`
|
||||
Label string `json:"label"`
|
||||
Checked bool `json:"checked"`
|
||||
Count int `json:"count"`
|
||||
} `json:"options"`
|
||||
} `json:"contractorTier"`
|
||||
ClientHires struct {
|
||||
Name string `json:"name"`
|
||||
Label string `json:"label"`
|
||||
LoggingSublocation string `json:"loggingSublocation"`
|
||||
LoggingLabel string `json:"loggingLabel"`
|
||||
Options []struct {
|
||||
Value string `json:"value"`
|
||||
Label string `json:"label"`
|
||||
Checked bool `json:"checked"`
|
||||
Count int `json:"count"`
|
||||
} `json:"options"`
|
||||
} `json:"clientHires"`
|
||||
Proposals struct {
|
||||
Name string `json:"name"`
|
||||
Label string `json:"label"`
|
||||
LoggingSublocation string `json:"loggingSublocation"`
|
||||
LoggingLabel string `json:"loggingLabel"`
|
||||
Options []struct {
|
||||
Value string `json:"value"`
|
||||
Label string `json:"label"`
|
||||
Checked bool `json:"checked"`
|
||||
Count int `json:"count"`
|
||||
ActiveLabel string `json:"activeLabel"`
|
||||
} `json:"options"`
|
||||
} `json:"proposals"`
|
||||
Amount struct {
|
||||
Name string `json:"name"`
|
||||
Label string `json:"label"`
|
||||
LoggingSublocation string `json:"loggingSublocation"`
|
||||
DisabledJobTypeValue string `json:"disabledJobTypeValue"`
|
||||
DisabledMessage string `json:"disabledMessage"`
|
||||
Options []struct {
|
||||
Value string `json:"value"`
|
||||
Label string `json:"label"`
|
||||
Checked bool `json:"checked"`
|
||||
Count int `json:"count"`
|
||||
} `json:"options"`
|
||||
CustomAmount struct {
|
||||
Value interface{} `json:"value"`
|
||||
Label string `json:"label"`
|
||||
Checked bool `json:"checked"`
|
||||
CustomMin interface{} `json:"customMin"`
|
||||
CustomMax interface{} `json:"customMax"`
|
||||
} `json:"customAmount"`
|
||||
} `json:"amount"`
|
||||
Workload struct {
|
||||
Name string `json:"name"`
|
||||
Label string `json:"label"`
|
||||
LoggingSublocation string `json:"loggingSublocation"`
|
||||
LoggingLabel string `json:"loggingLabel"`
|
||||
Options []struct {
|
||||
Value string `json:"value"`
|
||||
Label string `json:"label"`
|
||||
Checked bool `json:"checked"`
|
||||
Count int `json:"count"`
|
||||
} `json:"options"`
|
||||
DisabledJobTypeValue string `json:"disabledJobTypeValue"`
|
||||
DisabledMessage string `json:"disabledMessage"`
|
||||
} `json:"workload"`
|
||||
DurationV3 struct {
|
||||
Name string `json:"name"`
|
||||
Label string `json:"label"`
|
||||
LoggingSublocation string `json:"loggingSublocation"`
|
||||
LoggingLabel string `json:"loggingLabel"`
|
||||
Options []struct {
|
||||
Value string `json:"value"`
|
||||
Label string `json:"label"`
|
||||
Checked bool `json:"checked"`
|
||||
Count int `json:"count"`
|
||||
} `json:"options"`
|
||||
DisabledJobTypeValue string `json:"disabledJobTypeValue"`
|
||||
DisabledMessage string `json:"disabledMessage"`
|
||||
} `json:"duration_v3"`
|
||||
PreviousClients struct {
|
||||
Name string `json:"name"`
|
||||
Label string `json:"label"`
|
||||
LoggingSublocation string `json:"loggingSublocation"`
|
||||
Value string `json:"value"`
|
||||
Checked bool `json:"checked"`
|
||||
Count int `json:"count"`
|
||||
} `json:"previousClients"`
|
||||
PaymentVerified struct {
|
||||
Name string `json:"name"`
|
||||
Label string `json:"label"`
|
||||
LoggingSublocation string `json:"loggingSublocation"`
|
||||
Value string `json:"value"`
|
||||
Checked bool `json:"checked"`
|
||||
Count int `json:"count"`
|
||||
} `json:"paymentVerified"`
|
||||
TalentClouds struct {
|
||||
Name string `json:"name"`
|
||||
Label string `json:"label"`
|
||||
LoggingSublocation string `json:"loggingSublocation"`
|
||||
Options []interface{} `json:"options"`
|
||||
} `json:"talentClouds"`
|
||||
UserLocationMatch struct {
|
||||
Name string `json:"name"`
|
||||
Label string `json:"label"`
|
||||
LoggingSublocation string `json:"loggingSublocation"`
|
||||
Value string `json:"value"`
|
||||
Checked bool `json:"checked"`
|
||||
Count int `json:"count"`
|
||||
} `json:"userLocationMatch"`
|
||||
Occupation struct {
|
||||
Name string `json:"name"`
|
||||
Label string `json:"label"`
|
||||
LoggingSublocation string `json:"loggingSublocation"`
|
||||
Options []interface{} `json:"options"`
|
||||
} `json:"occupation"`
|
||||
OntologySkills struct {
|
||||
Name string `json:"name"`
|
||||
Label string `json:"label"`
|
||||
LoggingSublocation string `json:"loggingSublocation"`
|
||||
Options []interface{} `json:"options"`
|
||||
} `json:"ontologySkills"`
|
||||
HourlyRate struct {
|
||||
Name string `json:"name"`
|
||||
Label string `json:"label"`
|
||||
LoggingSublocation string `json:"loggingSublocation"`
|
||||
Options []interface{} `json:"options"`
|
||||
CustomHourlyRate struct {
|
||||
Value interface{} `json:"value"`
|
||||
Label string `json:"label"`
|
||||
Checked bool `json:"checked"`
|
||||
CustomMin interface{} `json:"customMin"`
|
||||
CustomMax interface{} `json:"customMax"`
|
||||
} `json:"customHourlyRate"`
|
||||
} `json:"hourly_rate"`
|
||||
FreelancersNeeded struct {
|
||||
Name string `json:"name"`
|
||||
Label string `json:"label"`
|
||||
LoggingSublocation string `json:"loggingSublocation"`
|
||||
LoggingLabel string `json:"loggingLabel"`
|
||||
Options []struct {
|
||||
Value string `json:"value"`
|
||||
Label string `json:"label"`
|
||||
Checked bool `json:"checked"`
|
||||
Count int `json:"count"`
|
||||
} `json:"options"`
|
||||
} `json:"freelancers_needed"`
|
||||
Location struct {
|
||||
Name string `json:"name"`
|
||||
Label string `json:"label"`
|
||||
LoggingSublocation string `json:"loggingSublocation"`
|
||||
Options []struct {
|
||||
UID string `json:"uid"`
|
||||
Label string `json:"label"`
|
||||
SubRegionUID string `json:"subRegionUid,omitempty"`
|
||||
RegionUID string `json:"regionUid,omitempty"`
|
||||
Value string `json:"value"`
|
||||
Type string `json:"type"`
|
||||
Checked bool `json:"checked"`
|
||||
Count int `json:"count"`
|
||||
} `json:"options"`
|
||||
} `json:"location"`
|
||||
Timezones struct {
|
||||
Name string `json:"name"`
|
||||
Label string `json:"label"`
|
||||
LoggingSublocation string `json:"loggingSublocation"`
|
||||
Options []struct {
|
||||
Value string `json:"value"`
|
||||
Label string `json:"label"`
|
||||
Checked bool `json:"checked"`
|
||||
Count int `json:"count"`
|
||||
} `json:"options"`
|
||||
} `json:"timezones"`
|
||||
Connects struct {
|
||||
Name string `json:"name"`
|
||||
Label string `json:"label"`
|
||||
LoggingSublocation string `json:"loggingSublocation"`
|
||||
LoggingLabel string `json:"loggingLabel"`
|
||||
Options []struct {
|
||||
Value string `json:"value"`
|
||||
Label string `json:"label"`
|
||||
Checked bool `json:"checked"`
|
||||
Count int `json:"count"`
|
||||
} `json:"options"`
|
||||
} `json:"connects"`
|
||||
ContractToHire struct {
|
||||
Name string `json:"name"`
|
||||
Label string `json:"label"`
|
||||
LoggingSublocation string `json:"loggingSublocation"`
|
||||
Value string `json:"value"`
|
||||
Checked bool `json:"checked"`
|
||||
Count int `json:"count"`
|
||||
} `json:"contractToHire"`
|
||||
} `json:"filters"`
|
||||
}
|
||||
@@ -46,8 +46,6 @@ func InitUpwork() *Upwork {
|
||||
"vnd-eo-span-id": "0d5bed38-342c-4b44-a885-b9c3de71a32a",
|
||||
"x-odesk-user-agent": "oDesk LM",
|
||||
"x-requested-with": "XMLHttpRequest",
|
||||
"authorization": "Bearer oauth2v2_029104aca6380322c31d5ff1beb45bdc",
|
||||
"cookie": "visitor_id=84.239.14.170.1672593945152000; lang=en; visitor_gql_token=oauth2v2_73062637ed25bfaab8a975d46e74a2a6; cookie_prefix=; cookie_domain=.upwork.com; __cf_bm=CeIOv6_PAxdKYRcfxgS.0i5S6T7H_FtSQJIRJ7TfxZE-1672593945-0-AWkqgfz2NFzVJjyMCtqW9XH5v601Yw3Faj7Rnd/lalOGPrlRTLVeEMwwxlSd8rxVIx7Gxmeil0zA3eoueK2DafA=; __cfruid=ea76c3d0a524b002bd34afa80c598fda56e6ec51-1672593945; lang=en; _sp_ses.2a16=*; G_ENABLED_IDPS=google; _cq_duid=1.1672593953.libVfRjYZI9g1VVb; _cq_suid=1.1672593953.iSaHfjf7kYLkR6pW; OptanonAlertBoxClosed=2023-01-01T17:26:03.771Z; odesk_signup.referer.raw=https://www.upwork.com/ab/account-security/login; device_view=full; recognized=66e1fe97; console_user=66e1fe97; user_uid=1561687738217156608; master_access_token=fff79f08.oauth2v2_96a3e8bf07ca7af5426f97ae5f3462a8; oauth2_global_js_token=oauth2v2_029104aca6380322c31d5ff1beb45bdc; SZ=8d599fc6dcae93ee0a7d31376c071dd5c53432dd15d764e8bcc69ecfa8c138aa; company_last_accessed=d1020106921; current_organization_uid=1561687738217156609; user_oauth2_slave_access_token=fff79f08.oauth2v2_96a3e8bf07ca7af5426f97ae5f3462a8:1561687738217156608.oauth2v2_11b83ba7d6c21a48610deaa621b1763b; visitor_account_security_token=oauth2v2_f14381d2a80fa116031ad0a28ea995dd; XSRF-TOKEN=3d96f0d3ccf4b54113e78176144f442b; jsTSize_1561687738217156608=2; channel=other; visitor_ui_gql_token_jobsearch=oauth2v2_492e535ef89cfea762ce623120e9f3ce; forterToken=964af2adf2e946bfa97715b691a68da4_1672594152809_30_UAS_14ck; OptanonConsent=isGpcEnabled=1&datestamp=Sun+Jan+01+2023+22:29:15+GMT+0500+(Pakistan+Standard+Time)&version=6.39.0&isIABGlobal=false&hosts=&consentId=50406b2b-45ad-45e3-a842-f3a0feed1af9&interactionCount=1&landingPath=NotLandingPage&groups=C0001:1,C0002:0,C0003:0,C0004:0&geolocation=RO;&AwaitingReconsent=false; _sp_id.2a16=afb5f089-5283-4f0c-a133-b22247a07924.1672593950.1.1672594164..0587ef56-a7dc-4f9e-bf69-87dc9b20091c..473139f6-344e-4278-8d49-d8d489e57cfb.1672594154654.8; AWSALB=hklmGZU3Xl6e/0vh9R+DEs8vb5000Uvr68FRLtAMSg8Rg6zJNTEB6uCIalp51mRkZoEgxVAUWfVK0aEtAotogkeLDPWDCv6VKFvJ/PlsbWBYTuNzvfDgC071eRXa; AWSALBCORS=hklmGZU3Xl6e/0vh9R+DEs8vb5000Uvr68FRLtAMSg8Rg6zJNTEB6uCIalp51mRkZoEgxVAUWfVK0aEtAotogkeLDPWDCv6VKFvJ/PlsbWBYTuNzvfDgC071eRXa; spt=37d89b33-2cbd-43c3-a090-aa585d1f4300; enabled_ff=CI9570Air2Dot5,OTBnrOn,!MP16400Air3Migration,!air2Dot76Qt,air2Dot76,!CI10270Air2Dot5QTAllocations,!CI10857Air3Dot0,!CI12577UniversalSearch,CI11132Air2Dot75,SSINav",
|
||||
}
|
||||
client := network.InitClient(headers)
|
||||
upwork := Upwork{
|
||||
|
||||
@@ -2,21 +2,21 @@ package upwork
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
type UpworkPipeLine struct {
|
||||
upworkClient *Upwork
|
||||
iterations int
|
||||
// queue := []int{}
|
||||
filepaths []string
|
||||
}
|
||||
|
||||
func InitPipeline() *UpworkPipeLine {
|
||||
u := UpworkPipeLine{
|
||||
upworkClient: InitUpwork(),
|
||||
iterations: 32,
|
||||
}
|
||||
return &u
|
||||
}
|
||||
@@ -24,12 +24,12 @@ func InitPipeline() *UpworkPipeLine {
|
||||
func (u *UpworkPipeLine) CombineFiles() error {
|
||||
|
||||
var all_jobs []interface{}
|
||||
all_filenames, err := filepath.Glob("data/*.json")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
// all_filenames, err := filepath.Glob("data/*.json")
|
||||
// if err != nil {
|
||||
// panic(err)
|
||||
// }
|
||||
|
||||
for _, file := range all_filenames {
|
||||
for _, file := range u.filepaths {
|
||||
data, err := os.ReadFile(file)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
@@ -83,11 +83,15 @@ func isJSON(s string) bool {
|
||||
}
|
||||
|
||||
func isApiError(data string) bool {
|
||||
|
||||
var result map[string]interface{}
|
||||
|
||||
//Parse data as json
|
||||
err := json.Unmarshal([]byte(data), &result)
|
||||
if err != nil {
|
||||
log.Print("The Api did not return expected response")
|
||||
log.Print("The following was the response from API")
|
||||
log.Print(data)
|
||||
panic(err)
|
||||
}
|
||||
|
||||
@@ -106,6 +110,32 @@ func (u *UpworkPipeLine) validateResponse(data string) bool {
|
||||
|
||||
}
|
||||
|
||||
func (u *UpworkPipeLine) getTotalDocuments(urlArgs UrlArgs) (int, error) {
|
||||
client := u.upworkClient
|
||||
url := client.ConstructUrl(urlArgs)
|
||||
resp, err := u.upworkClient.SendRequest(url)
|
||||
|
||||
if err != nil {
|
||||
|
||||
log.Fatal(err)
|
||||
return 0, err
|
||||
}
|
||||
|
||||
// check if response is valid
|
||||
if !u.validateResponse(resp) {
|
||||
return 0, fmt.Errorf("invalid response")
|
||||
}
|
||||
|
||||
var API_Response UpworkApiResponse
|
||||
|
||||
json.Unmarshal([]byte(resp), &API_Response)
|
||||
|
||||
total_docs := API_Response.SearchResults.Paging.Total
|
||||
s := fmt.Sprintf("totalzz is %d", total_docs)
|
||||
log.Print(s)
|
||||
return total_docs, nil
|
||||
}
|
||||
|
||||
func (u *UpworkPipeLine) handleRequest(urlArgs UrlArgs, iteration int) {
|
||||
client := u.upworkClient
|
||||
url := client.ConstructUrl(urlArgs)
|
||||
@@ -120,7 +150,12 @@ func (u *UpworkPipeLine) handleRequest(urlArgs UrlArgs, iteration int) {
|
||||
err = u.saveToFile([]byte(resp), filename)
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
panic(errors.New("could not save file"))
|
||||
} else {
|
||||
u.filepaths = append(u.filepaths, filename)
|
||||
|
||||
}
|
||||
|
||||
} else {
|
||||
log.Println("Invalid response returned")
|
||||
|
||||
@@ -140,27 +175,64 @@ func (u *UpworkPipeLine) Run(query string) error {
|
||||
}
|
||||
|
||||
var iteration int
|
||||
for iteration = 1; iteration <= u.iterations; iteration++ {
|
||||
log.Println("Iteration: ", iteration)
|
||||
var perPage int
|
||||
var total_docs int
|
||||
|
||||
info_message := fmt.Sprintf("Finding Total Jobs for %s", query)
|
||||
fmt.Println(info_message)
|
||||
|
||||
urlArgs := UrlArgs{
|
||||
Page: 1,
|
||||
Per_Page: 10,
|
||||
Query: query,
|
||||
}
|
||||
|
||||
//Find total number of iterations
|
||||
perPage = 50
|
||||
total_docs, err = u.getTotalDocuments(urlArgs)
|
||||
|
||||
log.Print(total_docs)
|
||||
if err == nil {
|
||||
info_message := fmt.Sprintf("%s has a total of %d jobs", query, total_docs)
|
||||
fmt.Println(info_message)
|
||||
iteration = total_docs / perPage
|
||||
|
||||
if iteration >= 100 {
|
||||
iteration = 100
|
||||
}
|
||||
|
||||
info_message = fmt.Sprintf("A total of %d iterations will be performed", iteration)
|
||||
fmt.Println(info_message)
|
||||
|
||||
//Found total iterations
|
||||
u.handledataIteration(perPage, iteration, query)
|
||||
err = u.CombineFiles()
|
||||
os.RemoveAll("data")
|
||||
|
||||
if err != nil {
|
||||
panic(err)
|
||||
|
||||
}
|
||||
|
||||
return nil
|
||||
|
||||
} else {
|
||||
log.Fatal("Could not retrive total number of jobs")
|
||||
panic(err)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func (u *UpworkPipeLine) handledataIteration(p_per int, iters int, query string) {
|
||||
for index := 1; index <= iters; index++ {
|
||||
log.Println("Iteration: ", index)
|
||||
urlArgs := UrlArgs{
|
||||
Page: iteration,
|
||||
Page: index,
|
||||
Per_Page: 50,
|
||||
Query: query,
|
||||
}
|
||||
//It is possible to use a go routine here but be nice to the api or you will be rate limited pretty quickly. It is technically possible to circumvent it using a proxy but it is not recommended.
|
||||
// go u.handleRequest(urlArgs, i)
|
||||
u.handleRequest(urlArgs, iteration)
|
||||
|
||||
u.handleRequest(urlArgs, index)
|
||||
|
||||
}
|
||||
|
||||
err = u.CombineFiles()
|
||||
os.RemoveAll("data")
|
||||
|
||||
if err != nil {
|
||||
panic(err)
|
||||
|
||||
}
|
||||
|
||||
return nil
|
||||
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user