Added types for upwork api respone

upwork/upworkPipeline.go

@@ -2,21 +2,21 @@ package upwork
 
 import (
 	"encoding/json"
+	"errors"
 	"fmt"
 	"log"
 	"os"
-	"path/filepath"
 )
 
 type UpworkPipeLine struct {
 	upworkClient *Upwork
-	iterations   int
+	// queue := []int{}
+	filepaths []string
 }
 
 func InitPipeline() *UpworkPipeLine {
 	u := UpworkPipeLine{
 		upworkClient: InitUpwork(),
-		iterations:   32,
 	}
 	return &u
 }
@@ -24,12 +24,12 @@ func InitPipeline() *UpworkPipeLine {
 func (u *UpworkPipeLine) CombineFiles() error {
 
 	var all_jobs []interface{}
-	all_filenames, err := filepath.Glob("data/*.json")
-	if err != nil {
-		panic(err)
-	}
+	// all_filenames, err := filepath.Glob("data/*.json")
+	// if err != nil {
+	// 	panic(err)
+	// }
 
-	for _, file := range all_filenames {
+	for _, file := range u.filepaths {
 		data, err := os.ReadFile(file)
 		if err != nil {
 			panic(err)
@@ -83,11 +83,15 @@ func isJSON(s string) bool {
 }
 
 func isApiError(data string) bool {
+
 	var result map[string]interface{}
 
 	//Parse data as json
 	err := json.Unmarshal([]byte(data), &result)
 	if err != nil {
+		log.Print("The Api did not return expected response")
+		log.Print("The following was the response from API")
+		log.Print(data)
 		panic(err)
 	}
 
@@ -106,6 +110,32 @@ func (u *UpworkPipeLine) validateResponse(data string) bool {
 
 }
 
+func (u *UpworkPipeLine) getTotalDocuments(urlArgs UrlArgs) (int, error) {
+	client := u.upworkClient
+	url := client.ConstructUrl(urlArgs)
+	resp, err := u.upworkClient.SendRequest(url)
+
+	if err != nil {
+
+		log.Fatal(err)
+		return 0, err
+	}
+
+	// check if response is valid
+	if !u.validateResponse(resp) {
+		return 0, fmt.Errorf("invalid response")
+	}
+
+	var API_Response UpworkApiResponse
+
+	json.Unmarshal([]byte(resp), &API_Response)
+
+	total_docs := API_Response.SearchResults.Paging.Total
+	s := fmt.Sprintf("totalzz is %d", total_docs)
+	log.Print(s)
+	return total_docs, nil
+}
+
 func (u *UpworkPipeLine) handleRequest(urlArgs UrlArgs, iteration int) {
 	client := u.upworkClient
 	url := client.ConstructUrl(urlArgs)
@@ -120,7 +150,12 @@ func (u *UpworkPipeLine) handleRequest(urlArgs UrlArgs, iteration int) {
 		err = u.saveToFile([]byte(resp), filename)
 		if err != nil {
 			log.Println(err)
+			panic(errors.New("could not save file"))
+		} else {
+			u.filepaths = append(u.filepaths, filename)
+
 		}
+
 	} else {
 		log.Println("Invalid response returned")
 
@@ -140,27 +175,64 @@ func (u *UpworkPipeLine) Run(query string) error {
 	}
 
 	var iteration int
-	for iteration = 1; iteration <= u.iterations; iteration++ {
-		log.Println("Iteration: ", iteration)
+	var perPage int
+	var total_docs int
+
+	info_message := fmt.Sprintf("Finding Total Jobs for %s", query)
+	fmt.Println(info_message)
+
+	urlArgs := UrlArgs{
+		Page:     1,
+		Per_Page: 10,
+		Query:    query,
+	}
+
+	//Find total number of iterations
+	perPage = 50
+	total_docs, err = u.getTotalDocuments(urlArgs)
+
+	log.Print(total_docs)
+	if err == nil {
+		info_message := fmt.Sprintf("%s has a total of %d jobs", query, total_docs)
+		fmt.Println(info_message)
+		iteration = total_docs / perPage
+
+		if iteration >= 100 {
+			iteration = 100
+		}
+
+		info_message = fmt.Sprintf("A total of %d iterations will be performed", iteration)
+		fmt.Println(info_message)
+
+		//Found total iterations
+		u.handledataIteration(perPage, iteration, query)
+		err = u.CombineFiles()
+		os.RemoveAll("data")
+
+		if err != nil {
+			panic(err)
+
+		}
+
+		return nil
+
+	} else {
+		log.Fatal("Could not retrive total number of jobs")
+		panic(err)
+	}
+
+}
+
+func (u *UpworkPipeLine) handledataIteration(p_per int, iters int, query string) {
+	for index := 1; index <= iters; index++ {
+		log.Println("Iteration: ", index)
 		urlArgs := UrlArgs{
-			Page:     iteration,
+			Page:     index,
 			Per_Page: 50,
 			Query:    query,
 		}
-		//It is possible to use a go routine here but be nice to the api or you will be rate limited pretty quickly. It is technically possible to circumvent it using a proxy but it is not recommended.
-		// go u.handleRequest(urlArgs, i)
-		u.handleRequest(urlArgs, iteration)
+
+		u.handleRequest(urlArgs, index)
 
 	}
-
-	err = u.CombineFiles()
-	os.RemoveAll("data")
-
-	if err != nil {
-		panic(err)
-
-	}
-
-	return nil
-
 }