Initial commit still with some hardcoded things

scraper/scrapeDeveloperNifs.go

@@ -0,0 +1 @@
+package scraper

scraper/scraper.go

@@ -0,0 +1,82 @@
+// Package scraper is a helper package to scrape the interwebs.
+package scraper
+
+import (
+	"log"
+	"strconv"
+	"time"
+
+	models "github.com/ESilva15/PTInvCertSWDB/sw/models"
+	"github.com/gocolly/colly"
+)
+
+const (
+	protocol = "https://"
+	domain   = "www.portaldasfinancas.gov.pt"
+	resource = "pt/consultaProgCertificadosM24.action"
+)
+
+var tableResource = domain + "/" + resource
+
+// Scrape will scrape the remote tableResource to get the registerd SWs
+func Scrape() models.SWList {
+	c := colly.NewCollector(colly.AllowedDomains(domain))
+
+	var softwares models.SWList
+
+	c.OnRequest(func(r *colly.Request) {
+		log.Println("Scraping:", r.URL)
+	})
+
+	c.OnResponse(func(r *colly.Response) {
+		log.Println("Scraping:", r.StatusCode)
+	})
+
+	c.OnError(func(r *colly.Response, err error) {
+		log.Println("Request URL:", r.Request.URL, "failed with response:", r,
+			"\nError:", err)
+	})
+
+	states := make(map[string]int)
+
+	c.OnHTML("#m24Table > tbody", func(e *colly.HTMLElement) {
+		log.Println("Looping over table")
+		e.ForEach("tr", func(_ int, el *colly.HTMLElement) {
+			sw := models.SW{}
+
+			certNo, err := strconv.Atoi(el.ChildText("td:nth-child(4)"))
+			if err != nil {
+				log.Println("failed to get certNo from:", el.ChildText("td:nth-child(4)"))
+				return
+			}
+
+			date, err := time.Parse("2006-01-02", el.ChildText("td:nth-child(6)"))
+			if err != nil {
+				log.Println("Failed to get date from:", el.ChildText("td:nth-child(6)"))
+				return
+			}
+
+			sw.Name = el.ChildText("td:nth-child(1)")
+			sw.Version = el.ChildText("td:nth-child(2)")
+			sw.Developer = el.ChildText("td:nth-child(3)")
+			sw.CertNo = certNo
+			sw.State = el.ChildText("td:nth-child(5)")
+			sw.CertDate = date
+
+			if _, ok := states[sw.State]; ok {
+				states[sw.State] += 1
+			} else {
+				states[sw.State] = 0
+			}
+
+			softwares = append(softwares, sw)
+		})
+	})
+
+	err := c.Visit(protocol + tableResource)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	return softwares
+}