83 lines
1.9 KiB
Go
83 lines
1.9 KiB
Go
// Package scraper is a helper package to scrape the interwebs.
|
|
package scraper
|
|
|
|
import (
|
|
"log"
|
|
"strconv"
|
|
"time"
|
|
|
|
models "github.com/ESilva15/PTInvCertSWDB/sw/models"
|
|
"github.com/gocolly/colly"
|
|
)
|
|
|
|
const (
|
|
protocol = "https://"
|
|
domain = "www.portaldasfinancas.gov.pt"
|
|
resource = "pt/consultaProgCertificadosM24.action"
|
|
)
|
|
|
|
var tableResource = domain + "/" + resource
|
|
|
|
// Scrape will scrape the remote tableResource to get the registerd SWs
|
|
func Scrape() models.SWList {
|
|
c := colly.NewCollector(colly.AllowedDomains(domain))
|
|
|
|
var softwares models.SWList
|
|
|
|
c.OnRequest(func(r *colly.Request) {
|
|
log.Println("Scraping:", r.URL)
|
|
})
|
|
|
|
c.OnResponse(func(r *colly.Response) {
|
|
log.Println("Scraping:", r.StatusCode)
|
|
})
|
|
|
|
c.OnError(func(r *colly.Response, err error) {
|
|
log.Println("Request URL:", r.Request.URL, "failed with response:", r,
|
|
"\nError:", err)
|
|
})
|
|
|
|
states := make(map[string]int)
|
|
|
|
c.OnHTML("#m24Table > tbody", func(e *colly.HTMLElement) {
|
|
log.Println("Looping over table")
|
|
e.ForEach("tr", func(_ int, el *colly.HTMLElement) {
|
|
sw := models.SW{}
|
|
|
|
certNo, err := strconv.Atoi(el.ChildText("td:nth-child(4)"))
|
|
if err != nil {
|
|
log.Println("failed to get certNo from:", el.ChildText("td:nth-child(4)"))
|
|
return
|
|
}
|
|
|
|
date, err := time.Parse("2006-01-02", el.ChildText("td:nth-child(6)"))
|
|
if err != nil {
|
|
log.Println("Failed to get date from:", el.ChildText("td:nth-child(6)"))
|
|
return
|
|
}
|
|
|
|
sw.Name = el.ChildText("td:nth-child(1)")
|
|
sw.Version = el.ChildText("td:nth-child(2)")
|
|
sw.Developer.Name = el.ChildText("td:nth-child(3)")
|
|
sw.CertNo = certNo
|
|
sw.State = el.ChildText("td:nth-child(5)")
|
|
sw.CertDate = date
|
|
|
|
if _, ok := states[sw.State]; ok {
|
|
states[sw.State] += 1
|
|
} else {
|
|
states[sw.State] = 0
|
|
}
|
|
|
|
softwares = append(softwares, sw)
|
|
})
|
|
})
|
|
|
|
err := c.Visit(protocol + tableResource)
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
|
|
return softwares
|
|
}
|