diff --git a/.gitignore b/.gitignore index 00fca87..8dc6725 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,8 @@ _private.js _private.json a_private.go banned.list +codetabsData.db3 +*.db3 # # diff --git a/README.md b/README.md index 4458c41..2e31cce 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,11 @@ -![Version](https://img.shields.io/badge/version-0.7.20-orange.svg) +![Version](https://img.shields.io/badge/version-0.8.0-orange.svg) ![Maintained YES](https://img.shields.io/badge/Maintained%3F-yes-green.svg) ![Ask Me Anything !](https://img.shields.io/badge/Ask%20me-anything-1abc9c.svg) # ![logo](https://github.com/jolav/codetabs/blob/master/www/_public/icons/ct/ct64r.png?raw=true) **ONLINE TOOLS ([codetabs.com](https://codetabs.com))** -**version 0.7.20** +**version 0.8.0** 1. [Count LOC (lines of code) online from github/gitlab repos or zipped uploaded folder](#count-loc-online) 2. [CORS proxy](#cors-proxy) diff --git a/geolocation/geolocation.go b/geolocation/geolocation.go index 9d60ae7..34a0ebf 100644 --- a/geolocation/geolocation.go +++ b/geolocation/geolocation.go @@ -93,7 +93,7 @@ func (g *geoip) getGeoDataFromDB() { g.cleanGeoData() return } - u.PrettyPrintStruct(results) + //u.PrettyPrintStruct(results) g.geoData.Country_code = results.Country_short g.geoData.Country_name = results.Country_long g.geoData.Region_name = results.Region diff --git a/go.mod b/go.mod index 8556547..c24263d 100644 --- a/go.mod +++ b/go.mod @@ -2,4 +2,7 @@ module github.com/jolav/codetabs go 1.14 -require github.com/ip2location/ip2location-go/v9 v9.1.0 +require ( + github.com/ip2location/ip2location-go/v9 v9.1.0 + github.com/mattn/go-sqlite3 v1.14.16 // indirect +) diff --git a/go.sum b/go.sum index 03c0ed5..d82cce8 100644 --- a/go.sum +++ b/go.sum @@ -1,2 +1,4 @@ github.com/ip2location/ip2location-go/v9 v9.1.0 h1:DVlKxIcjA7CsA0cgzbidwe6eKzpbkS313LsH9ceutxI= github.com/ip2location/ip2location-go/v9 v9.1.0/go.mod h1:s5SV6YZL10TpfPpXw//7fEJC65G/yH7Oh+Tjq9JcQEQ= +github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y= +github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= diff --git a/loc/loc.go b/loc/loc.go index 0131788..c283fca 100644 --- a/loc/loc.go +++ b/loc/loc.go @@ -11,8 +11,10 @@ import ( "os" "strconv" "strings" + "time" u "github.com/jolav/codetabs/_utils" + "github.com/jolav/codetabs/store" ) const ( @@ -84,6 +86,7 @@ func (l *loc) Router(w http.ResponseWriter, r *http.Request) { l.doLocUploadRequest(w, r) return } + r.ParseForm() l.branch = r.Form.Get("branch") if r.Form.Get("ignored") != "" { @@ -134,6 +137,7 @@ func (l *loc) doLocRepoRequest(w http.ResponseWriter, r *http.Request) { total2.Lines += v.Lines } l.languagesOUT = append(l.languagesOUT, total2) + l.storeData() u.SendJSONToClient(w, l.languagesOUT, 200) return */ @@ -218,7 +222,12 @@ func (l *loc) doLocRepoRequest(w http.ResponseWriter, r *http.Request) { total.Lines += v.Lines } l.languagesOUT = append(l.languagesOUT, total) - + if l.branch == "" { // cant call gitlab with master + l.branch = "master" + } + if len(l.ignored) == 0 && l.branch == "master" { + l.storeData() + } u.SendJSONToClient(w, l.languagesOUT, 200) u.GenericCommand(destroyTemporalDir) } @@ -317,11 +326,29 @@ func (l *loc) doLocUploadRequest(w http.ResponseWriter, r *http.Request) { total.Lines += v.Lines } l.languagesOUT = append(l.languagesOUT, total) - + //l.storeData() u.SendJSONToClient(w, l.languagesOUT, 200) u.GenericCommand(destroyTemporalDir) } +func (l *loc) storeData() { + d := store.NewDataLoc() + + dataJSON, err := json.Marshal(l.languagesOUT) + if err != nil { + log.Printf("ERROR Marshaling %s\n", err) + d.Data = string(`{}`) + } else { + d.Data = string(dataJSON) + } + d.Date = time.Now().Format("2006-01-02 15:04:05.000") + d.Repo = l.source + "/" + l.repo + d.Source = l.source + //u.PrettyPrintStruct(d) + go d.SaveDataLoc() + //go store.SaveDataLoc(d) +} + func NewLoc(test bool) loc { l := loc{ order: "0", diff --git a/main.go b/main.go index 12ebe5c..a077a46 100644 --- a/main.go +++ b/main.go @@ -20,11 +20,12 @@ import ( lo "github.com/jolav/codetabs/loc" px "github.com/jolav/codetabs/proxy" st "github.com/jolav/codetabs/stars" + "github.com/jolav/codetabs/store" vg "github.com/jolav/codetabs/video2gif" we "github.com/jolav/codetabs/weather" ) -var version = "0.7.20" +var version = "0.8.0" var when = "undefined" type Conf struct { @@ -45,6 +46,13 @@ func main() { checkMode(&c) //u.PrettyPrintStruct(c) + // Database + db, err := store.NewDB() + if err != nil { + log.Fatal("Error connecting DataBase => ", err) + } + store.MyDB = db + // Custom Error Log File + Custom Hits Log File var mylog *os.File if c.Mode == "production" { @@ -101,10 +109,31 @@ func mw(next http.HandlerFunc, service string, c Conf) http.HandlerFunc { return } } + next.ServeHTTP(w, r) }) } +/* +mux.HandleFunc("/vnstats/v1/", checkValid( + func(w http.ResponseWriter, r *http.Request) { + vnstatsRouter(w, r, a) + }, a.c.APIKey), +) + +func checkValid(next http.HandlerFunc, test string) http.HandlerFunc { +return func(w http.ResponseWriter, r *http.Request) { + r.ParseForm() + valid := r.Form.Get("test") + if valid != test { + http.Error(w, "Unauthorized", http.StatusUnauthorized) + return + } + next.ServeHTTP(w, r) +} +} +*/ + func checkFlags() { versionFlag := flag.Bool("v", false, "Show current version and exit") flag.Parse() diff --git a/stars/github.go b/stars/github.go index c4e2d64..f79ca3b 100644 --- a/stars/github.go +++ b/stars/github.go @@ -27,6 +27,7 @@ func (s *stars) doGHStarsRequest(w http.ResponseWriter, r *http.Request) { } } if totalStars == 0 { // repo exists but has no stars + s.storeData() u.SendJSONToClient(w, s.data, 200) return } @@ -47,6 +48,7 @@ func (s *stars) doGHStarsRequest(w http.ResponseWriter, r *http.Request) { last.Quantity = totalStars s.data = append(s.data, last) } + s.storeData() u.SendJSONToClient(w, s.data, 200) } diff --git a/stars/gitlab.go b/stars/gitlab.go index ffbde17..151a235 100644 --- a/stars/gitlab.go +++ b/stars/gitlab.go @@ -27,6 +27,7 @@ func (s *stars) doGLStarsRequest(w http.ResponseWriter, r *http.Request) { } } if totalStars == 0 { // repo exists but has no stars + s.storeData() u.SendJSONToClient(w, s.data, 200) return } @@ -36,6 +37,7 @@ func (s *stars) doGLStarsRequest(w http.ResponseWriter, r *http.Request) { if len(s.stars) > 0 { s.convertGLStarsToData() } + s.storeData() u.SendJSONToClient(w, s.data, 200) } diff --git a/stars/stars.go b/stars/stars.go index 7ec8918..25401be 100644 --- a/stars/stars.go +++ b/stars/stars.go @@ -3,12 +3,15 @@ package stars import ( + "encoding/json" "fmt" + "log" "net/http" "strings" "time" u "github.com/jolav/codetabs/_utils" + "github.com/jolav/codetabs/store" ) type star struct { @@ -79,6 +82,24 @@ func (s *stars) Router(w http.ResponseWriter, r *http.Request) { } } +func (s *stars) storeData() { + d := store.NewDataStars() + + dataJSON, err := json.Marshal(s.data) + if err != nil { + log.Printf("ERROR Marshaling %s\n", err) + d.Data = string(`{}`) + } else { + d.Data = string(dataJSON) + } + d.Date = time.Now().Format("2006-01-02 15:04:05.000") + d.Repo = s.source + "/" + s.repo + d.Source = s.source + d.Total = len(s.stars) + //u.PrettyPrintStruct(d) + go d.SaveDataStars() +} + func (s *stars) cleanStarsStruct() { s.stars = []*star{} s.headerLink = "" diff --git a/store/loc.go b/store/loc.go new file mode 100644 index 0000000..13ebafa --- /dev/null +++ b/store/loc.go @@ -0,0 +1,68 @@ +/* */ + +package store + +import ( + "encoding/json" + "fmt" + "log" + "time" + + u "github.com/jolav/codetabs/_utils" +) + +type dataLoc struct { + Repo string `json:"repo"` + Source string `json:"source"` + Date string `json:"date"` + Data string `json:"data"` +} + +func (d dataLoc) SaveDataLoc() { + d.replaceLoc() + //LoadDataLoc(d) +} + +func (d dataLoc) replaceLoc() { + sql := fmt.Sprintf("REPLACE INTO %s (repo, source, date, data) VALUES (?, ?, ?, ?)", + locTable) + stmt, err := MyDB.Prepare(sql) + if err != nil { + log.Printf("Error DB 1 replaceLoc => %s\n", err) + return + } + defer stmt.Close() + + _, err = stmt.Exec(d.Repo, d.Source, d.Date, d.Data) + if err != nil { + log.Printf("Error DB 2 replaceLoc => %s\n", err) + return + } +} + +func (d dataLoc) LoadDataLoc() { + type languageOUT struct { + Name string `json:"language"` + Files int `json:"files"` + Lines int `json:"lines"` + Blanks int `json:"blanks"` + Comments int `json:"comments"` + Code int `json:"linesOfCode"` + } + var languagesOUT []languageOUT + err := json.Unmarshal([]byte(d.Data), &languagesOUT) + if err != nil { + log.Printf("ERROR LoadConfig %s\n", err) + } + u.PrettyPrintStruct(languagesOUT) +} + +func NewDataLoc() dataLoc { + d := dataLoc{ + Repo: "", + Source: "", + Date: time.Now().Format("2006-01-02 15:04:05.000"), + Data: "{}", + } + return d +} diff --git a/store/stars.go b/store/stars.go new file mode 100644 index 0000000..f69dfff --- /dev/null +++ b/store/stars.go @@ -0,0 +1,67 @@ +/* */ + +package store + +import ( + "encoding/json" + "fmt" + "log" + "time" + + u "github.com/jolav/codetabs/_utils" +) + +type dataStars struct { + Repo string `json:"repo"` + Source string `json:"source"` + Date string `json:"date"` + Total int `json:"total"` + Data string `json:"data"` +} + +func (d dataStars) SaveDataStars() { + d.replaceStars() + //LoadDataStars(d) + +} + +func (d dataStars) replaceStars() { + sql := fmt.Sprintf("REPLACE INTO %s (repo, source, date, total, data) VALUES (?, ?, ?, ?, ?)", + starsTable) + stmt, err := MyDB.Prepare(sql) + if err != nil { + log.Printf("Error DB 1 replaceStars => %s\n", err) + return + } + defer stmt.Close() + + _, err = stmt.Exec(d.Repo, d.Source, d.Date, d.Total, d.Data) + if err != nil { + log.Printf("Error DB 2 replaceStars => %s\n", err) + return + } +} + +func (d dataStars) LoadDataStars() { + type point struct { + When string `json:"x"` + Quantity int `json:"y"` + } + var data []point + err := json.Unmarshal([]byte(d.Data), &data) + if err != nil { + log.Printf("ERROR LoadConfig %s\n", err) + } + u.PrettyPrintStruct(data) +} + +func NewDataStars() dataStars { + d := dataStars{ + Repo: "", + Source: "", + Date: time.Now().Format("2006-01-02 15:04:05.000"), + Total: 0, + Data: "{}", + } + return d +} diff --git a/store/store.go b/store/store.go new file mode 100644 index 0000000..53b1d51 --- /dev/null +++ b/store/store.go @@ -0,0 +1,115 @@ +/* */ + +package store + +import ( + "database/sql" + "encoding/json" + "fmt" + "log" + + _ "github.com/mattn/go-sqlite3" +) + +type configDB struct { + DatabaseType string `json:"databaseType"` + Host string `json:"host"` + Port int `json:"port"` + DB string `json:"db"` + User string `json:"user"` + Password string `json:"password"` +} + +var MyDB *DB + +// DB ... +type DB struct { + *sql.DB +} + +// NewDB ... +func NewDB() (*DB, error) { + var c configDB + err := json.Unmarshal(getDBConfigJSON(), &c) + if err != nil { + log.Fatal("Error parsing JSON config => \n", err) + } + var connPath string + if c.DatabaseType == "mysql" { + connPath = fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?parseTime=true", + c.User, + c.Password, + c.Host, + c.Port, + c.DB, + ) + } + if c.DatabaseType == "sqlite3" { + connPath = c.DB + } + //fmt.Println("CONNPATH => ", connPath) + db, err := sql.Open(c.DatabaseType, connPath) + if err != nil { + return nil, err + } + err = db.Ping() + if err != nil { + return nil, err + } + + createLocTable(&DB{db}) + createStarsTable(&DB{db}) + + return &DB{db}, nil +} + +func createLocTable(MyDB *DB) { + sql := fmt.Sprintf(` + CREATE TABLE IF NOT EXISTS %s ( + "repo" varchar(100) UNIQUE PRIMARY KEY, + "source" varchar(20) NOT NULL, + "date" text NOT NULL, + "data" json NOT NULL + ) + `, + locTable, + ) + stmt, err := MyDB.Prepare(sql) + if err != nil { + log.Printf("Error DB 1 Create Table %s => %s\n", locTable, err) + return + } + defer stmt.Close() + + _, err = stmt.Exec() + if err != nil { + log.Printf("Error DB 2 Create Table %s => %s\n", locTable, err) + return + } +} + +func createStarsTable(MyDB *DB) { + sql := fmt.Sprintf(` + CREATE TABLE IF NOT EXISTS %s ( + "repo" varchar(100) UNIQUE PRIMARY KEY, + "source" varchar(20) NOT NULL, + "date" text NOT NULL, + "total" INTEGER DEFAULT '0' NOT NULL, + "data" json NOT NULL + ) + `, + starsTable, + ) + stmt, err := MyDB.Prepare(sql) + if err != nil { + log.Printf("Error DB 1 Create Table %s => %s\n", starsTable, err) + return + } + defer stmt.Close() + + _, err = stmt.Exec() + if err != nil { + log.Printf("Error DB 2 Create Table %s => %s\n", starsTable, err) + return + } +} diff --git a/www/_public/templates/version.html b/www/_public/templates/version.html index 78d5b61..c4b19ec 100644 --- a/www/_public/templates/version.html +++ b/www/_public/templates/version.html @@ -2,7 +2,7 @@

API doc - (version 0.7.20) + (version 0.8.0)