Initial support for seeding multiple networks and improve scaling for…

… larger networks

README.md

@@ -1,14 +1,12 @@
 # dnsseeder
-Go Language dns seeder for the Twister P2P network
-
-This is a dns seeder for the [Twister P2P network](http://twister.net.co/)
+Go Language dns seeder for Networks that use Bitcoin technology such as the [Twister P2P network](http://twister.net.co/)
 
 It is based on the original twister-seeder https://github.com/miguelfreitas/twister-seeder
 
-Also see the associated utility to display information about [non-standard ip addresses](https://github.com/gombadi/nonstd/)
+This codebase can now seed different networks. At the moment it supports Twister and Bitcoin networks. These are configured in network.go and selected at runtime with -net command line option.
 
+Also see the associated utility to display information about [non-standard ip addresses](https://github.com/gombadi/nonstd/)
 
-> **NOTE:** This repository is under ongoing development. Stable releases have been tagged and should be used for production systems.
 
 
 ## Installing
@@ -36,7 +34,7 @@ The binary will then be available in ${HOME}/go/bin
 
 ## Usage
 
-    $ dnsseeder -h <domain respond to>
+    $ dnsseeder -h <domain respond to> -net <network to seed>
 
 An easy way to run the program is with tmux or screen. This enables you to log out and leave the program running.
 
@@ -48,7 +46,8 @@ If you want to be able to view the web interface then add -w port for the web se
 
 Command line Options:
 -h hostname to serve 
--p port to listen on
+-net The network to seed. Currently twister, bitcoin, bitcoin-test
+-p port to listen on for DNS requests
 -d Produce debug output
 -v Produce verbose output
 -w Port to listen on for Web Interface
@@ -69,16 +68,16 @@ gzip ${LOGDIR}/*.log
 
 # pass through the logging level needed
 if [ -z ${1} ]; then
-        LOGLV="-v"
+        THENET="twister"
 else
-        LOGLV="${1}"
+        THENET="${1}"
 fi
 
 cd
 echo
 echo "======= Run the Go Language dnsseed ======="
 echo
-${HOME}/go/bin/dnsseeder -h <host.to.serve> -p <port.to.listen.on> ${LOGLV} -w 8880 2>&1 | tee ${LOGDIR}/$(date +%F-%s)-goseeder.log
+${HOME}/go/bin/dnsseeder -h <host.to.serve> -p <dns.port.to.listen.on> ${THENET -v} -w 8880 2>&1 | tee ${LOGDIR}/$(date +%F-%s)-goseeder.log
 
 
 ```

crawler.go

@@ -22,7 +22,7 @@ func (e *crawlError) Error() string {
 
 // crawlTwistee runs in a goroutine, crawls the remote ip and updates the master
 // list of currently active addresses
-func crawlTwistee(tw *twistee) {
+func crawlTwistee(s *dnsseeder, tw *twistee) {
 
 	tw.crawlActive = true
 	tw.crawlStart = time.Now()
@@ -39,7 +39,7 @@ func crawlTwistee(tw *twistee) {
 	}
 
 	// connect to the remote ip and ask them for their addr list
-	ras, e := crawlIP(tw)
+	rna, e := crawlIP(s.net.pver, s.net.id, tw)
 
 	if e != nil {
 		// update the fact that we have not connected to this twistee
@@ -50,17 +50,23 @@ func crawlTwistee(tw *twistee) {
 		// update the status of this failed twistee
 		switch tw.status {
 		case statusRG:
-			if tw.rating += 25; tw.rating > 30 {
-				tw.status = statusWG
+			// if we are full then any RG failures will skip directly to NG
+			if s.isFull() {
+				tw.status = statusNG // not able to connect to this twistee so ignore
 				tw.statusTime = time.Now()
+			} else {
+				if tw.rating += 25; tw.rating > 30 {
+					tw.status = statusWG
+					tw.statusTime = time.Now()
+				}
 			}
 		case statusCG:
 			if tw.rating += 25; tw.rating >= 50 {
 				tw.status = statusWG
 				tw.statusTime = time.Now()
 			}
 		case statusWG:
-			if tw.rating += 30; tw.rating >= 100 {
+			if tw.rating += 15; tw.rating >= 100 {
 				tw.status = statusNG // not able to connect to this twistee so ignore
 				tw.statusTime = time.Now()
 			}
@@ -93,11 +99,14 @@ func crawlTwistee(tw *twistee) {
 
 	added := 0
 
-	// loop through all the received network addresses and add to thelist if not present
-	for _, na := range ras {
-		// a new network address so add to the system
-		if x := config.seeder.addNa(na); x == true {
-			added++
+	// if we are full then skip adding more possible clients
+	if s.isFull() == false {
+		// loop through all the received network addresses and add to thelist if not present
+		for _, na := range rna {
+			// a new network address so add to the system
+			if x := s.addNa(na); x == true {
+				added++
+			}
 		}
 	}
 
@@ -108,7 +117,7 @@ func crawlTwistee(tw *twistee) {
 			tw.status,
 			tw.rating,
 			tw.connectFails,
-			len(ras),
+			len(rna),
 			added,
 			time.Since(tw.crawlStart).String(),
 			time.Since(cs).String())
@@ -125,7 +134,7 @@ func crawlEnd(tw *twistee) {
 }
 
 // crawlIP retrievs a slice of ip addresses from a client
-func crawlIP(tw *twistee) ([]*wire.NetAddress, *crawlError) {
+func crawlIP(pver uint32, netID wire.BitcoinNet, tw *twistee) ([]*wire.NetAddress, *crawlError) {
 
 	ip := tw.na.IP.String()
 	port := strconv.Itoa(int(tw.na.Port))
@@ -155,14 +164,14 @@ func crawlIP(tw *twistee) ([]*wire.NetAddress, *crawlError) {
 		return nil, &crawlError{"Create NewMsgVersionFromConn", err}
 	}
 
-	err = wire.WriteMessage(conn, msgver, pver, twistNet)
+	err = wire.WriteMessage(conn, msgver, pver, netID)
 	if err != nil {
 		// Log and handle the error
 		return nil, &crawlError{"Write Version Message", err}
 	}
 
 	// first message received should be version
-	msg, _, err := wire.ReadMessage(conn, pver, twistNet)
+	msg, _, err := wire.ReadMessage(conn, pver, netID)
 	if err != nil {
 		// Log and handle the error
 		return nil, &crawlError{"Read message after sending Version", err}
@@ -190,13 +199,13 @@ func crawlIP(tw *twistee) ([]*wire.NetAddress, *crawlError) {
 	// send verack command
 	msgverack := wire.NewMsgVerAck()
 
-	err = wire.WriteMessage(conn, msgverack, pver, twistNet)
+	err = wire.WriteMessage(conn, msgverack, pver, netID)
 	if err != nil {
 		return nil, &crawlError{"writing message VerAck", err}
 	}
 
 	// second message received should be verack
-	msg, _, err = wire.ReadMessage(conn, pver, twistNet)
+	msg, _, err = wire.ReadMessage(conn, pver, netID)
 	if err != nil {
 		return nil, &crawlError{"reading expected Ver Ack from remote client", err}
 	}
@@ -213,7 +222,7 @@ func crawlIP(tw *twistee) ([]*wire.NetAddress, *crawlError) {
 	// send getaddr command
 	msgGetAddr := wire.NewMsgGetAddr()
 
-	err = wire.WriteMessage(conn, msgGetAddr, pver, twistNet)
+	err = wire.WriteMessage(conn, msgGetAddr, pver, netID)
 	if err != nil {
 		return nil, &crawlError{"writing Addr message to remote client", err}
 	}
@@ -225,7 +234,7 @@ func crawlIP(tw *twistee) ([]*wire.NetAddress, *crawlError) {
 		// Using the Bitcoin lib for the Twister Net means it does not understand some
 		// of the commands and will error. We can ignore these as we are only
 		// interested in the addr message and its content.
-		msgaddr, _, _ := wire.ReadMessage(conn, pver, twistNet)
+		msgaddr, _, _ := wire.ReadMessage(conn, pver, netID)
 		if msgaddr != nil {
 			switch msg := msgaddr.(type) {
 			case *wire.MsgAddr:

dns.go

@@ -51,7 +51,7 @@ func updateDNS(s *dnsseeder) {
 			if t == dnsV4Std || t == dnsV4Non {
 				if t == dnsV4Std && tw.dnsType == dnsV4Std {
 					r := new(dns.A)
-					r.Hdr = dns.RR_Header{Name: config.host + ".", Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: 60}
+					r.Hdr = dns.RR_Header{Name: config.host + ".", Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: s.net.ttl}
 					r.A = tw.na.IP
 					rr4std = append(rr4std, r)
 					numRR++
@@ -60,12 +60,12 @@ func updateDNS(s *dnsseeder) {
 				// if the twistee is using a non standard port then add the encoded port info to DNS
 				if t == dnsV4Non && tw.dnsType == dnsV4Non {
 					r := new(dns.A)
-					r.Hdr = dns.RR_Header{Name: "nonstd." + config.host + ".", Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: 60}
+					r.Hdr = dns.RR_Header{Name: "nonstd." + config.host + ".", Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: s.net.ttl}
 					r.A = tw.na.IP
 					rr4non = append(rr4non, r)
 					numRR++
 					r = new(dns.A)
-					r.Hdr = dns.RR_Header{Name: "nonstd." + config.host + ".", Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: 60}
+					r.Hdr = dns.RR_Header{Name: "nonstd." + config.host + ".", Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: s.net.ttl}
 					r.A = tw.nonstdIP
 					rr4non = append(rr4non, r)
 					numRR++
@@ -74,20 +74,20 @@ func updateDNS(s *dnsseeder) {
 			if t == dnsV6Std || t == dnsV6Non {
 				if t == dnsV6Std && tw.dnsType == dnsV6Std {
 					r := new(dns.AAAA)
-					r.Hdr = dns.RR_Header{Name: config.host + ".", Rrtype: dns.TypeAAAA, Class: dns.ClassINET, Ttl: 60}
+					r.Hdr = dns.RR_Header{Name: config.host + ".", Rrtype: dns.TypeAAAA, Class: dns.ClassINET, Ttl: s.net.ttl}
 					r.AAAA = tw.na.IP
 					rr6std = append(rr6std, r)
 					numRR++
 				}
 				// if the twistee is using a non standard port then add the encoded port info to DNS
 				if t == dnsV6Non && tw.dnsType == dnsV6Non {
 					r := new(dns.AAAA)
-					r.Hdr = dns.RR_Header{Name: "nonstd." + config.host + ".", Rrtype: dns.TypeAAAA, Class: dns.ClassINET, Ttl: 60}
+					r.Hdr = dns.RR_Header{Name: "nonstd." + config.host + ".", Rrtype: dns.TypeAAAA, Class: dns.ClassINET, Ttl: s.net.ttl}
 					r.AAAA = tw.na.IP
 					rr6non = append(rr6non, r)
 					numRR++
 					r = new(dns.AAAA)
-					r.Hdr = dns.RR_Header{Name: "nonstd." + config.host + ".", Rrtype: dns.TypeAAAA, Class: dns.ClassINET, Ttl: 60}
+					r.Hdr = dns.RR_Header{Name: "nonstd." + config.host + ".", Rrtype: dns.TypeAAAA, Class: dns.ClassINET, Ttl: s.net.ttl}
 					r.AAAA = tw.nonstdIP
 					rr6non = append(rr6non, r)
 					numRR++
@@ -156,8 +156,8 @@ func handleDNSStd(w dns.ResponseWriter, r *dns.Msg) {
 
 	w.WriteMsg(m)
 
-	if config.verbose {
-		log.Printf("status - DNS response Type: standard  To IP: %s  Query Type: %s\n", w.RemoteAddr().String(), qtype)
+	if config.debug {
+		log.Printf("debug - DNS response Type: standard  To IP: %s  Query Type: %s\n", w.RemoteAddr().String(), qtype)
 	}
 }
 
@@ -195,8 +195,8 @@ func handleDNSNon(w dns.ResponseWriter, r *dns.Msg) {
 
 	w.WriteMsg(m)
 
-	if config.verbose {
-		log.Printf("status - DNS response Type: non-standard  To IP: %s  Query Type: %s\n", w.RemoteAddr().String(), qtype)
+	if config.debug {
+		log.Printf("debug - DNS response Type: non-standard  To IP: %s  Query Type: %s\n", w.RemoteAddr().String(), qtype)
 	}
 }
 

doc.go

@@ -1,8 +1,17 @@
 /*
-This application provides a seeder service to the Twister Network.
+This application provides a DNS seeder service to network based on Bitcoin technology.
+For example -
 http://twister.net.co/
+https://bitcoin.org/
 
-It crawls the Twister Network for active clients and records their ip address and port. It then replies to DNS queries with the ip addresses.
+
+This application crawls the Network for active clients and records their ip address and port. It then replies to DNS queries with this information.
+
+Features:
+- Preconfigured support for Twister & Bitcoin networks. use -net <network> to load config data.
+- supports ipv4 & ipv6 addresses
+- revisits clients on a configurable time basis to make sure they are still available
+- Low memory & cpu requirements
 
 */
 package main

http.go

@@ -231,9 +231,7 @@ func statusHandler(w http.ResponseWriter, r *http.Request, status uint32) {
 		}
 
 	}
-
 	writeFooter(w, r, startT)
-
 }
 
 // copy Twistee details into a template friendly struct

main.go

@@ -38,29 +38,48 @@ type configData struct {
 
 var config configData
 var counts twCounts
+var nwname string
 
 func main() {
 
 	// FIXME - update with git hash during build
-	config.version = "0.5.0"
+	config.version = "0.6.0"
 
 	// initialize the stats counters
 	counts.TwStatus = make([]uint32, maxStatusTypes)
 	counts.TwStarts = make([]uint32, maxStatusTypes)
 	counts.DNSCounts = make([]uint32, maxDNSTypes)
 
+	flag.StringVar(&nwname, "net", "", "Preconfigured Network config")
 	flag.StringVar(&config.host, "h", "", "DNS host to serve")
-	flag.StringVar(&config.port, "p", "8053", "Port to listen on")
+	flag.StringVar(&config.port, "p", "8053", "DNS Port to listen on")
+	flag.StringVar(&config.http, "w", "", "Web Port to listen on. No port specified & no web server running")
 	flag.BoolVar(&config.verbose, "v", false, "Display verbose output")
 	flag.BoolVar(&config.debug, "d", false, "Display debug output")
 	flag.BoolVar(&config.stats, "s", false, "Display stats output")
-	flag.StringVar(&config.http, "w", "", "Web Port to listen on. No port specified & no web server running")
 	flag.Parse()
 
 	if config.host == "" {
-		log.Fatalf("error - no hostname provided\n")
+		fmt.Printf("error - no hostname provided\n")
+		os.Exit(1)
+	}
+
+	// configure the network options so we can start crawling
+	thenet := selectNetwork(nwname)
+	if thenet == nil {
+		fmt.Printf("Error - No valid network specified. Please add -net=<network> from one of the following:\n")
+		for _, n := range getNetworkNames() {
+			fmt.Printf("%s\n", n)
+		}
+		os.Exit(1)
 	}
 
+	// init the seeder
+	config.seeder = &dnsseeder{}
+	config.seeder.theList = make(map[string]*twistee)
+	config.seeder.uptime = time.Now()
+	config.seeder.net = thenet
+
 	if config.debug == true {
 		config.verbose = true
 		config.stats = true
@@ -73,38 +92,34 @@ func main() {
 
 	if config.verbose == false {
 		log.Printf("status - Running in quiet mode with limited output produced\n")
+	} else {
+		log.Printf("status - system is configured for %s\n", config.seeder.net.name)
 	}
 
 	// start the web interface if we want it running
 	if config.http != "" {
 		go startHTTP(config.http)
 	}
-
-	// init the seeder
-	config.seeder = &dnsseeder{}
-	config.seeder.theList = make(map[string]*twistee)
-	config.seeder.uptime = time.Now()
-
 	// start dns server
 	dns.HandleFunc("nonstd."+config.host, handleDNSNon)
 	dns.HandleFunc(config.host, handleDNSStd)
 	go serve("udp", config.port)
 	//go serve("tcp", config.port)
 
 	// seed the seeder with some ip addresses
-	initCrawlers()
+	config.seeder.initCrawlers()
 	// start first crawl
 	config.seeder.startCrawlers()
 
 	sig := make(chan os.Signal)
 	signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM)
 
 	// extract good dns records from all twistees on regular basis
-	dnsChan := time.NewTicker(time.Second * 57).C
+	dnsChan := time.NewTicker(time.Second * dnsDelay).C
 	// used to start crawlers on a regular basis
-	crawlChan := time.NewTicker(time.Second * 22).C
+	crawlChan := time.NewTicker(time.Second * crawlDelay).C
 	// used to remove old statusNG twistees that have reached fail count
-	auditChan := time.NewTicker(time.Hour * 1).C
+	auditChan := time.NewTicker(time.Minute * auditDelay).C
 
 	dowhile := true
 	for dowhile == true {
@@ -115,7 +130,7 @@ func main() {
 			if config.debug {
 				log.Printf("debug - Audit twistees timer triggered\n")
 			}
-			config.seeder.auditTwistees()
+			config.seeder.auditClients()
 		case <-dnsChan:
 			if config.debug {
 				log.Printf("debug - DNS - Updating latest ip addresses timer triggered\n")