Skip to content

Commit

Permalink
Performance improvements
Browse files Browse the repository at this point in the history
  • Loading branch information
@gombadi
gombadi committed Oct 2, 2015
1 parent eebd609 commit 6123b46
Show file tree
Hide file tree
Showing 3 changed files with 32 additions and 58 deletions.
51 changes: 15 additions & 36 deletions crawler.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,15 +24,13 @@ func (e *crawlError) Error() string {
// list of currently active addresses
func crawlNode(rc chan *result, s *dnsseeder, nd *node) {

// connect to the remote ip and ask them for their addr list
rna, e := crawlIP(s, nd)

res := &result{
nas: rna,
msg: e,
node: net.JoinHostPort(nd.na.IP.String(), strconv.Itoa(int(nd.na.Port))),
}

// connect to the remote ip and ask them for their addr list
res.nas, res.msg = crawlIP(s, res)

// all done so push the result back to the seeder.
//This will block until the seeder reads the result
rc <- res
Expand All @@ -41,34 +39,19 @@ func crawlNode(rc chan *result, s *dnsseeder, nd *node) {
}

// crawlIP retrievs a slice of ip addresses from a client
func crawlIP(s *dnsseeder, nd *node) ([]*wire.NetAddress, *crawlError) {

ip := nd.na.IP.String()
port := strconv.Itoa(int(nd.na.Port))
func crawlIP(s *dnsseeder, r *result) ([]*wire.NetAddress, *crawlError) {

// get correct formatting for ipv6 addresses
dialString := net.JoinHostPort(ip, port)

if config.debug {
log.Printf("%s - debug - start crawl: node %s status: %v:%v lastcrawl: %s\n",
s.name,
dialString,
nd.status,
nd.rating,
time.Since(nd.crawlStart).String())
}

conn, err := net.DialTimeout("tcp", dialString, time.Second*10)
conn, err := net.DialTimeout("tcp", r.node, time.Second*10)
if err != nil {
if config.debug {
log.Printf("%s - debug - Could not connect to %s - %v\n", s.name, ip, err)
log.Printf("%s - debug - Could not connect to %s - %v\n", s.name, r.node, err)
}
return nil, &crawlError{"", err}
}

defer conn.Close()
if config.debug {
log.Printf("%s - debug - Connected to remote address: %s Last connect was %v ago\n", s.name, ip, time.Since(nd.lastConnect).String())
log.Printf("%s - debug - Connected to remote address: %s\n", s.name, r.node)
}

// set a deadline for all comms to be done by. After this all i/o will error
Expand Down Expand Up @@ -98,17 +81,13 @@ func crawlIP(s *dnsseeder, nd *node) ([]*wire.NetAddress, *crawlError) {
case *wire.MsgVersion:
// The message is a pointer to a MsgVersion struct.
if config.debug {
log.Printf("%s - debug - %s - Remote version: %v\n", s.name, ip, msg.ProtocolVersion)
log.Printf("%s - debug - %s - Remote version: %v\n", s.name, r.node, msg.ProtocolVersion)
}
// fill the node struct with the remote details
nd.version = msg.ProtocolVersion
nd.services = msg.Services
nd.lastBlock = msg.LastBlock
if nd.strVersion != msg.UserAgent {
// if the srtVersion is already the same then don't overwrite it.
// saves the GC having to cleanup a perfectly good string
nd.strVersion = msg.UserAgent
}
r.version = msg.ProtocolVersion
r.services = msg.Services
r.lastBlock = msg.LastBlock
r.strVersion = msg.UserAgent
default:
return nil, &crawlError{"Did not receive expected Version message from remote client", errors.New("")}
}
Expand All @@ -130,7 +109,7 @@ func crawlIP(s *dnsseeder, nd *node) ([]*wire.NetAddress, *crawlError) {
switch msg.(type) {
case *wire.MsgVerAck:
if config.debug {
log.Printf("%s - debug - %s - received Version Ack\n", s.name, ip)
log.Printf("%s - debug - %s - received Version Ack\n", s.name, r.node)
}
default:
return nil, &crawlError{"Did not receive expected Ver Ack message from remote client", errors.New("")}
Expand Down Expand Up @@ -162,13 +141,13 @@ func crawlIP(s *dnsseeder, nd *node) ([]*wire.NetAddress, *crawlError) {
case *wire.MsgAddr:
// received the addr message so return the result
if config.debug {
log.Printf("%s - debug - %s - received valid addr message\n", s.name, ip)
log.Printf("%s - debug - %s - received valid addr message\n", s.name, r.node)
}
dowhile = false
return msg.AddrList, nil
default:
if config.debug {
log.Printf("%s - debug - %s - ignoring message - %v\n", s.name, ip, msg.Command())
log.Printf("%s - debug - %s - ignoring message - %v\n", s.name, r.node, msg.Command())
}
}
}
Expand Down
1 change: 0 additions & 1 deletion node.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@ type node struct {
lastConnect time.Time // last time we sucessfully connected to this client
lastTry time.Time // last time we tried to connect to this client
crawlStart time.Time // time when we started the last crawl
statusTime time.Time // time the status was last updated
nonstdIP net.IP // if not using the default port then this is the encoded ip containing the actual port
statusStr string // string with last error or OK details
strVersion string // remote client user agent
Expand Down
38 changes: 17 additions & 21 deletions seeder.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,10 +50,6 @@ type dnsseeder struct {
id wire.BitcoinNet // Magic number - Unique ID for this network. Sent in header of all messages
theList map[string]*node // the list of current nodes
mtx sync.RWMutex // protect thelist
maxSize int // max number of clients before we start restricting new entries
port uint16 // default network port this seeder uses
pver uint32 // minimum block height for the seeder
ttl uint32 // DNS TTL to use for this seeder
dnsHost string // dns host we will serve results for this domain
name string // Short name for the network
desc string // Long description for the network
Expand All @@ -62,12 +58,20 @@ type dnsseeder struct {
maxStart []uint32 // max number of goroutines to start each run for each status type
delay []int64 // number of seconds to wait before we connect to a known client for each status
counts NodeCounts // structure to hold stats for this seeder
pver uint32 // minimum block height for the seeder
ttl uint32 // DNS TTL to use for this seeder
maxSize int // max number of clients before we start restricting new entries
port uint16 // default network port this seeder uses
}

type result struct {
nas []*wire.NetAddress // slice of node addresses returned from a node
msg *crawlError // error string or nil if no problems
node string // theList key to the node that was crawled
nas []*wire.NetAddress // slice of node addresses returned from a node
msg *crawlError // error string or nil if no problems
node string // theList key to the node that was crawled
version int32 // remote node protocol version
services wire.ServiceFlag // remote client supported services
lastBlock int32 // last block seen by the node
strVersion string // remote client user agent
}

// initCrawlers needs to be run before the startCrawlers so it can get
Expand Down Expand Up @@ -173,7 +177,7 @@ func (s *dnsseeder) startCrawlers(resultsChan chan *result) {
tcount := uint32(len(s.theList))
if tcount == 0 {
if config.debug {
log.Printf("%s - debug - startCrawlers fail: no node ailable\n", s.name)
log.Printf("%s - debug - startCrawlers fail: no node available\n", s.name)
}
return
}
Expand Down Expand Up @@ -247,22 +251,18 @@ func (s *dnsseeder) processResult(r *result) {
// if we are full then any RG failures will skip directly to NG
if len(s.theList) > s.maxSize {
nd.status = statusNG // not able to connect to this node so ignore
nd.statusTime = time.Now()
} else {
if nd.rating += 25; nd.rating > 30 {
nd.status = statusWG
nd.statusTime = time.Now()
}
}
case statusCG:
if nd.rating += 25; nd.rating >= 50 {
nd.status = statusWG
nd.statusTime = time.Now()
}
case statusWG:
if nd.rating += 15; nd.rating >= 100 {
nd.status = statusNG // not able to connect to this node so ignore
nd.statusTime = time.Now()
}
}
// no more to do so return which will shutdown the goroutine & call
Expand All @@ -281,23 +281,21 @@ func (s *dnsseeder) processResult(r *result) {
}

// succesful connection and addresses received so mark status
if nd.status != statusCG {
nd.status = statusCG
nd.statusTime = time.Now()
}
nd.status = statusCG
cs := nd.lastConnect
nd.rating = 0
nd.connectFails = 0
nd.lastConnect = time.Now()
nd.lastTry = time.Now()
nd.lastTry = nd.lastConnect
nd.statusStr = "ok: received remote address list"

added := 0
// do not accept more than one third of maxSize addresses from one node
oneThird := int(float64(s.maxSize / 3))

// if we are full then skip adding more possible clients
if len(s.theList) < s.maxSize {
// do not accept more than one third of maxSize addresses from one node
oneThird := int(float64(s.maxSize / 3))

// loop through all the received network addresses and add to thelist if not present
for _, na := range r.nas {
// a new network address so add to the system
Expand All @@ -322,7 +320,6 @@ func (s *dnsseeder) processResult(r *result) {
time.Since(nd.crawlStart).String(),
time.Since(cs).String())
}

}

// crawlEnd is run as a defer to make sure node status is correctly updated
Expand Down Expand Up @@ -358,7 +355,6 @@ func (s *dnsseeder) addNa(nNa *wire.NetAddress) bool {
lastConnect: time.Now(),
version: 0,
status: statusRG,
statusTime: time.Now(),
dnsType: dnsV4Std,
}

Expand Down

0 comments on commit 6123b46

Please sign in to comment.