fix: Reconnection attempts delay should be constant (#120)

* Constant delay between reconnection attempts

- In 5f3e237 we added a backoff up to a
  configurable number of retries.
- When this number of retries is unset we could end up sleeping for a
  very long time before reconnection attempts.
- Instead preserve the previous behaviour of sleeping for a fixed 1
  second interval between reconnection attempts when the max retries is
  unset.
- Remove ServerPollingInterval const that was only in use by the tests.

* Cap reconnect delay at one minute

---------

Signed-off-by: Rebecca Roberts <[email protected]>

integration/syslog_test.go

@@ -627,7 +627,7 @@ var _ = Describe("Blackbox", func() {
 			session, err := gexec.Start(blackboxCmd, GinkgoWriter, GinkgoWriter)
 			Expect(err).NotTo(HaveOccurred())
 
-			time.Sleep(2 * syslog.ServerPollingInterval)
+			time.Sleep(2 * time.Second)
 
 			buffer := gbytes.NewBuffer()
 			serverProcess = ginkgomon.Invoke(&TcpSyslogServer{
@@ -648,7 +648,10 @@ var _ = Describe("Blackbox", func() {
 			Write(logFile, "can't log this\n", false, false)
 			Write(logFile, "more\n", true, true)
 
-			time.Sleep(2 * syslog.ServerPollingInterval)
+			Eventually(session.Err, "5s").Should(gbytes.Say("Error connecting.*Will retry in 1 seconds"))
+			Eventually(session.Err, "5s").Should(gbytes.Say("Error connecting.*Will retry in 1 seconds"))
+
+			time.Sleep(2 * time.Second)
 
 			buffer2 := gbytes.NewBuffer()
 			serverProcess = ginkgomon.Invoke(&TcpSyslogServer{
@@ -791,6 +794,9 @@ var _ = Describe("Blackbox", func() {
 				Write(logFile, "try to log this\n", false, false)
 				Write(logFile, "try to log more and notice can't write to socket\n", true, true)
 
+				Eventually(session.Err, "5s").Should(gbytes.Say("Error connecting.*Will retry in 2 seconds"))
+				Eventually(session.Err, "5s").Should(gbytes.Say("Error connecting.*Will retry in 4 seconds"))
+
 				Expect(session.Wait("20s")).To(gexec.Exit(1))
 			})
 		})

syslog/drainer.go

@@ -26,8 +26,6 @@ type Drainer interface {
 	Drain(line string, tag string) error
 }
 
-const ServerPollingInterval = 5 * time.Second
-
 type drainer struct {
 	conn           net.Conn
 	dialFunction   func() (net.Conn, error)
@@ -173,7 +171,9 @@ func (d *drainer) resetAttempts() {
 
 func (d *drainer) incrementAttempts() {
 	d.connAttempts++
-	d.sleepSeconds = d.sleepSeconds << 1
+	if d.maxRetries > 0 && d.sleepSeconds < 60 {
+		d.sleepSeconds = d.sleepSeconds << 1
+	}
 }
 
 func (d *drainer) ensureConnection() {