Skip to content
This repository was archived by the owner on May 21, 2020. It is now read-only.

Commit c6fd1d7

Browse files
CatTailaserrallerios
authored andcommitted
fix BackpressureTimeout after checkpoint throw ShutdownException (#15)
* fix substream exception cause BackpressureTimeout * update jdk to openjdk8
1 parent 1a9c22d commit c6fd1d7

File tree

3 files changed

+18
-14
lines changed

3 files changed

+18
-14
lines changed

.travis.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ scala:
44
- 2.11.12
55
- 2.12.7
66
jdk:
7-
- oraclejdk8
7+
- openjdk8
88
script:
99
- sbt -J-XX:ReservedCodeCacheSize=128m ++$TRAVIS_SCALA_VERSION ";test:compile"
1010
# make 'git branch' work again

src/main/scala/aserralle/akka/stream/kcl/CommittableRecord.scala

+10-3
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
package aserralle.akka.stream.kcl
66

77
import akka.Done
8+
import software.amazon.kinesis.exceptions.ShutdownException
89
import software.amazon.kinesis.lifecycle.ShutdownReason
910
import software.amazon.kinesis.processor.RecordProcessorCheckpointer
1011
import software.amazon.kinesis.retrieval.KinesisClientRecord
@@ -30,10 +31,16 @@ class CommittableRecord(
3031
def canBeCheckpointed(): Boolean =
3132
recordProcessorShutdownReason().isEmpty
3233

33-
def tryToCheckpoint(): Future[Done] =
34+
def tryToCheckpoint(): Future[Boolean] =
3435
Future {
35-
checkpointer.checkpoint(sequenceNumber, subSequenceNumber)
36-
Done
36+
try {
37+
checkpointer.checkpoint(sequenceNumber, subSequenceNumber)
38+
true
39+
} catch {
40+
case _: ShutdownException =>
41+
false
42+
case exception => throw exception
43+
}
3744
}
3845
}
3946

src/main/scala/aserralle/akka/stream/kcl/scaladsl/KinesisWorkerSource.scala

+7-10
Original file line numberDiff line numberDiff line change
@@ -86,26 +86,23 @@ object KinesisWorkerSource {
8686

8787
val `{` =
8888
b.add(scaladsl.Broadcast[immutable.Seq[CommittableRecord]](2))
89-
val `}` = b.add(Zip[Done, immutable.Seq[CommittableRecord]])
89+
val `}` = b.add(Zip[Boolean, immutable.Seq[CommittableRecord]])
9090
val `=` = b.add(Flow[KinesisClientRecord])
9191

9292
`{`.out(0)
9393
.map(_.max)
94-
.mapAsync(1)(r =>
95-
if (r.canBeCheckpointed()) r.tryToCheckpoint()
96-
else Future.successful(Done)) ~> `}`.in0
94+
.mapAsync(1)(
95+
r =>
96+
if (r.canBeCheckpointed()) r.tryToCheckpoint()
97+
else Future.successful(true)
98+
) ~> `}`.in0
9799
`{`.out(1) ~> `}`.in1
98100

99-
`}`.out.map(_._2).mapConcat(identity).map(_.record) ~> `=`
101+
`}`.out.filter(_._1).map(_._2).mapConcat(identity).map(_.record) ~> `=`
100102

101103
FlowShape(`{`.in, `=`.out)
102104
})
103105
.mergeSubstreams
104-
.withAttributes(ActorAttributes.supervisionStrategy {
105-
case _: ShutdownException =>
106-
Resume
107-
case _ => Stop
108-
})
109106

110107
def checkpointRecordsSink(
111108
settings: KinesisWorkerCheckpointSettings =

0 commit comments

Comments
 (0)