@@ -40,6 +40,7 @@ object MTurkMethods {
4040 tg.head._1
4141 }
4242
43+ // Handle the user manual qualification requests
4344 private [worker] def mturk_grantQualifications (hitstate : HITState , state : MTState , backend : AmazonMTurk ) : MTState = {
4445 var internal_state = state
4546
@@ -68,20 +69,27 @@ object MTurkMethods {
6869 if (internal_state.worker_whitelist.contains(worker_id, group_id)) {
6970 // if that disqualification is not the same as the one they're asking for, sorry, reject;
7071 // granting this would violate i.i.d. guarantee
72+ // note by Ye Shu: this is the case where disqualification has been granted for previous batch
7173 if (internal_state.getHITTypeForWhitelistedWorker(worker_id, group_id).id != hit_type_id) {
7274 backend.rejectQualificationRequest(new model.RejectQualificationRequestRequest ()
7375 .withQualificationRequestId(request.getQualificationRequestId)
7476 .withReason(" You have already requested a qualification or submitted work for an associated HITType " +
7577 " that disqualifies you from participating in this HITType."
7678 // otherwise, they're requesting something we've already granted; reject
7779 } else {
80+ // note by Ye Shu: this is the case where disqualification has been granted for this branch
7881 backend.rejectQualificationRequest(new model.RejectQualificationRequestRequest ()
7982 .withQualificationRequestId(request.getQualificationRequestId)
8083 .withReason(" You cannot request this qualification more than once."
8184 )
8285 }
8386 } else {
8487 // if we don't know them, record the user and grant their disqualification request
88+ // note by Ye Shu: this code is unlikely to be reached, because we are now using qualification requirement
89+ // that "xxx has not been granted". So if a worker asks for qualification, they are likely already disqualified
90+ // However I'm keeping this code here just in case
91+ DebugLog (s " Granting qualification request: (worker ID: ${worker_id}, group ID: ${group_id}). " , LogLevelInfo (), LogType .ADAPTER , null )
92+
8593 internal_state = internal_state.updateWorkerWhitelist(worker_id, group_id, hit_type_id)
8694 // get the BatchKey associated with the HITType; guaranteed to exist
8795 val batchKey = internal_state.getBatchKeyByHITTypeId(hit_type_id).get
@@ -97,35 +105,44 @@ object MTurkMethods {
97105 internal_state
98106 }
99107
108+ /**
109+ * Creates `QualificationRequirement` for a question.
110+ *
111+ * This QualificationRequirement is then reused for the entire question.
112+ * It'll be set when a worker answers a question, so the worker will not be
113+ * able to answer the recreated question in future batches.
114+ */
100115 private [worker] def mturk_createQualification (title : String ,
101116 batchKey : BatchKey ,
102117 batch_no : Int ,
103- backend : AmazonMTurk ) : QualificationRequirement = {
118+ backend : AmazonMTurk ): QualificationRequirement = {
104119 // get a simply-formatted date
105120 val sdf = new SimpleDateFormat (" yyyy-MM-dd:z"
106121 val datestr = sdf.format(new Date ())
107122
108- val qualtxt = s " AutoMan automatically generated Disqualification (title: $title, date: $datestr, batchKey: $batchKey, batch_no: $batch_no) "
109- // val qualRequest = new CreateQualificationTypeRequest()()
110- // qualRequest.setName("AutoMan").setKeywords()
111- // val id : UUID = UUID.randomUUID()
112- val qual = backend.createQualificationType(new CreateQualificationTypeRequest ()
113- // .withName("AutoMan")
114- .withName(" Automan " + UUID .randomUUID().toString)
115- // .setDescription(UUID.randomUUID())
123+ // Creates qualification type
124+ val qualID : QualificationID = {
125+ val qualtxt = s " AutoMan automatically generated Disqualification (title: \" $title\" , date: $datestr) " // , groupID: ${batchKey._1}
126+ // Commented out groupID in qualtxt since groupID is just title now.
127+
128+ val qual = backend.createQualificationType(new CreateQualificationTypeRequest ()
129+ .withName(qualtxt)
116130 .withKeywords(" automan"
117- .withDescription(qualtxt)
131+ .withDescription(s " Automan ${ UUID .randomUUID().toString} : $ qualtxt" )
118132 // .withQualificationTypeStatus(QualificationTypeStatus.Inactive)
119133 .withQualificationTypeStatus(QualificationTypeStatus .Active )
120- ) : CreateQualificationTypeResult // UUID keyword?
121- // "AutoMan " + UUID.randomUUID(), "automan", qualtxt)
134+ ) : CreateQualificationTypeResult // UUID keyword?
135+
136+ DebugLog (s " Created disqualification type ID: ${qual.getQualificationType.getQualificationTypeId}. " ,LogLevelInfo (),LogType .ADAPTER ,null )
137+ qual.getQualificationType.getQualificationTypeId
138+ }
122139
123- DebugLog ( s " Creating disqualification ID: ${qual.getQualificationType.getQualificationTypeId} . " , LogLevelInfo (), LogType . ADAPTER , null )
140+ // Creates Qualification Requirement
124141 new QualificationRequirement ()
125- .withQualificationTypeId(qual.getQualificationType.getQualificationTypeId )
142+ .withQualificationTypeId(qualID )
126143 .withComparator(model.Comparator .DoesNotExist )
127144// .withIntegerValues(batch_no)
128- .withActionsGuarded(" Accept" // TODO: Check if this is what we want
145+ .withActionsGuarded(" Accept"
129146
130147 // model.Comparator.EqualTo, batch_no, null, false)
131148 }
@@ -238,23 +255,21 @@ object MTurkMethods {
238255 // get just-created batch number; guaranteed to exist because we just created it
239256 val batch_no = internal_state.getBatchNo(batch_key).get
240257
241- // create disqualification for batch
258+ // create disqualification for question
259+ // if already exists (i.e. not first batch) then reuse disqualification
260+ // This disqualification ensures that the question and its recreated version
261+ // (with longer worker timeouts) are only answered once maximum by each worker
242262 // TODO: tinker with imports
243- val disqualification : QualificationRequirement = mturk_createQualification(title, batch_key, batch_no, backend)
244- DebugLog (s " Created disqualification ${disqualification.getQualificationTypeId} for batch key = " + batch_key, LogLevelDebug (), LogType .ADAPTER , null )
263+ val disqualification : QualificationRequirement =
264+ internal_state.qualificationRequirements.getOrElse(group_id, {
265+ val qual = mturk_createQualification(title, batch_key, batch_no, backend)
266+ DebugLog (s " Created new disqualification with type ID ${qual.getQualificationTypeId} for group id " + group_id, LogLevelInfo (), LogType .ADAPTER , null )
267+ // update qualifications so it can be reused for future batches
268+ internal_state = internal_state.addQualificationRequirement(group_id, qual)
269+ qual
270+ })
245271 // getQTId is in the doc...
246272
247- // whenever we create a new group, we need to add the disqualification to the HITType
248- // EXCEPT if it's the very first time the group is posted
249- val qs : util.Collection [QualificationRequirement ] =
250- if (batch_no != 1 ) {
251- DebugLog (s " Batch # ${batch_no} run, using disqualification ${disqualification.getQualificationTypeId} for batch " + batch_key, LogLevelDebug (), LogType .ADAPTER , null )
252- List (disqualification)
253- } else {
254- DebugLog (s " Batch # ${batch_no} run, using no qualifications for batch " + batch_key, LogLevelDebug (), LogType .ADAPTER , null )
255- Nil
256- }
257-
258273 val autoApprovalDelayInSeconds = (3 * 24 * 60 * 60 ).toLong // 3 days
259274
260275 val hit_type_id = backend.createHITType(new CreateHITTypeRequest ()
@@ -264,7 +279,7 @@ object MTurkMethods {
264279 .withTitle(title) // title
265280 .withKeywords(keywords.mkString(" ," // keywords
266281 .withDescription(desc) // description
267- .withQualificationRequirements(qs) // qualifications
282+ .withQualificationRequirements(List (disqualification)) // qualifications
268283 )
269284 val hittype = HITType (hit_type_id.getHITTypeId, disqualification, group_id)
270285
0 commit comments