Skip to content

Commit

Permalink
feat(tagstatistics): cleaning and refactoring
Browse files Browse the repository at this point in the history
  • Loading branch information
@lucashimpens
lucashimpens committed Jul 17, 2024
1 parent 5b530d0 commit 8771cfc
Show file tree
Hide file tree
Showing 8 changed files with 400 additions and 360 deletions.
269 changes: 34 additions & 235 deletions source/src/main/java/org/cerberus/core/apiprivate/CampaignExecutionPrivateController.java
View file

Large diffs are not rendered by default.

26 changes: 11 additions & 15 deletions source/src/main/java/org/cerberus/core/crud/dao/impl/TagStatisticDAO.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,7 @@ public AnswerList<TagStatistic> readByCriteria(List<String> systems, List<String

StringBuilder query = new StringBuilder();

query.append("SELECT * from tagstatistic tac WHERE `Campaign` IN (SELECT DISTINCT `Campaign` FROM tagstatistic tac");
query.append("SELECT `Id`, `Tag`, `Country`, `Environment`, `Campaign`, `CampaignGroup1`, `SystemList`, `ApplicationList`, `DateStartExe`, `DateEndExe`, `NbExe`, `NbExeUsefull`, `NbOK`, `NbKO`, `NbFA`, `NbNA`, `NbNE`, `NbPE`, `NbWE`, `NbPE`, `NbQU`, `NbQE`, `NbCA` from tagstatistic WHERE `Campaign` IN (SELECT DISTINCT `Campaign` FROM tagstatistic");

String systemRegex = "";
String applicationRegex = "";
Expand All @@ -146,11 +146,15 @@ public AnswerList<TagStatistic> readByCriteria(List<String> systems, List<String
systemRegex = systems.stream()
.map(sys -> "\"" + sys + "\"")
.collect(Collectors.joining("|"));
query.append(" WHERE tac.`SystemList` REGEXP ?");
query.append(" WHERE `SystemList` REGEXP ?");
}

if (!applications.isEmpty()) {
query.append(" AND tac.`ApplicationList` REGEXP ?");
if (systems.isEmpty()) {
query.append(" WHERE `ApplicationList` REGEXP ?");
} else {
query.append(" AND `ApplicationList` REGEXP ?");
}
applicationRegex = applications.stream()
.map(app -> "\"" + app + "\"")
.collect(Collectors.joining("|"));
Expand All @@ -160,16 +164,16 @@ public AnswerList<TagStatistic> readByCriteria(List<String> systems, List<String
query.append(" AND ").append(SqlUtil.generateInClause("CampaignGroup1", group1List));
}

query.append(") AND tac.`DateStartExe` >= ? AND tac.`DateEndExe` <= ?");
query.append(") AND `DateStartExe` >= ? AND `DateEndExe` <= ?");

try (Connection connection = this.databaseSpring.connect();
PreparedStatement preStat = connection.prepareStatement(query.toString());
Statement stm = connection.createStatement()) {

int i = 1;

preStat.setString(i++, systemRegex);
preStat.setString(i++, applicationRegex);
if (!systems.isEmpty()) preStat.setString(i++, systemRegex);
if (!applications.isEmpty()) preStat.setString(i++, applicationRegex);

if (!group1List.isEmpty()) {
for (String group1 : group1List) {
Expand All @@ -183,7 +187,7 @@ public AnswerList<TagStatistic> readByCriteria(List<String> systems, List<String
try (ResultSet resultSet = preStat.executeQuery();
ResultSet rowSet = stm.executeQuery("SELECT FOUND_ROWS()")) {

LOG.debug("Execute SQL Statement: {} ", preStat);
LOG.info("Execute SQL Statement: {} ", preStat);

while (resultSet.next()) {
tagStatistics.add(this.loadFromResultSet(resultSet));
Expand Down Expand Up @@ -320,10 +324,6 @@ private TagStatistic loadFromResultSet(ResultSet resultSet) throws SQLException
int nbQU = resultSet.getInt("nbQU");
int nbQE = resultSet.getInt("nbQE");
int nbCA = resultSet.getInt("nbCA");
String usrModif = ParameterParserUtil.parseStringParam(resultSet.getString("UsrModif"), "");
String usrCreated = ParameterParserUtil.parseStringParam(resultSet.getString("UsrCreated"), "");
Timestamp dateModif = resultSet.getTimestamp("DateModif");
Timestamp dateCreated = resultSet.getTimestamp("DateCreated");

return TagStatistic.builder()
.id(id)
Expand All @@ -348,10 +348,6 @@ private TagStatistic loadFromResultSet(ResultSet resultSet) throws SQLException
.nbQU(nbQU)
.nbQE(nbQE)
.nbCA(nbCA)
.usrCreated(usrCreated)
.usrModif(usrModif)
.dateCreated(dateCreated)
.dateModif(dateModif)
.build();
}
}
9 changes: 9 additions & 0 deletions source/src/main/java/org/cerberus/core/crud/service/ITagStatisticService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,11 @@
import org.cerberus.core.crud.entity.Tag;
import org.cerberus.core.crud.entity.TagStatistic;
import org.cerberus.core.crud.entity.TestCaseExecution;
import org.cerberus.core.exception.CerberusException;
import org.cerberus.core.util.answer.Answer;
import org.cerberus.core.util.answer.AnswerList;
import org.json.JSONException;
import org.json.JSONObject;

import java.util.List;
import java.util.Map;
Expand Down Expand Up @@ -53,4 +56,10 @@ public interface ITagStatisticService {
*/
void populateTagStatisticsMap(Map<String, TagStatistic> tagStatistics, List<TestCaseExecution> executions, Tag tag);

Map<String, Map<String, JSONObject>> createMapAggregateByTag(List<TagStatistic> tagStatistics) throws JSONException;
Map<String, JSONObject> createMapAggregateByCampaign(Map<String, Map<String, JSONObject>> aggregateByTag) throws JSONException;
List<String> getSystemsAllowedForUser(String user) throws CerberusException;
List<String> getApplicationsSystems(List<String> systems);
String formatDateForDb(String date);

}
247 changes: 226 additions & 21 deletions source/src/main/java/org/cerberus/core/crud/service/impl/TagStatisticService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,23 +22,28 @@
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.cerberus.core.crud.dao.ITagStatisticDAO;
import org.cerberus.core.crud.entity.Tag;
import org.cerberus.core.crud.entity.TagStatistic;
import org.cerberus.core.crud.entity.TestCaseExecution;
import org.cerberus.core.crud.entity.*;
import org.cerberus.core.crud.service.IApplicationService;
import org.cerberus.core.crud.service.ICampaignService;
import org.cerberus.core.crud.service.ITagStatisticService;
import org.cerberus.core.crud.service.IUserSystemService;
import org.cerberus.core.exception.CerberusException;
import org.cerberus.core.util.JSONUtil;
import org.cerberus.core.util.StringUtil;
import org.cerberus.core.util.answer.Answer;
import org.cerberus.core.util.answer.AnswerList;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;

import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.text.DateFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.stream.Collectors;

@Service
public class TagStatisticService implements ITagStatisticService {
Expand All @@ -47,29 +52,20 @@ public class TagStatisticService implements ITagStatisticService {

@Autowired
private ITagStatisticDAO tagStatisticDAO;

@Autowired
private ICampaignService campaignService;
@Autowired
private IUserSystemService userSystemService;
@Autowired
private IApplicationService applicationService;

/**
* @param tag
* @return AnswerList that contains data from database
*/
public AnswerList<TagStatistic> readByTag(String tag) {
return tagStatisticDAO.readByTag(tag);
}

@Override
public AnswerList<TagStatistic> readByCriteria(List<String> systems, List<String> applications, List<String> group1List, String minDate, String maxDate) {
return tagStatisticDAO.readByCriteria(systems, applications, group1List, minDate, maxDate);
}
//Methods used when we create TagStatistics//

@Override
public Answer createWithMap(Map<String, TagStatistic> map) {
return tagStatisticDAO.createWithMap(map);
}


/**
* Initialize TagStatistics objects
* @param tag
Expand Down Expand Up @@ -220,4 +216,213 @@ private String getCampaignGroup1(String campaign) {
}
return campaignGroup1;
}

//Methods used when we retrieve and aggregate TagStatistics by campaign//

/**
* @param tag
* @return AnswerList that contains data from database
*/
public AnswerList<TagStatistic> readByTag(String tag) {
return tagStatisticDAO.readByTag(tag);
}

@Override
public AnswerList<TagStatistic> readByCriteria(List<String> systems, List<String> applications, List<String> group1List, String minDate, String maxDate) {
return tagStatisticDAO.readByCriteria(systems, applications, group1List, minDate, maxDate);
}

public Map<String, Map<String, JSONObject>> createMapAggregateByTag(List<TagStatistic> tagStatistics) throws JSONException {
Map<String, Map<String, JSONObject>> aggregateByTag = new HashMap<>();
for (TagStatistic tagStatistic : tagStatistics) {
int nbExeUsefull = 0;
int nbExe = 0;
int nbOK = 0;
long duration = 0;
String campaign = tagStatistic.getCampaign();
String tag = tagStatistic.getTag();
Timestamp minTagDateStartExe = new Timestamp(0);
Timestamp maxTagDateEndExe = new Timestamp(0);
long msMinTagDateStart = 0;
long msMaxTagDateEnd = 0;
JSONArray systemsInTagMap;
JSONArray applicationsInTagMap;

if (!aggregateByTag.containsKey(campaign)) {
aggregateByTag.put(campaign, new HashMap<>());
}

if (!aggregateByTag.get(campaign).containsKey(tag)) {
aggregateByTag.get(campaign).put(tag, new JSONObject());
aggregateByTag.get(campaign).get(tag)
.put("campaign", tagStatistic.getCampaign())
.put("campaignGroup1", tagStatistic.getCampaignGroup1())
.put("systemList", new JSONArray().toString())
.put("applicationList", new JSONArray().toString())
.put("maxTagDateEnd", minTagDateStartExe)
.put("minTagDateStart", maxTagDateEndExe)
.put("nbExeUsefull", 0)
.put("nbExe", 0)
.put("nbOK", 0)
.put("duration", 0);
}

JSONObject mapTag = aggregateByTag.get(campaign).get(tag);

systemsInTagMap = JSONUtil.jsonArrayAddUniqueElement(
new JSONArray(tagStatistic.getSystemList()),
new JSONArray(mapTag.getString("systemList"))
);

applicationsInTagMap = JSONUtil.jsonArrayAddUniqueElement(
new JSONArray(tagStatistic.getApplicationList()),
new JSONArray(mapTag.getString("applicationList"))
);

minTagDateStartExe = Timestamp.valueOf(mapTag.getString("minTagDateStart"));
if (minTagDateStartExe.equals(new Timestamp(0)) || tagStatistic.getDateStartExe().getTime() < minTagDateStartExe.getTime()) {
minTagDateStartExe = tagStatistic.getDateStartExe();
}

maxTagDateEndExe = Timestamp.valueOf(mapTag.getString("maxTagDateEnd"));
if (maxTagDateEndExe.equals(new Timestamp(0)) || tagStatistic.getDateEndExe().getTime() > maxTagDateEndExe.getTime()) {
maxTagDateEndExe = tagStatistic.getDateEndExe();
}

msMinTagDateStart = minTagDateStartExe.getTime();
msMaxTagDateEnd = maxTagDateEndExe.getTime();
duration = (msMaxTagDateEnd - msMinTagDateStart) / 1000;
nbExeUsefull += tagStatistic.getNbExeUsefull() + mapTag.getInt("nbExeUsefull");
nbExe += tagStatistic.getNbExe() + mapTag.getInt("nbExe");
nbOK += tagStatistic.getNbOK() + mapTag.getInt("nbOK");

mapTag.put("systemList", systemsInTagMap.toString())
.put("applicationList", applicationsInTagMap.toString())
.put("minTagDateStart", minTagDateStartExe)
.put("maxTagDateEnd", maxTagDateEndExe)
.put("duration", duration)
.put("nbExeUsefull", nbExeUsefull)
.put("nbExe", nbExe)
.put("nbOK", nbOK);
}
return aggregateByTag;
}

public Map<String, JSONObject> createMapAggregateByCampaign(Map<String, Map<String, JSONObject>> aggregateByTag) throws JSONException {
Map<String, JSONObject> aggregateByCampaign = new HashMap<>();
JSONArray globalGroup1List = new JSONArray();
for (Map.Entry<String, Map<String, JSONObject>> aggregateByTagEntry : aggregateByTag.entrySet()) {
String campaign = aggregateByTagEntry.getKey();
double totalDuration = 0;
String minDateStart = "";
String maxDateEnd = "";
double sumPercOK = 0;
double sumPercReliability = 0;
int sumNumberExeUsefull = 0;
String campaignGroup1 = "";
JSONArray systemsByCampaign = new JSONArray();
JSONArray applicationsByCampaign = new JSONArray();

//If campaign is not present in the map we create a new entry
aggregateByCampaign.computeIfAbsent(campaign, key -> {
JSONObject entry = new JSONObject(new LinkedHashMap<>());
try {
entry.put("campaign", campaign)
.put("systemList", new JSONArray().toString())
.put("applicationList", new JSONArray().toString());
} catch (JSONException exception) {
LOG.error(exception);
}
return entry;
});

JSONObject campaignStatistic = aggregateByCampaign.get(campaign);

for (JSONObject mapTagEntry : aggregateByTagEntry.getValue().values()) {

campaignGroup1 = mapTagEntry.getString("campaignGroup1");

systemsByCampaign = JSONUtil.jsonArrayAddUniqueElement(
new JSONArray(mapTagEntry.getString("systemList")),
new JSONArray(campaignStatistic.getString("systemList"))
);

applicationsByCampaign = JSONUtil.jsonArrayAddUniqueElement(
new JSONArray(mapTagEntry.getString("applicationList")),
new JSONArray(campaignStatistic.getString("applicationList"))
);

if (StringUtil.isNotEmpty(campaignGroup1)) {
updateGlobalGroup1List(globalGroup1List, campaignGroup1);
}

minDateStart = updateMinCampaignDateStart(minDateStart, mapTagEntry);
maxDateEnd = updateMaxCampaignDateEnd(maxDateEnd, mapTagEntry);
sumPercOK += ((double) mapTagEntry.getInt("nbOK") / mapTagEntry.getInt("nbExeUsefull"));
sumPercReliability += ((double) mapTagEntry.getInt("nbExeUsefull") / mapTagEntry.getInt("nbExe"));
sumNumberExeUsefull += mapTagEntry.getInt("nbExeUsefull");
totalDuration += mapTagEntry.getLong("duration");
}

campaignStatistic
.put("systemList", systemsByCampaign)
.put("systemList", systemsByCampaign)
.put("applicationList", applicationsByCampaign)
.put("campaignGroup1", campaignGroup1)
.put("avgDuration", totalDuration / aggregateByTagEntry.getValue().size())
.put("minDateStart", minDateStart)
.put("maxDateEnd", maxDateEnd)
.put("avgOK", (sumPercOK * 100.0) / aggregateByTagEntry.getValue().size())
.put("avgReliability", (sumPercReliability * 100) / aggregateByTagEntry.getValue().size())
.put("avgNbExeUsefull", sumNumberExeUsefull / aggregateByTagEntry.getValue().size());
}
aggregateByCampaign.put("globalGroup1List", new JSONObject().put("array", globalGroup1List));
return aggregateByCampaign;
}

public List<String> getSystemsAllowedForUser(String user) throws CerberusException {
List<UserSystem> systemsAllowedForUser = userSystemService.findUserSystemByUser(user);
return systemsAllowedForUser.stream().map(UserSystem::getSystem).collect(Collectors.toList());
}
public List<String> getApplicationsSystems(List<String> systems) {
List<Application> applicationsAllowedForUser = applicationService.readBySystem(systems).getDataList();
return applicationsAllowedForUser.stream().map(Application::getApplication).collect(Collectors.toList());
}

public String formatDateForDb(String date) {
TimeZone tz = TimeZone.getTimeZone("UTC");
DateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S'Z'");
df.setTimeZone(tz);

String dateFormatted = "";
try {
dateFormatted = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(df.parse(date));
} catch (ParseException exception) {
LOG.error("Exception when parsing date, ", exception);
}
return dateFormatted;
}

private JSONArray updateGlobalGroup1List(JSONArray campaignGroup1Array, String campaignGroup1) throws JSONException {
if (!JSONUtil.jsonArrayContains(campaignGroup1Array, campaignGroup1)) {
campaignGroup1Array.put(campaignGroup1);
}
return campaignGroup1Array;
}

private String updateMaxCampaignDateEnd(String maxDateEnd, JSONObject mapTagEntry) throws JSONException {
String currentEndDate = mapTagEntry.getString("maxTagDateEnd");
if (StringUtil.isEmpty(maxDateEnd) || Timestamp.valueOf(currentEndDate).getTime() > Timestamp.valueOf(maxDateEnd).getTime()) {
return currentEndDate;
}
return maxDateEnd;
}

private String updateMinCampaignDateStart(String minDateStart, JSONObject mapTagEntry) throws JSONException {
String currentStartDate = mapTagEntry.getString("minTagDateStart");
if (StringUtil.isEmpty(minDateStart) || Timestamp.valueOf(currentStartDate).getTime() < Timestamp.valueOf(minDateStart).getTime()) {
return currentStartDate;
}
return minDateStart;
}
}
4 changes: 2 additions & 2 deletions source/src/main/java/org/cerberus/core/database/DocumentationDatabaseService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2823,9 +2823,9 @@ public ArrayList<String> getSqlDocumentation() {
b.append(",('page_campaignstatistics','applications_col','','en','Applications','',NULL)");
b.append(",('page_campaignstatistics','group1_col','','fr','Groupe 1','',NULL)");
b.append(",('page_campaignstatistics','group1_col','','en','Group 1','',NULL)");
b.append(",('page_campaignstatistics','minDateStart_col','','fr','Date de début minimale de la première exécution','',NULL)");
b.append(",('page_campaignstatistics','minDateStart_col','','fr','Date de début minimale','',NULL)");
b.append(",('page_campaignstatistics','minDateStart_col','','en','Minimum start date','',NULL)");
b.append(",('page_campaignstatistics','maxDateEnd_col','','fr','Date de fin maximale de la dernière exécution','',NULL)");
b.append(",('page_campaignstatistics','maxDateEnd_col','','fr','Date de fin maximale','',NULL)");
b.append(",('page_campaignstatistics','maxDateEnd_col','','en','Maximum end date','',NULL)");
b.append(",('page_campaignstatistics','avgOK_col','','fr','Pourcentage moyen de OK','',NULL)");
b.append(",('page_campaignstatistics','avgOK_col','','en','Average percentage of OK','',NULL)");
Expand Down
Loading

0 comments on commit 8771cfc

Please sign in to comment.