From e4911cb2c863456b007f6654c314f73707ecdd45 Mon Sep 17 00:00:00 2001 From: Ondrej Zizka Date: Tue, 26 Dec 2017 11:11:04 +0100 Subject: [PATCH 1/2] Upgrade Maven plugins --- victi.ms/pom.xml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/victi.ms/pom.xml b/victi.ms/pom.xml index 5259429..776fe02 100644 --- a/victi.ms/pom.xml +++ b/victi.ms/pom.xml @@ -146,7 +146,7 @@ org.apache.maven.plugins maven-surefire-plugin - 2.18.1 + 2.19.1 -Xms512m -Xmx2048m -XX:MaxPermSize=768m -XX:ReservedCodeCacheSize=128m false @@ -158,7 +158,7 @@ org.jboss.forge.furnace furnace-maven-plugin - 2.25.2.Final + 2.25.4.Final generate-dot @@ -170,7 +170,7 @@ maven-jar-plugin - 2.5 + 3.0.2 create-forge-addon @@ -184,7 +184,7 @@ maven-dependency-plugin - 2.9 + 2.10 copy-test-jar From 1409910121889cd61f04ba79ffdd98af82f96fc0 Mon Sep 17 00:00:00 2001 From: Ondrej Zizka Date: Tue, 26 Dec 2017 23:42:48 +0100 Subject: [PATCH 2/2] Proper usage of the Victims API. --- victi.ms/pom.xml | 5 +- .../CheckArchivesWithVictimsRules.java | 2 +- ...a => ComputeArchivesVictimsHashRules.java} | 38 ++++++++--- .../qs/victims/test/VictimsLibTest.java | 63 +++++++++++++++++-- .../qs/victims/test/VictimsRulesetTest.java | 5 +- 5 files changed, 95 insertions(+), 18 deletions(-) rename victi.ms/src/main/java/org/jboss/windup/qs/victims/{ComputeArchivesSHA512Rules.java => ComputeArchivesVictimsHashRules.java} (50%) diff --git a/victi.ms/pom.xml b/victi.ms/pom.xml index 776fe02..f5313a9 100644 --- a/victi.ms/pom.xml +++ b/victi.ms/pom.xml @@ -112,11 +112,11 @@ junit junit 4.12 - jar + test - + xerces xercesImpl @@ -150,6 +150,7 @@ -Xms512m -Xmx2048m -XX:MaxPermSize=768m -XX:ReservedCodeCacheSize=128m false + alphabetical diff --git a/victi.ms/src/main/java/org/jboss/windup/qs/victims/CheckArchivesWithVictimsRules.java b/victi.ms/src/main/java/org/jboss/windup/qs/victims/CheckArchivesWithVictimsRules.java index 5618143..62d6540 100644 --- a/victi.ms/src/main/java/org/jboss/windup/qs/victims/CheckArchivesWithVictimsRules.java +++ b/victi.ms/src/main/java/org/jboss/windup/qs/victims/CheckArchivesWithVictimsRules.java @@ -75,7 +75,7 @@ public Configuration getConfiguration(final RuleLoaderContext ruleLoaderContext) public void perform(GraphRewrite event, EvaluationContext context, ArchiveModel archive) { log.info("\tVicti.ms checking archive: " + archive.getFilePath()); GraphService vulGS = new GraphService(event.getGraphContext(), VulnerabilityModel.class); - String hash = archive.asVertex().getProperty(ComputeArchivesSHA512Rules.KEY_SHA512); + String hash = archive.asVertex().getProperty(ComputeArchivesVictimsHashRules.KEY_VICTIMS_HASH); try { HashSet vuls = db.getVulnerabilities(hash); if (vuls.isEmpty()) diff --git a/victi.ms/src/main/java/org/jboss/windup/qs/victims/ComputeArchivesSHA512Rules.java b/victi.ms/src/main/java/org/jboss/windup/qs/victims/ComputeArchivesVictimsHashRules.java similarity index 50% rename from victi.ms/src/main/java/org/jboss/windup/qs/victims/ComputeArchivesSHA512Rules.java rename to victi.ms/src/main/java/org/jboss/windup/qs/victims/ComputeArchivesVictimsHashRules.java index d068d3a..57c1708 100644 --- a/victi.ms/src/main/java/org/jboss/windup/qs/victims/ComputeArchivesSHA512Rules.java +++ b/victi.ms/src/main/java/org/jboss/windup/qs/victims/ComputeArchivesVictimsHashRules.java @@ -1,9 +1,15 @@ package org.jboss.windup.qs.victims; +import com.redhat.victims.VictimsConfig; +import com.redhat.victims.VictimsRecord; +import com.redhat.victims.VictimsScanner; +import com.redhat.victims.fingerprint.*; import java.io.IOException; import java.io.InputStream; +import java.util.ArrayList; import org.apache.commons.codec.digest.DigestUtils; +import org.apache.commons.io.IOUtils; import org.jboss.windup.config.GraphRewrite; import org.jboss.windup.config.metadata.RuleMetadata; import org.jboss.windup.config.phase.ArchiveExtractionPhase; @@ -16,15 +22,14 @@ import org.ocpsoft.rewrite.context.EvaluationContext; /** - * Calculates SHA512 hash for each archive. + * Calculates the Victims proprietary normalized hash for each archive. * * @author Ondrej Zizka - * */ @RuleMetadata(tags = { "java" }, after = { UnzipArchivesToOutputRuleProvider.class }, phase = ArchiveExtractionPhase.class) -public class ComputeArchivesSHA512Rules extends IteratingRuleProvider +public class ComputeArchivesVictimsHashRules extends IteratingRuleProvider { - public static final String KEY_SHA512 = "SHA512"; + public static final String KEY_VICTIMS_HASH = "VICTIMS_HASH"; @Override @@ -39,17 +44,36 @@ public void perform(GraphRewrite event, EvaluationContext context, ArchiveModel { try (InputStream is = archive.asInputStream()) { - String hash = DigestUtils.sha512Hex(is); - archive.asVertex().setProperty(KEY_SHA512, hash); + String hash = computeVictimsHash(is, archive.getFileName()); + archive.asVertex().setProperty(KEY_VICTIMS_HASH, hash); } catch (IOException e) { throw new WindupException("Failed to read archive: " + archive.getFilePath() + - "\n Due to: " + e.getMessage(), e); + "\n Due to: " + e.getMessage(), e); } } // @formatter:on + public static String computeVictimsHash(InputStream is, String fileName) throws IOException + { + // The Victims API is not much understandable so this may look chaotic. + + /* + Artifact artifact = Processor.process(is, archive.getFileName()); + ArrayList records = new ArrayList(); + VictimsScanner.scanArtifact(artifact, new VictimsScanner.ArrayOutputStream(records)); + return records.get(0).hash; + */ + + // This only gives a simple hash? + //Fingerprint fingerprint = Processor.fingerprint(IOUtils.toByteArray(is)); + //return fingerprint.get(VictimsConfig.DEFAULT_ALGORITHM_STRING); + + JarFile jarFile = new JarFile(is, fileName); + return jarFile.getFingerprint().get(Algorithms.SHA512); + } + @Override public String toStringPerform() diff --git a/victi.ms/src/test/java/org/jboss/windup/qs/victims/test/VictimsLibTest.java b/victi.ms/src/test/java/org/jboss/windup/qs/victims/test/VictimsLibTest.java index 4ea4189..1897ebe 100644 --- a/victi.ms/src/test/java/org/jboss/windup/qs/victims/test/VictimsLibTest.java +++ b/victi.ms/src/test/java/org/jboss/windup/qs/victims/test/VictimsLibTest.java @@ -3,9 +3,19 @@ import com.redhat.victims.VictimsException; import com.redhat.victims.database.VictimsDB; import com.redhat.victims.database.VictimsDBInterface; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; import java.io.IOException; +import java.util.HashSet; +import java.util.Spliterators; import java.util.logging.Logger; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; +import org.jboss.windup.qs.victims.ComputeArchivesVictimsHashRules; import org.jboss.windup.util.Logging; +import org.junit.Assert; +import org.junit.Ignore; import org.junit.Test; /** @@ -18,18 +28,20 @@ public class VictimsLibTest private static final Logger log = Logging.get(VictimsLibTest.class); - // Path to a jar known to contain a vulnerability. - private static final String BAD_JAR = "target/testJars/xercesImpl-2.9.1.jar"; + // Path to a jars known to contain a vulnerability. + private static final String VULNERABLE_JAR1_PATH = "target/testJars/xercesImpl-2.9.1.jar"; + // Looks like the Xerces vulnerability is not in the Victims database. Adding another one. + private static final String VULNERABLE_JAR2_PATH = "src/test/resources/commons-fileupload-1.0-beta-1.jar"; // SHA-512 checksum of xerces:xercesImpl:2.9.1 - private static final String BAD_JAR_SHA512 = "ec2200e5a5a70f5c64744f6413a546f5e4979b3fb1649b02756ff035d36dde31170eaadc70842230296b60896f04877270c26b40415736299aef44ac16c5811c"; + private static final String VULNERABLE_JAR1_SHA512 = "ec2200e5a5a70f5c64744f6413a546f5e4979b3fb1649b02756ff035d36dde31170eaadc70842230296b60896f04877270c26b40415736299aef44ac16c5811c"; - // Contained in FILEHASHES table. - private static final String BAD_SHA512 = "851eba12748a1aada5829e3a8e2eba05435efaaef9f0e7f68f6246dc1f6407ca56830ef00d587e91c3d889bb70eaf605a305652479ba6986a90b3986f0e74daf"; + // Contained in FILEHASHES table. Not sure if it is supposed to be found by Victims API. + private static final String SOME_VICTIMS_HASH = "851eba12748a1aada5829e3a8e2eba05435efaaef9f0e7f68f6246dc1f6407ca56830ef00d587e91c3d889bb70eaf605a305652479ba6986a90b3986f0e74daf"; @Test - public void testUpdate() throws IOException, VictimsException + public void test01Update() throws IOException, VictimsException { try { VictimsDBInterface db = VictimsDB.db(); @@ -38,6 +50,7 @@ public void testUpdate() throws IOException, VictimsException // Update (goes to ~/.victims) db.synchronize(); System.out.println(" DB records: " + db.getRecordCount()); + Assert.assertTrue("DB has some recods after update.", db.getRecordCount() > 0); System.out.println("Database last updated on: " + db.lastUpdated().toString()); } catch (VictimsException ex){ @@ -49,4 +62,42 @@ public void testUpdate() throws IOException, VictimsException } } + @Test @Ignore + public void test02IdentifyVulnerableJarHash(){ + try + { + VictimsDBInterface db = VictimsDB.db(); + final HashSet vulnerabilities = db.getVulnerabilities(SOME_VICTIMS_HASH); + Assert.assertTrue("Found some vulnerability for hash " + SOME_VICTIMS_HASH, !vulnerabilities.isEmpty()); + } + catch (VictimsException ex){ + // Prevent failure if offline. Just a warning. + throw new RuntimeException("Failed when identifying a vulnerable jar", ex); + } + } + + @Test + public void test03IdentifyVulnerableXercesJarHash(){ + try + { + final File vulnerableJar = new File(VULNERABLE_JAR2_PATH); + final String hash = ComputeArchivesVictimsHashRules.computeVictimsHash(new FileInputStream(vulnerableJar), vulnerableJar.getName()); + + VictimsDBInterface db = VictimsDB.db(); + final HashSet vulnerabilities = db.getVulnerabilities(hash); + Assert.assertTrue("Found some vulnerability for hash " + hash, !vulnerabilities.isEmpty()); + log.info(String.format("Vulnerabilities found in %s: ", vulnerableJar.getPath()) + StreamSupport.stream(vulnerabilities.spliterator(), false).collect(Collectors.joining(", "))); + } + catch (VictimsException ex){ + // Prevent failure if offline. Just a warning. + throw new RuntimeException("Failed when identifying a vulnerable jar", ex); + } + catch (FileNotFoundException e) { + e.printStackTrace(); + } + catch (IOException e) { + e.printStackTrace(); + } + } + } diff --git a/victi.ms/src/test/java/org/jboss/windup/qs/victims/test/VictimsRulesetTest.java b/victi.ms/src/test/java/org/jboss/windup/qs/victims/test/VictimsRulesetTest.java index e72d271..2c787ad 100644 --- a/victi.ms/src/test/java/org/jboss/windup/qs/victims/test/VictimsRulesetTest.java +++ b/victi.ms/src/test/java/org/jboss/windup/qs/victims/test/VictimsRulesetTest.java @@ -26,6 +26,7 @@ import org.jboss.windup.rules.apps.java.config.SourceModeOption; import org.jboss.windup.util.Logging; import org.junit.Assert; +import org.junit.Ignore; import org.junit.Test; import org.junit.runner.RunWith; @@ -61,7 +62,7 @@ public static AddonArchive getDeployment() @Inject private GraphContextFactory contextFactory; - @Test + @Test @Ignore public void testAffectedJarsFound() throws Exception { try (GraphContext ctx = contextFactory.create()) @@ -85,7 +86,7 @@ public void testAffectedJarsFound() throws Exception boolean found = false; for (AffectedJarModel jar : jarsGS.findAll()) { - log.info(jar.getFilePath()); + log.info("\n\n*************************\nVulnerabilities for file " + jar.getFilePath()); found = true; for (VulnerabilityModel vul : jar.getVulnerabilities()) log.info(" " + vul.getCve());