Skip to content

Commit 70367d1

Browse files
DavidBakerEffendiDavidBakerEffendi
authored
Joern Upgrades and Stripping of Old Functionality (#253)
* Removed incremental/hash functionality * Upgraded Joern * Removed source file support via Jimple * Handling new docker compose command * Commented out TG testing since licensing is an issue now
1 parent 9334c27 commit 70367d1

File tree

31 files changed

+289
-968
lines changed

31 files changed

+289
-968
lines changed

‎.github/workflows/ci.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ jobs:
2929
run: sbt "project neo4j" test
3030
# - name: Run TigerGraph tests
3131
# run: |
32-
# curl https://dl.tigergraph.com/enterprise-edition/gsql_client/tigergraph-3.9.3-1-gsql_client.jar \
32+
# curl https://dl.tigergraph.com/enterprise-edition/gsql_client/tigergraph-3.10.1-gsql_client.jar \
3333
# --output gsql_client.jar &&
3434
# export GSQL_HOME=`pwd`/gsql_client.jar
3535
# sbt "project tigergraph" test

‎build.sbt

+2-1
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ inThisBuild(
44
List(
55
organization := "com.github.plume-oss",
66
version := "2.0.0",
7-
scalaVersion := "3.3.1",
7+
scalaVersion := "3.4.1",
88
resolvers ++= Seq(
99
Resolver.mavenLocal,
1010
Resolver.mavenCentral,
@@ -36,6 +36,7 @@ libraryDependencies ++= Seq(
3636
"io.joern" %% "semanticcpg" % Versions.joern,
3737
"io.joern" %% "x2cpg" % Versions.joern,
3838
"io.joern" %% "jimple2cpg" % Versions.joern,
39+
"io.joern" %% "jimple2cpg" % Versions.joern % Test classifier "tests",
3940
"io.joern" %% "x2cpg" % Versions.joern % Test classifier "tests",
4041
"org.slf4j" % "slf4j-api" % Versions.slf4j,
4142
"org.apache.logging.log4j" % "log4j-core" % Versions.log4j % Test,

‎commons/build.sbt

+1-2
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,5 @@ name := "commons"
22

33
libraryDependencies ++= Seq(
44
"io.shiftleft" %% "overflowdb-traversal" % Versions.overflowDb,
5-
"io.shiftleft" %% "codepropertygraph" % Versions.codePropertyGraph,
6-
"org.lz4" % "lz4-java" % Versions.lz4
5+
"io.shiftleft" %% "codepropertygraph" % Versions.codePropertyGraph
76
)

‎commons/src/main/scala/com/github/plume/oss/util/HashUtil.scala

-44
This file was deleted.

‎drivers/base/src/main/scala/com/github/plume/oss/drivers/IDriver.scala

-40
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@ import scala.collection.mutable.ListBuffer
1515
*/
1616
trait IDriver extends AutoCloseable {
1717

18-
private val logger = LoggerFactory.getLogger(IDriver.getClass)
1918
// ID Tracking
2019
protected val currId = new AtomicLong(1)
2120
private val nodeId = TrieMap.empty[overflowdb.NodeOrDetachedNode, Long]
@@ -41,10 +40,6 @@ trait IDriver extends AutoCloseable {
4140
*/
4241
def bulkTx(dg: DiffOrBuilder): Int
4342

44-
/** Given filenames, will remove related TYPE, TYPE_DECL, METHOD (with AST children), and NAMESPACE_BLOCK.
45-
*/
46-
def removeSourceFiles(filenames: String*): Unit
47-
4843
/** Obtains properties from the specified node type and key(s). By default will return the ID property as one of the
4944
* keys as "id".
5045
*/
@@ -148,38 +143,3 @@ trait ISchemaSafeDriver extends IDriver {
148143
def buildSchemaPayload(): String
149144

150145
}
151-
152-
object IDriver {
153-
val STRING_DEFAULT: String = "<empty>"
154-
val INT_DEFAULT: Int = -1
155-
val LONG_DEFAULT: Long = -1L
156-
val BOOL_DEFAULT: Boolean = false
157-
val LIST_DEFAULT: Seq[String] = Seq.empty[String]
158-
159-
/** Given a property, returns its known default.
160-
*/
161-
def getPropertyDefault(prop: String): Any = {
162-
import PropertyNames.*
163-
prop match {
164-
case AST_PARENT_TYPE => STRING_DEFAULT
165-
case AST_PARENT_FULL_NAME => STRING_DEFAULT
166-
case NAME => STRING_DEFAULT
167-
case CODE => STRING_DEFAULT
168-
case ORDER => INT_DEFAULT
169-
case SIGNATURE => ""
170-
case ARGUMENT_INDEX => INT_DEFAULT
171-
case FULL_NAME => STRING_DEFAULT
172-
case TYPE_FULL_NAME => STRING_DEFAULT
173-
case TYPE_DECL_FULL_NAME => STRING_DEFAULT
174-
case IS_EXTERNAL => BOOL_DEFAULT
175-
case DISPATCH_TYPE => STRING_DEFAULT
176-
case LINE_NUMBER => INT_DEFAULT
177-
case COLUMN_NUMBER => INT_DEFAULT
178-
case LINE_NUMBER_END => INT_DEFAULT
179-
case COLUMN_NUMBER_END => INT_DEFAULT
180-
case OVERLAYS => LIST_DEFAULT
181-
case INHERITS_FROM_TYPE_FULL_NAME => LIST_DEFAULT
182-
case _ => STRING_DEFAULT
183-
}
184-
}
185-
}

‎drivers/base/src/main/scala/com/github/plume/oss/drivers/SchemaBuilder.scala

+166
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,166 @@
1+
package com.github.plume.oss.drivers
2+
3+
import io.shiftleft.codepropertygraph.generated.EdgeTypes
4+
import io.shiftleft.codepropertygraph.generated.nodes.*
5+
import org.slf4j.LoggerFactory
6+
7+
/** A single utility to build out the CPG schema in other databases.
8+
*/
9+
object SchemaBuilder {
10+
11+
private val logger = LoggerFactory.getLogger(getClass)
12+
13+
val STRING_DEFAULT: String = "<empty>"
14+
val INT_DEFAULT: Int = -1
15+
val LONG_DEFAULT: Long = -1L
16+
val BOOL_DEFAULT: Boolean = false
17+
val LIST_DEFAULT: Seq[String] = Seq.empty[String]
18+
19+
/** Given a property, returns its known default.
20+
*/
21+
def getPropertyDefault(prop: String): Any = {
22+
import io.shiftleft.codepropertygraph.generated.PropertyNames.*
23+
prop match {
24+
case AST_PARENT_TYPE => STRING_DEFAULT
25+
case AST_PARENT_FULL_NAME => STRING_DEFAULT
26+
case NAME => STRING_DEFAULT
27+
case CODE => STRING_DEFAULT
28+
case ORDER => INT_DEFAULT
29+
case SIGNATURE => ""
30+
case ARGUMENT_INDEX => INT_DEFAULT
31+
case FULL_NAME => STRING_DEFAULT
32+
case TYPE_FULL_NAME => STRING_DEFAULT
33+
case TYPE_DECL_FULL_NAME => STRING_DEFAULT
34+
case IS_EXTERNAL => BOOL_DEFAULT
35+
case DISPATCH_TYPE => STRING_DEFAULT
36+
case LINE_NUMBER => INT_DEFAULT
37+
case COLUMN_NUMBER => INT_DEFAULT
38+
case LINE_NUMBER_END => INT_DEFAULT
39+
case COLUMN_NUMBER_END => INT_DEFAULT
40+
case OVERLAYS => LIST_DEFAULT
41+
case INHERITS_FROM_TYPE_FULL_NAME => LIST_DEFAULT
42+
case POSSIBLE_TYPES => LIST_DEFAULT
43+
case _ => STRING_DEFAULT
44+
}
45+
}
46+
47+
/** Edges that should be specified as being between any kind of vertex.
48+
*/
49+
val WILDCARD_EDGE_LABELS: Set[String] =
50+
Set(EdgeTypes.EVAL_TYPE, EdgeTypes.REF, EdgeTypes.INHERITS_FROM, EdgeTypes.ALIAS_OF)
51+
52+
/** Determines if an edge type between two node types is valid.
53+
*/
54+
def checkEdgeConstraint(from: String, to: String, edge: String): Boolean = {
55+
val fromCheck = from match {
56+
case MetaData.Label => MetaData.Edges.Out.contains(edge)
57+
case File.Label => File.Edges.Out.contains(edge)
58+
case Method.Label => Method.Edges.Out.contains(edge)
59+
case MethodParameterIn.Label => MethodParameterIn.Edges.Out.contains(edge)
60+
case MethodParameterOut.Label => MethodParameterOut.Edges.Out.contains(edge)
61+
case MethodReturn.Label => MethodReturn.Edges.Out.contains(edge)
62+
case Modifier.Label => Modifier.Edges.Out.contains(edge)
63+
case Type.Label => Type.Edges.Out.contains(edge)
64+
case TypeDecl.Label => TypeDecl.Edges.Out.contains(edge)
65+
case TypeParameter.Label => TypeParameter.Edges.Out.contains(edge)
66+
case TypeArgument.Label => TypeArgument.Edges.Out.contains(edge)
67+
case Member.Label => Member.Edges.Out.contains(edge)
68+
case Namespace.Label => Namespace.Edges.Out.contains(edge)
69+
case NamespaceBlock.Label => NamespaceBlock.Edges.Out.contains(edge)
70+
case Literal.Label => Literal.Edges.Out.contains(edge)
71+
case Call.Label => Call.Edges.Out.contains(edge)
72+
case ClosureBinding.Label => ClosureBinding.Edges.Out.contains(edge)
73+
case Local.Label => Local.Edges.Out.contains(edge)
74+
case Identifier.Label => Identifier.Edges.Out.contains(edge)
75+
case FieldIdentifier.Label => FieldIdentifier.Edges.Out.contains(edge)
76+
case Return.Label => Return.Edges.Out.contains(edge)
77+
case Block.Label => Block.Edges.Out.contains(edge)
78+
case MethodRef.Label => MethodRef.Edges.Out.contains(edge)
79+
case TypeRef.Label => TypeRef.Edges.Out.contains(edge)
80+
case JumpTarget.Label => JumpTarget.Edges.Out.contains(edge)
81+
case ControlStructure.Label => ControlStructure.Edges.Out.contains(edge)
82+
case Annotation.Label => Annotation.Edges.Out.contains(edge)
83+
case AnnotationLiteral.Label => AnnotationLiteral.Edges.Out.contains(edge)
84+
case AnnotationParameter.Label => AnnotationParameter.Edges.Out.contains(edge)
85+
case AnnotationParameterAssign.Label => AnnotationParameterAssign.Edges.Out.contains(edge)
86+
case Unknown.Label => Unknown.Edges.Out.contains(edge)
87+
case x =>
88+
logger.warn(s"Unhandled node type '$x'")
89+
false
90+
}
91+
val toCheck = to match {
92+
case MetaData.Label => MetaData.Edges.In.contains(edge)
93+
case File.Label => File.Edges.In.contains(edge)
94+
case Method.Label => Method.Edges.In.contains(edge)
95+
case MethodParameterIn.Label => MethodParameterIn.Edges.In.contains(edge)
96+
case MethodParameterOut.Label => MethodParameterOut.Edges.In.contains(edge)
97+
case MethodReturn.Label => MethodReturn.Edges.In.contains(edge)
98+
case Modifier.Label => Modifier.Edges.In.contains(edge)
99+
case Type.Label => Type.Edges.In.contains(edge)
100+
case TypeDecl.Label => TypeDecl.Edges.In.contains(edge)
101+
case TypeParameter.Label => TypeParameter.Edges.In.contains(edge)
102+
case TypeArgument.Label => TypeArgument.Edges.In.contains(edge)
103+
case Member.Label => Member.Edges.In.contains(edge)
104+
case Namespace.Label => Namespace.Edges.In.contains(edge)
105+
case NamespaceBlock.Label => NamespaceBlock.Edges.In.contains(edge)
106+
case Literal.Label => Literal.Edges.In.contains(edge)
107+
case Call.Label => Call.Edges.In.contains(edge)
108+
case ClosureBinding.Label => ClosureBinding.Edges.Out.contains(edge)
109+
case Local.Label => Local.Edges.In.contains(edge)
110+
case Identifier.Label => Identifier.Edges.In.contains(edge)
111+
case FieldIdentifier.Label => FieldIdentifier.Edges.In.contains(edge)
112+
case Return.Label => Return.Edges.In.contains(edge)
113+
case Block.Label => Block.Edges.In.contains(edge)
114+
case MethodRef.Label => MethodRef.Edges.In.contains(edge)
115+
case TypeRef.Label => TypeRef.Edges.In.contains(edge)
116+
case JumpTarget.Label => JumpTarget.Edges.In.contains(edge)
117+
case ControlStructure.Label => ControlStructure.Edges.In.contains(edge)
118+
case Annotation.Label => Annotation.Edges.In.contains(edge)
119+
case AnnotationLiteral.Label => AnnotationLiteral.Edges.In.contains(edge)
120+
case AnnotationParameter.Label => AnnotationParameter.Edges.In.contains(edge)
121+
case AnnotationParameterAssign.Label => AnnotationParameterAssign.Edges.In.contains(edge)
122+
case Unknown.Label => Unknown.Edges.In.contains(edge)
123+
case x =>
124+
logger.warn(s"Unhandled node type '$x'")
125+
false
126+
}
127+
128+
fromCheck && toCheck
129+
}
130+
131+
def allProperties: Set[String] = NodeToProperties.flatMap(_._2).toSet
132+
133+
val NodeToProperties: Map[String, Set[String]] = Map(
134+
MetaData.Label -> MetaData.PropertyNames.all,
135+
File.Label -> File.PropertyNames.all,
136+
Method.Label -> Method.PropertyNames.all,
137+
MethodParameterIn.Label -> MethodParameterIn.PropertyNames.all,
138+
MethodParameterOut.Label -> MethodParameterOut.PropertyNames.all,
139+
MethodReturn.Label -> MethodReturn.PropertyNames.all,
140+
Modifier.Label -> Modifier.PropertyNames.all,
141+
Type.Label -> Type.PropertyNames.all,
142+
TypeDecl.Label -> TypeDecl.PropertyNames.all,
143+
TypeParameter.Label -> TypeParameter.PropertyNames.all,
144+
TypeArgument.Label -> TypeArgument.PropertyNames.all,
145+
Member.Label -> Member.PropertyNames.all,
146+
Namespace.Label -> Namespace.PropertyNames.all,
147+
NamespaceBlock.Label -> NamespaceBlock.PropertyNames.all,
148+
Literal.Label -> Literal.PropertyNames.all,
149+
Call.Label -> Call.PropertyNames.all,
150+
Local.Label -> Local.PropertyNames.all,
151+
Identifier.Label -> Identifier.PropertyNames.all,
152+
FieldIdentifier.Label -> FieldIdentifier.PropertyNames.all,
153+
Return.Label -> Return.PropertyNames.all,
154+
Block.Label -> Block.PropertyNames.all,
155+
MethodRef.Label -> MethodRef.PropertyNames.all,
156+
TypeRef.Label -> TypeRef.PropertyNames.all,
157+
JumpTarget.Label -> JumpTarget.PropertyNames.all,
158+
ControlStructure.Label -> ControlStructure.PropertyNames.all,
159+
Annotation.Label -> Annotation.PropertyNames.all,
160+
AnnotationLiteral.Label -> AnnotationLiteral.PropertyNames.all,
161+
AnnotationParameter.Label -> AnnotationParameter.PropertyNames.all,
162+
AnnotationParameterAssign.Label -> AnnotationParameterAssign.PropertyNames.all,
163+
Unknown.Label -> Unknown.PropertyNames.all
164+
)
165+
166+
}

‎drivers/base/src/test/scala/com/github/plume/oss/DockerManager.scala

+10-8
Original file line numberDiff line numberDiff line change
@@ -3,31 +3,33 @@ package com.github.plume.oss
33
import io.circe.Json
44
import org.slf4j.LoggerFactory
55

6-
import java.io.{File => JavaFile}
6+
import better.files.File
77
import scala.collection.mutable.ListBuffer
88
import scala.sys.process.{Process, ProcessLogger, stringSeqToProcess}
99

1010
object DockerManager {
1111

1212
private val logger = LoggerFactory.getLogger(getClass)
1313

14-
def toDockerComposeFile(dbName: String): JavaFile =
15-
new JavaFile(getClass.getResource(s"/docker/$dbName.yml").toURI)
14+
def toDockerComposeFile(dbName: String)(implicit classLoader: ClassLoader): File =
15+
File(classLoader.getResource(s"docker/$dbName.yml").toURI)
1616

17-
def closeAnyDockerContainers(dbName: String): Unit = {
17+
def closeAnyDockerContainers(dbName: String)(implicit classLoader: ClassLoader): Unit = {
1818
logger.info(s"Stopping Docker services for $dbName...")
19-
val dockerComposeUp = Process(Seq("docker-compose", "-f", toDockerComposeFile(dbName).getAbsolutePath, "down"))
20-
dockerComposeUp.run(ProcessLogger(_ => ()))
19+
val dockerComposeDown = Process(Seq("docker", "compose", "-f", toDockerComposeFile(dbName).pathAsString, "down"))
20+
dockerComposeDown.run(ProcessLogger(_ => ()))
2121
}
2222

23-
def startDockerFile(dbName: String, containers: List[String] = List.empty[String]): Unit = {
23+
def startDockerFile(dbName: String, containers: List[String] = List.empty[String])(implicit
24+
classLoader: ClassLoader
25+
): Unit = {
2426
val healthChecks = ListBuffer.empty[String]
2527
if (containers.isEmpty) healthChecks += dbName else healthChecks ++= containers
2628
logger.info(s"Docker Compose file found for $dbName, starting...")
2729
closeAnyDockerContainers(dbName) // Easiest way to clear the db
2830
Thread.sleep(3000)
2931
val dockerComposeUp = Process(
30-
Seq("docker-compose", "-f", toDockerComposeFile(dbName).getAbsolutePath, "up", "--remove-orphans")
32+
Seq("docker", "compose", "-f", toDockerComposeFile(dbName).pathAsString, "up", "--remove-orphans")
3133
)
3234
logger.info(s"Starting process $dockerComposeUp")
3335
dockerComposeUp.run(ProcessLogger(_ => ()))

‎drivers/base/src/test/scala/com/github/plume/oss/testfixtures/PlumeDriverFixture.scala

+1-18
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ class PlumeDriverFixture(val driver: IDriver)
3434
val props: Array[Object] = n.properties.flatMap { case (k, v) =>
3535
Iterable(k.asInstanceOf[Object], v.asInstanceOf[Object])
3636
}.toArray
37-
new DetachedNodeGeneric(n.label(), props: _*)
37+
new DetachedNodeGeneric(n.label(), props*)
3838
}
3939

4040
"overflowdb.BatchedUpdate.DiffGraph based changes" should {
@@ -97,23 +97,6 @@ class PlumeDriverFixture(val driver: IDriver)
9797
}
9898
}
9999

100-
"should delete a source file's nodes precisely" in {
101-
val diffGraph = new DiffGraphBuilder
102-
// Create some basic method
103-
createSimpleGraph(diffGraph)
104-
driver.bulkTx(diffGraph.build())
105-
106-
// remove f1 nodes
107-
driver.removeSourceFiles(f1.name)
108-
109-
val List(m: Map[String, Any]) = driver.propertyFromNodes(METHOD, NAME)
110-
val List(td: Map[String, Any]) = driver.propertyFromNodes(TYPE_DECL, NAME)
111-
val List(n: Map[String, Any]) = driver.propertyFromNodes(NAMESPACE_BLOCK, NAME)
112-
m.getOrElse(NAME, -1L).asInstanceOf[String] shouldBe m2.name
113-
td.getOrElse(NAME, -1L).asInstanceOf[String] shouldBe td2.name
114-
n.getOrElse(NAME, -1L).asInstanceOf[String] shouldBe n2.name
115-
}
116-
117100
override def afterAll(): Unit = {
118101
if (driver.isConnected) driver.close()
119102
}

0 commit comments

Comments
 (0)