Skip to content

Commit ebf02f9

Browse files
renovate-botolafurpg
authored andcommitted
Add server to host sources of published Java libraries.
Previously, Sourcegraph was only able to LSIF index Java repositories. There was no good way to LSIF index a Java library on Maven Central because they don't have an associated git repository. This commit adds a new `PackageHub` server that is implemented as a proxy on top of Maven repositories that serves git repositories that Sourcegraph can understand. This new server can be used with lsif-java to enable navigation between repositories and their transitive Java library dependencies.
1 parent 2d9fc92 commit ebf02f9

File tree

30 files changed

+1084
-30
lines changed

30 files changed

+1084
-30
lines changed

.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -55,3 +55,4 @@ test-report.json
5555
dump.lsif
5656

5757
./generated
58+
/sources

Dockerfile

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
FROM openjdk:8-jdk-alpine
2+
COPY bin/coursier coursier
3+
RUN apk add --no-cache git curl \
4+
&& git config --global user.email "[email protected]" \
5+
&& git config --global user.name "Your Name" \
6+
&& git config --global http.postBuffer 1048576000 \
7+
&& curl -L https://sourcegraph.com/.api/src-cli/src_linux_amd64 -o /src \
8+
&& chmod +x /src \
9+
&& /coursier bootstrap -r sonatype:snapshots com.sourcegraph:packagehub_2.13:0.5.0-12-69905fcb-SNAPSHOT -o /packagehub
10+
ENV COURSIER_REPOSITORIES=central|https://maven.google.com/|jitpack
11+
ENTRYPOINT /packagehub --host 0.0.0.0 --port $PORT --src /src --coursier /coursier --postgres.username=$DB_USER --postgres.password=$DB_PASS --postgres.url=$DB_URL --auto-index-delay=PT1M

Dockerfile.template

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
FROM openjdk:8-jdk-alpine
2+
COPY bin/coursier coursier
3+
RUN apk add --no-cache git curl \
4+
&& git config --global user.email "[email protected]" \
5+
&& git config --global user.name "Your Name" \
6+
&& git config --global http.postBuffer 1048576000 \
7+
&& curl -L https://sourcegraph.com/.api/src-cli/src_linux_amd64 -o /src \
8+
&& chmod +x /src \
9+
&& /coursier bootstrap -r sonatype:snapshots com.sourcegraph:packagehub_2.13:VERSION -o /packagehub
10+
ENV COURSIER_REPOSITORIES=central|https://maven.google.com/|jitpack
11+
ENTRYPOINT /packagehub --host 0.0.0.0 --port $PORT --src /src --coursier /coursier --postgres.username=$DB_USER --postgres.password=$DB_PASS --postgres.url=$DB_URL --auto-index-delay=PT1M

build.sbt

+43-1
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ lazy val V =
1515
def scala213 = "2.13.4"
1616
def scala212 = "2.12.12"
1717
def scalameta = "4.4.8"
18+
def testcontainers = "0.39.3"
1819
def requests = "0.6.5"
1920
}
2021

@@ -195,6 +196,45 @@ lazy val cli = project
195196
)
196197
.enablePlugins(NativeImagePlugin, BuildInfoPlugin)
197198
.dependsOn(lsif)
199+
commands +=
200+
Command.command("dockerfile") { s =>
201+
"commitall" :: "reload" :: "packagehub/dockerfileUpdate" :: "publish" :: s
202+
}
203+
204+
def commitAll(): Unit = {
205+
import scala.sys.process._
206+
"git add .".!!
207+
"git commit --allow-empty -m WIP".!!
208+
209+
}
210+
commands +=
211+
Command.command("commitall") { s =>
212+
commitAll()
213+
s
214+
}
215+
216+
lazy val packagehub = project
217+
.in(file("packagehub"))
218+
.settings(
219+
moduleName := "packagehub",
220+
mainClass.in(Compile) := Some("com.sourcegraph.packagehub.PackageHub"),
221+
TaskKey[Unit]("dockerfileUpdate") := {
222+
val template = IO.read(file("Dockerfile.template"))
223+
IO.write(file("Dockerfile"), template.replace("VERSION", version.value))
224+
commitAll()
225+
},
226+
libraryDependencies ++=
227+
List(
228+
"com.google.cloud.sql" % "postgres-socket-factory" % "1.2.1",
229+
"com.zaxxer" % "HikariCP" % "4.0.3",
230+
"org.flywaydb" % "flyway-core" % "7.7.1",
231+
"org.postgresql" % "postgresql" % "42.2.14",
232+
"org.scalameta" %% "scalameta" % V.scalameta,
233+
"com.lihaoyi" %% "cask" % "0.7.8"
234+
)
235+
)
236+
.enablePlugins(AssemblyPlugin)
237+
.dependsOn(cli)
198238

199239
commands +=
200240
Command.command("nativeImageProfiled") { s =>
@@ -273,7 +313,7 @@ lazy val unit = project
273313
),
274314
buildInfoPackage := "tests"
275315
)
276-
.dependsOn(plugin, cli)
316+
.dependsOn(plugin, cli, packagehub)
277317
.enablePlugins(BuildInfoPlugin)
278318

279319
lazy val buildTools = project
@@ -344,6 +384,8 @@ lazy val testSettings = List(
344384
libraryDependencies ++=
345385
List(
346386
"org.scalameta" %% "munit" % "0.7.23",
387+
"com.dimafeng" %% "testcontainers-scala-munit" % V.testcontainers,
388+
"com.dimafeng" %% "testcontainers-scala-postgresql" % V.testcontainers,
347389
"org.scalameta" %% "moped-testkit" % V.moped,
348390
"org.scalameta" %% "scalameta" % V.scalameta,
349391
"io.get-coursier" %% "coursier" % V.coursier,

lsif-java/src/main/scala/com/sourcegraph/lsif_java/buildtools/LsifBuildTool.scala

+2-1
Original file line numberDiff line numberDiff line change
@@ -167,6 +167,7 @@ class LsifBuildTool(index: IndexCommand) extends BuildTool("LSIF", index) {
167167
/** Recursively collects all Java files in the working directory */
168168
private def collectAllJavaFiles(dir: Path): List[Path] = {
169169
val javaPattern = FileSystems.getDefault.getPathMatcher("glob:**.java")
170+
val moduleInfo = Paths.get("module-info.java")
170171
val buf = ListBuffer.empty[Path]
171172
Files.walkFileTree(
172173
dir,
@@ -184,7 +185,7 @@ class LsifBuildTool(index: IndexCommand) extends BuildTool("LSIF", index) {
184185
file: Path,
185186
attrs: BasicFileAttributes
186187
): FileVisitResult = {
187-
if (javaPattern.matches(file)) {
188+
if (javaPattern.matches(file) && !file.endsWith(moduleInfo)) {
188189
buf += file
189190
}
190191
FileVisitResult.CONTINUE
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
CREATE TABLE packages (
2+
id VARCHAR(1000) NOT NULL PRIMARY KEY
3+
);
4+
5+
CREATE TABLE indexed_packages (
6+
id VARCHAR(1000) NOT NULL PRIMARY KEY
7+
);
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
package com.sourcegraph.packagehub
2+
3+
import moped.json.DecodingContext
4+
import moped.json.ErrorResult
5+
import moped.json.JsonCodec
6+
import moped.json.JsonElement
7+
import moped.json.JsonString
8+
import moped.json.Result
9+
import moped.macros.ClassShape
10+
import moped.reporters.Diagnostic
11+
12+
/**
13+
* Codec that always fails the decoding step.
14+
*
15+
* Useful for types that cannot be configured from the command-line.
16+
*/
17+
class EmptyJsonCodec[T] extends JsonCodec[T] {
18+
def decode(context: DecodingContext): Result[T] =
19+
ErrorResult(Diagnostic.error(s"not supported: $context"))
20+
def encode(value: T): JsonElement = JsonString(value.toString())
21+
def shape: ClassShape = ClassShape.empty
22+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
package com.sourcegraph.packagehub
2+
3+
import java.nio.file.Path
4+
import java.nio.file.Paths
5+
6+
import scala.util.control.NonFatal
7+
8+
import com.sourcegraph.lsif_java.Dependencies
9+
import coursier.core.Dependency
10+
import coursier.core.Module
11+
import coursier.core.ModuleName
12+
import coursier.core.Organization
13+
import ujson.Obj
14+
15+
/**
16+
* Package represents a published library such as a Java artifact, or the JDK.
17+
*
18+
* @param id
19+
* unique representation for this package that does not include the forward
20+
* slash character. Can be used as the primary key in a relational database.
21+
* Should ideally be human-readable and be easy to parse.
22+
* @param path
23+
* relative URL of this package.
24+
*/
25+
sealed abstract class Package(
26+
val id: String,
27+
val path: String,
28+
val version: String
29+
) {
30+
def toJsonRepo: Obj = Obj("Name" -> path, "URI" -> s"/repos/$path")
31+
def relativePath: Path = Paths.get(path)
32+
}
33+
object Package {
34+
def jdk(version: String): JdkPackage = {
35+
JdkPackage(version)
36+
}
37+
def maven(org: String, name: String, version: String): MavenPackage = {
38+
MavenPackage(
39+
Dependency(
40+
Module(Organization(org), ModuleName(name), Map.empty),
41+
version
42+
)
43+
)
44+
}
45+
def parse(value: String): Package = {
46+
value match {
47+
case s"jdk:$version" =>
48+
JdkPackage(version)
49+
case s"maven:$library" =>
50+
val Right(dep) = Dependencies.parseDependencyEither(library)
51+
MavenPackage(dep)
52+
}
53+
}
54+
def fromPath(path: List[String]): Option[(Package, List[String])] =
55+
path match {
56+
case "maven" :: org :: name :: version :: requestPath =>
57+
Some(Package.maven(org, name, version) -> requestPath)
58+
case "jdk" :: version :: requestPath =>
59+
Some(Package.jdk(version) -> requestPath)
60+
case _ =>
61+
None
62+
}
63+
def fromString(value: String, coursier: String): Either[String, Package] = {
64+
value match {
65+
case s"jdk:$version" =>
66+
val exit = os
67+
.proc(coursier, "java-home", "--jvm", version)
68+
.call(check = false)
69+
if (exit.exitCode == 0)
70+
Right(JdkPackage(version))
71+
else
72+
Left(exit.out.trim())
73+
case s"maven:$library" =>
74+
Dependencies
75+
.parseDependencyEither(library)
76+
.flatMap { dep =>
77+
try {
78+
// Report an error if the dependency can't be resolved.
79+
Dependencies.resolveProvidedDeps(dep)
80+
Right(MavenPackage(dep))
81+
} catch {
82+
case NonFatal(e) =>
83+
Left(e.getMessage())
84+
}
85+
}
86+
case other =>
87+
Left(
88+
s"unsupported package '$other'. To fix this problem, use a valid syntax " +
89+
s"such as 'maven:ORGANIZATION:ARTIFACT_NAME_VERSION' for Java libraries."
90+
)
91+
}
92+
}
93+
}
94+
95+
/**
96+
* A Java library that is published "Maven style".
97+
*
98+
* The most widely used Maven package host is "Maven Central"
99+
* https://search.maven.org/. Most companies self-host an Artifactory instance
100+
* to publish internal libraries and to proxy Maven Central.
101+
*/
102+
case class MavenPackage(dep: Dependency)
103+
extends Package(
104+
s"maven:${dep.module.repr}:${dep.version}",
105+
s"maven/${dep.module.organization.value}/${dep.module.name.value}/${dep.version}",
106+
dep.version
107+
) {
108+
def repr = id.stripPrefix("maven:")
109+
}
110+
111+
/**
112+
* The Java standard library.
113+
*
114+
* The sources of the Java standard library are typically available under
115+
* JAVA_HOME.
116+
*/
117+
case class JdkPackage(override val version: String)
118+
extends Package(s"jdk:${version}", s"jdk/${version}", version) {
119+
def repr = id.stripPrefix("jdk:")
120+
}

0 commit comments

Comments
 (0)