AI - MVP - v1

build.gradle

@@ -44,6 +44,7 @@ version = project.findProperty('projVersion') ?: '100.0.0'
 java {
     sourceCompatibility = JavaVersion.VERSION_21
     targetCompatibility = JavaVersion.VERSION_21
+
     // Workaround needed for Eclipse, probably because of https://github.com/gradle/gradle/issues/16922
     // Should be removed as soon as Gradle 7.0.1 is released ( https://github.com/gradle/gradle/issues/16922#issuecomment-828217060 )
     modularity.inferModulePath.set(false)
@@ -71,6 +72,11 @@ application {
 
         // Note that the arguments are cleared for the "run" task to avoid messages like "WARNING: Unknown module: org.jabref.merged.module specified to --add-exports"
 
+        // '--add-exports=langchain4j/dev.langchain4j.model.chat=org.jabref',
+        // '--add-exports=langchain4j/dev.langchain4j.model.chat=org.jabref.merged.module',
+        // '--add-exports=langchain4j/dev.langchain4j.model.openai=org.jabref',
+        // '--add-exports=langchain4j/dev.langchain4j.model.openai=org.jabref.merged.module',
+
         // Fix for https://github.com/JabRef/jabref/issues/11188
         '--add-exports=javafx.base/com.sun.javafx.event=org.jabref.merged.module',
         '--add-exports=javafx.controls/com.sun.javafx.scene.control=org.jabref.merged.module',
@@ -94,6 +100,13 @@ application {
 // See also https://github.com/java9-modularity/gradle-modules-plugin/issues/165
 modularity.disableEffectiveArgumentsAdjustment()
 
+// Required as workaround for  https://github.com/langchain4j/langchain4j/issues/1066
+// modularity.patchModule("langchain4j", "langchain4j-0.28.0.jar")
+modularity.patchModule("langchain4j", "langchain4j-core-0.28.0.jar")
+modularity.patchModule("langchain4j", "langchain4j-embeddings-0.28.0.jar")
+modularity.patchModule("langchain4j", "langchain4j-embeddings-all-minilm-l6-v2-0.28.0.jar")
+modularity.patchModule("langchain4j", "langchain4j-open-ai-0.28.0.jar")
+
 sourceSets {
     main {
         java {
@@ -219,14 +232,19 @@ dependencies {
     implementation 'org.fxmisc.flowless:flowless:0.7.2'
     implementation 'org.fxmisc.richtext:richtextfx:0.11.2'
     implementation (group: 'com.dlsc.gemsfx', name: 'gemsfx', version: '2.12.0') {
-        exclude module: 'javax.inject' // Split package, use only jakarta.inject
         exclude module: 'commons-lang3'
+        exclude module: 'javax.inject' // Split package, use only jakarta.inject
+        exclude module: 'kotlin-stdlib-jdk8'
+        exclude group: 'com.squareup.retrofit2'
         exclude group: 'org.openjfx'
         exclude group: 'org.apache.logging.log4j'
         exclude group: 'tech.units'
     }
     // Required by gemsfx
     implementation 'tech.units:indriya:2.2'
+    implementation ('com.squareup.retrofit2:retrofit:2.11.0') {
+        exclude group: 'com.squareup.okhttp3'
+    }
 
     implementation 'org.controlsfx:controlsfx:11.2.1'
 
@@ -296,6 +314,19 @@ dependencies {
     // YAML formatting
     implementation 'org.yaml:snakeyaml:2.2'
 
+    // AI
+    implementation 'dev.langchain4j:langchain4j-embeddings-all-minilm-l6-v2:0.28.0'
+    implementation('dev.langchain4j:langchain4j-open-ai:0.28.0') {
+        exclude group: 'org.jetbrains.kotlin', module: 'kotlin-stdlib-jdk8'
+    }
+    // openai depends on okhttp, which needs kotlin - see https://github.com/square/okhttp/issues/5299 for details
+    // GemxFX also (transitively) depends on kotlin
+    implementation 'org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.9.24'
+
+    implementation ('com.squareup.okhttp3:okhttp:4.12.0') {
+        exclude group: 'org.jetbrains.kotlin', module: 'kotlin-stdlib-jdk8'
+    }
+
     implementation 'commons-io:commons-io:2.16.1'
 
     testImplementation 'io.github.classgraph:classgraph:4.8.172'
@@ -436,7 +467,18 @@ compileJava {
         // TODO: Remove access to internal api
         addExports = [
                 'javafx.controls/com.sun.javafx.scene.control' : 'org.jabref',
-                'org.controlsfx.controls/impl.org.controlsfx.skin' : 'org.jabref'
+                'org.controlsfx.controls/impl.org.controlsfx.skin' : 'org.jabref',
+
+                'langchain4j/dev.langchain4j.data.document' : 'org.jabref',
+                'langchain4j/dev.langchain4j.data.segment' : 'org.jabref',
+                'langchain4j/dev.langchain4j.model.chat' : 'org.jabref',
+                'langchain4j/dev.langchain4j.model.embedding' : 'org.jabref',
+                'langchain4j/dev.langchain4j.model.openai' : 'org.jabref',
+                'langchain4j/dev.langchain4j.rag.content.retriever' : 'org.jabref',
+                'langchain4j/dev.langchain4j.store.embedding' : 'org.jabref',
+                'langchain4j/dev.langchain4j.memory' : 'org.jabref',
+                'langchain4j/dev.langchain4j.store.memory.chat' : 'org.jabref',
+                'langchain4j/dev.langchain4j.data.message' : 'org.jabref',
         ]
     }
 }
@@ -455,6 +497,9 @@ run {
                 'javafx.base/com.sun.javafx.event' : 'org.jabref.merged.module',
                 'javafx.controls/com.sun.javafx.scene.control' : 'org.jabref',
 
+                'langchain4j/dev.langchain4j.model.chat' : 'org.jabref',
+                'langchain4j/dev.langchain4j.model.chat' : 'org.jabref.merged.module',
+
                 // We need to restate the ControlsFX exports, because we get following error otherwise:
                 //   java.lang.IllegalAccessError:
                 //     class org.controlsfx.control.textfield.AutoCompletionBinding (in module org.controlsfx.controls)

src/main/java/module-info.java

@@ -124,6 +124,10 @@
 
     requires org.jooq.jool;
 
+    // AI
+    requires langchain4j;
+    requires kotlin.stdlib;
+
     // fulltext search
     requires org.apache.lucene.core;
     // In case the version is updated, please also adapt SearchFieldConstants#VERSION to the newly used version

src/main/java/org/jabref/gui/entryeditor/AiChatTab.java

@@ -0,0 +1,238 @@
+package org.jabref.gui.entryeditor;
+
+import java.util.List;
+
+import javafx.geometry.Insets;
+import javafx.geometry.NodeOrientation;
+import javafx.geometry.Pos;
+import javafx.scene.Node;
+import javafx.scene.control.Button;
+import javafx.scene.control.Label;
+import javafx.scene.control.ScrollPane;
+import javafx.scene.control.TextField;
+import javafx.scene.control.Tooltip;
+import javafx.scene.layout.HBox;
+import javafx.scene.layout.Pane;
+import javafx.scene.layout.Priority;
+import javafx.scene.layout.VBox;
+
+import org.jabref.gui.DialogService;
+import org.jabref.logic.ai.AiChat;
+import org.jabref.logic.ai.AiChatData;
+import org.jabref.logic.ai.AiConnection;
+import org.jabref.logic.ai.AiIngestor;
+import org.jabref.logic.l10n.Localization;
+import org.jabref.model.database.BibDatabaseContext;
+import org.jabref.model.entry.BibEntry;
+import org.jabref.model.entry.LinkedFile;
+import org.jabref.preferences.AiPreferences;
+import org.jabref.preferences.FilePreferences;
+import org.jabref.preferences.PreferencesService;
+
+import com.tobiasdiez.easybind.EasyBind;
+import dev.langchain4j.data.message.AiMessage;
+import dev.langchain4j.data.message.ChatMessage;
+import dev.langchain4j.data.message.UserMessage;
+
+public class AiChatTab extends EntryEditorTab {
+    public static final String NAME = "AI chat";
+
+    private final DialogService dialogService;
+
+    private final FilePreferences filePreferences;
+    private final AiPreferences aiPreferences;
+
+    private final BibDatabaseContext bibDatabaseContext;
+
+    private AiConnection aiConnection = null;
+    private AiChat aiChat = null;
+
+    private VBox chatVBox = null;
+
+    public AiChatTab(DialogService dialogService, PreferencesService preferencesService, BibDatabaseContext bibDatabaseContext) {
+        this.dialogService = dialogService;
+
+        this.filePreferences = preferencesService.getFilePreferences();
+        this.aiPreferences = preferencesService.getAiPreferences();
+
+        this.bibDatabaseContext = bibDatabaseContext;
+
+        setText(Localization.lang(NAME));
+        setTooltip(new Tooltip(Localization.lang("AI chat with full-text article")));
+
+        setUpAiConnection();
+    }
+
+    // Set up the AI connection if AI is used.
+    // Also listen for AI preferences changes and update the classes appropriately.
+    private void setUpAiConnection() {
+        if (aiPreferences.isUseAi()) {
+            aiConnection = new AiConnection(aiPreferences.getOpenAiToken());
+        }
+
+        EasyBind.listen(aiPreferences.useAiProperty(), (obs, oldValue, newValue) -> {
+            if (newValue) {
+                aiConnection = new AiConnection(aiPreferences.getOpenAiToken());
+                rebuildAiChat();
+            } else {
+                aiConnection = null;
+                // QUESTION: If user chose AI but then unchooses, what should we do with the AI chat?
+                aiChat = null;
+            }
+        });
+
+        EasyBind.listen(aiPreferences.openAiTokenProperty(), (obs, oldValue, newValue) -> {
+            if (aiConnection != null) {
+                aiConnection = new AiConnection(newValue);
+                rebuildAiChat();
+            }
+        });
+    }
+
+    private void rebuildAiChat() {
+        if (aiChat != null) {
+            AiChatData data = aiChat.getData();
+            aiChat = new AiChat(data, aiConnection);
+        }
+    }
+
+    @Override
+    public boolean shouldShow(BibEntry entry) {
+        return aiPreferences.isUseAi();
+    }
+
+    @Override
+    protected void bindToEntry(BibEntry entry) {
+        Node node;
+
+        if (entry.getFiles().isEmpty()) {
+            node = stateNoFiles();
+        } else if (!entry.getFiles().stream().allMatch(file -> "PDF".equals(file.getFileType()))) {
+            /*
+                QUESTION: What is the type of file.getFileType()????
+                I thought it is the part after the dot, but it turns out not.
+                I got the "PDF" string by looking at tests.
+             */
+            node = stateWrongFilesFormat();
+        } else {
+            configureAiChat(entry);
+            node = stateAiChat();
+            restoreMessages(aiChat.getData().getChatMemoryStore().getMessages(aiChat.getChatId()));
+        }
+
+        setContent(node);
+    }
+
+    private Node stateNoFiles() {
+        return new Label(Localization.lang("No files attached"));
+    }
+
+    private Node stateWrongFilesFormat() {
+        return new Label(Localization.lang("Only PDF files are supported"));
+    }
+
+    private Node stateAiChat() {
+        // Don't bully me for this style.
+
+        VBox aiChatBox = new VBox(10);
+        aiChatBox.setPadding(new Insets(10));
+
+            ScrollPane chatScrollPane = new ScrollPane();
+            chatScrollPane.setStyle("-fx-border-color: black;");
+            chatScrollPane.setPadding(new Insets(10, 10, 0, 10));
+            VBox.setVgrow(chatScrollPane, Priority.ALWAYS);
+
+                chatVBox = new VBox(10);
+
+                    // Chat messages will be children of chatVBox.
+
+                chatScrollPane.setContent(chatVBox);
+
+            aiChatBox.getChildren().add(chatScrollPane);
+
+            HBox userPromptHBox = new HBox(10);
+            userPromptHBox.setAlignment(Pos.CENTER);
+
+                TextField userPromptTextField = new TextField();
+                HBox.setHgrow(userPromptTextField, Priority.ALWAYS);
+
+                userPromptHBox.getChildren().add(userPromptTextField);
+
+                Button userPromptSubmitButton = new Button(Localization.lang("Submit"));
+                userPromptSubmitButton.setOnAction(e -> {
+                    String userPrompt = userPromptTextField.getText();
+                    userPromptTextField.setText("");
+
+                    addMessage(true, userPrompt);
+
+                    String aiMessage = aiChat.execute(userPrompt);
+
+                    addMessage(false, aiMessage);
+                });
+
+                userPromptHBox.getChildren().add(userPromptSubmitButton);
+
+            aiChatBox.getChildren().add(userPromptHBox);
+
+        return aiChatBox;
+    }
+
+    private void addMessage(boolean isUser, String text) {
+        Node messageNode = generateMessage(isUser, text);
+        chatVBox.getChildren().add(messageNode);
+    }
+
+    private static final String USER_MESSAGE_COLOR = "#7ee3fb";
+    private static final String AI_MESSAGE_COLOR = "#bac8cb";
+
+    private static Node generateMessage(boolean isUser, String text) {
+        Pane pane = new Pane();
+
+        if (isUser) {
+            pane.setNodeOrientation(NodeOrientation.RIGHT_TO_LEFT);
+        }
+
+        VBox paneVBox = new VBox(10);
+
+        paneVBox.setStyle("-fx-background-color: " + (isUser ? USER_MESSAGE_COLOR : AI_MESSAGE_COLOR) + ";");
+        paneVBox.setPadding(new Insets(10));
+
+        Label authorLabel = new Label(Localization.lang(isUser ? "User" : "AI"));
+        authorLabel.setStyle("-fx-font-weight: bold");
+        paneVBox.getChildren().add(authorLabel);
+
+        Label messageLabel = new Label(text);
+        paneVBox.getChildren().add(messageLabel);
+
+        pane.getChildren().add(paneVBox);
+
+        return pane;
+    }
+
+    private void configureAiChat(BibEntry entry) {
+        aiChat = new AiChat(aiConnection);
+
+        AiIngestor ingestor = new AiIngestor(aiChat.getData().getEmbeddingStore(), aiConnection.getEmbeddingModel());
+
+        for (LinkedFile linkedFile : entry.getFiles()) {
+            try {
+                ingestor.ingestLinkedFile(linkedFile, bibDatabaseContext, filePreferences);
+            } catch (Exception e) {
+                dialogService.showErrorDialogAndWait(Localization.lang("Error while loading file"),
+                        Localization.lang("An error occurred while loading a file into the AI") + ":\n"
+                                + e.getMessage() + "\n"
+                                + Localization.lang("This file will be skipped") + ".");
+            }
+        }
+    }
+
+    private void restoreMessages(List<ChatMessage> messages) {
+        for (ChatMessage message : messages) {
+            if (message instanceof UserMessage userMessage) {
+                addMessage(true, userMessage.singleText());
+            } else if (message instanceof AiMessage aiMessage) {
+                addMessage(false, aiMessage.text());
+            }
+        }
+    }
+}

src/main/java/org/jabref/gui/entryeditor/EntryEditor.java

@@ -314,6 +314,7 @@ private List<EntryEditorTab> createTabs() {
         entryEditorTabs.add(sourceTab);
         entryEditorTabs.add(new LatexCitationsTab(databaseContext, preferencesService, dialogService, directoryMonitorManager));
         entryEditorTabs.add(new FulltextSearchResultsTab(stateManager, preferencesService, dialogService, taskExecutor));
+        entryEditorTabs.add(new AiChatTab(dialogService, preferencesService, databaseContext));
 
         return entryEditorTabs;
     }

src/main/java/org/jabref/gui/preferences/PreferencesDialogViewModel.java

@@ -13,6 +13,7 @@
 import org.jabref.gui.AbstractViewModel;
 import org.jabref.gui.DialogService;
 import org.jabref.gui.Globals;
+import org.jabref.gui.preferences.ai.AiTab;
 import org.jabref.gui.preferences.autocompletion.AutoCompletionTab;
 import org.jabref.gui.preferences.citationkeypattern.CitationKeyPatternTab;
 import org.jabref.gui.preferences.customentrytypes.CustomEntryTypesTab;
@@ -81,7 +82,8 @@ public PreferencesDialogViewModel(DialogService dialogService, PreferencesServic
                 new XmpPrivacyTab(),
                 new CustomImporterTab(),
                 new CustomExporterTab(),
-                new NetworkTab()
+                new NetworkTab(),
+                new AiTab()
         );
     }
 

src/main/java/org/jabref/gui/preferences/ai/AiTab.fxml

@@ -0,0 +1,16 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<?import javafx.scene.control.*?>
+<?import javafx.scene.layout.*?>
+
+<fx:root spacing="10.0" type="VBox" xmlns="http://javafx.com/javafx/17.0.2-ea" xmlns:fx="http://javafx.com/fxml/1" fx:controller="org.jabref.gui.preferences.ai.AiTab">
+   <children>
+      <CheckBox fx:id="useAi" mnemonicParsing="false" text="%Use modern AI technologies in JabRef" />
+      <HBox alignment="CENTER_LEFT" spacing="10.0">
+         <children>
+            <Label alignment="BASELINE_CENTER" text="%OpenAI API token" />
+            <TextField fx:id="openAiToken" HBox.hgrow="ALWAYS" />
+         </children>
+      </HBox>
+   </children>
+</fx:root>