diff --git a/build.gradle b/build.gradle index 6466119ed31..a952f3676f1 100644 --- a/build.gradle +++ b/build.gradle @@ -44,6 +44,7 @@ version = project.findProperty('projVersion') ?: '100.0.0' java { sourceCompatibility = JavaVersion.VERSION_21 targetCompatibility = JavaVersion.VERSION_21 + // Workaround needed for Eclipse, probably because of https://github.com/gradle/gradle/issues/16922 // Should be removed as soon as Gradle 7.0.1 is released ( https://github.com/gradle/gradle/issues/16922#issuecomment-828217060 ) modularity.inferModulePath.set(false) @@ -94,6 +95,13 @@ application { // See also https://github.com/java9-modularity/gradle-modules-plugin/issues/165 modularity.disableEffectiveArgumentsAdjustment() +// Required as workaround for https://github.com/langchain4j/langchain4j/issues/1066 +// modularity.patchModule("langchain4j", "langchain4j-0.28.0.jar") +modularity.patchModule("langchain4j", "langchain4j-core-0.28.0.jar") +modularity.patchModule("langchain4j", "langchain4j-embeddings-0.28.0.jar") +modularity.patchModule("langchain4j", "langchain4j-embeddings-all-minilm-l6-v2-0.28.0.jar") +modularity.patchModule("langchain4j", "langchain4j-open-ai-0.28.0.jar") + sourceSets { main { java { @@ -219,14 +227,19 @@ dependencies { implementation 'org.fxmisc.flowless:flowless:0.7.2' implementation 'org.fxmisc.richtext:richtextfx:0.11.2' implementation (group: 'com.dlsc.gemsfx', name: 'gemsfx', version: '2.12.0') { - exclude module: 'javax.inject' // Split package, use only jakarta.inject exclude module: 'commons-lang3' + exclude module: 'javax.inject' // Split package, use only jakarta.inject + exclude module: 'kotlin-stdlib-jdk8' + exclude group: 'com.squareup.retrofit2' exclude group: 'org.openjfx' exclude group: 'org.apache.logging.log4j' exclude group: 'tech.units' } // Required by gemsfx implementation 'tech.units:indriya:2.2' + implementation ('com.squareup.retrofit2:retrofit:2.11.0') { + exclude group: 'com.squareup.okhttp3' + } implementation 'org.controlsfx:controlsfx:11.2.1' @@ -296,6 +309,19 @@ dependencies { // YAML formatting implementation 'org.yaml:snakeyaml:2.2' + // AI + implementation 'dev.langchain4j:langchain4j-embeddings-all-minilm-l6-v2:0.28.0' + implementation('dev.langchain4j:langchain4j-open-ai:0.28.0') { + exclude group: 'org.jetbrains.kotlin', module: 'kotlin-stdlib-jdk8' + } + // openai depends on okhttp, which needs kotlin - see https://github.com/square/okhttp/issues/5299 for details + // GemxFX also (transitively) depends on kotlin + implementation 'org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.9.24' + + implementation ('com.squareup.okhttp3:okhttp:4.12.0') { + exclude group: 'org.jetbrains.kotlin', module: 'kotlin-stdlib-jdk8' + } + implementation 'commons-io:commons-io:2.16.1' testImplementation 'io.github.classgraph:classgraph:4.8.172' @@ -436,7 +462,19 @@ compileJava { // TODO: Remove access to internal api addExports = [ 'javafx.controls/com.sun.javafx.scene.control' : 'org.jabref', - 'org.controlsfx.controls/impl.org.controlsfx.skin' : 'org.jabref' + 'org.controlsfx.controls/impl.org.controlsfx.skin' : 'org.jabref', + + 'langchain4j/dev.langchain4j.data.document' : 'org.jabref', + 'langchain4j/dev.langchain4j.data.segment' : 'org.jabref', + 'langchain4j/dev.langchain4j.model.chat' : 'org.jabref', + 'langchain4j/dev.langchain4j.model.embedding' : 'org.jabref', + 'langchain4j/dev.langchain4j.model.openai' : 'org.jabref', + 'langchain4j/dev.langchain4j.rag.content.retriever' : 'org.jabref', + 'langchain4j/dev.langchain4j.store.embedding' : 'org.jabref', + 'langchain4j/dev.langchain4j.memory' : 'org.jabref', + 'langchain4j/dev.langchain4j.store.memory.chat' : 'org.jabref', + 'langchain4j/dev.langchain4j.data.message' : 'org.jabref', + 'langchain4j/dev.langchain4j.store.embedding.filter' : 'org.jabref', ] } } @@ -455,6 +493,9 @@ run { 'javafx.base/com.sun.javafx.event' : 'org.jabref.merged.module', 'javafx.controls/com.sun.javafx.scene.control' : 'org.jabref', + 'langchain4j/dev.langchain4j.model.chat' : 'org.jabref', + 'langchain4j/dev.langchain4j.model.chat' : 'org.jabref.merged.module', + // We need to restate the ControlsFX exports, because we get following error otherwise: // java.lang.IllegalAccessError: // class org.controlsfx.control.textfield.AutoCompletionBinding (in module org.controlsfx.controls) diff --git a/src/main/java/module-info.java b/src/main/java/module-info.java index 352ef69c95b..5a0eba3059f 100644 --- a/src/main/java/module-info.java +++ b/src/main/java/module-info.java @@ -124,6 +124,10 @@ requires org.jooq.jool; + // AI + requires langchain4j; + requires kotlin.stdlib; + // fulltext search requires org.apache.lucene.core; // In case the version is updated, please also adapt SearchFieldConstants#VERSION to the newly used version diff --git a/src/main/java/org/jabref/gui/Base.css b/src/main/java/org/jabref/gui/Base.css index c0fd90990ce..fa0a90f5a53 100644 --- a/src/main/java/org/jabref/gui/Base.css +++ b/src/main/java/org/jabref/gui/Base.css @@ -252,6 +252,10 @@ /* Consistent size for headers of tab-pane and side-panels*/ -jr-header-height: 3em; + + /* AI chat style */ + -jr-ai-message-user: #7ee3fb; + -jr-ai-message-ai: #bac8cb; } .unchanged { diff --git a/src/main/java/org/jabref/gui/entryeditor/EntryEditor.java b/src/main/java/org/jabref/gui/entryeditor/EntryEditor.java index 759c46fc0ad..9a0c728834c 100644 --- a/src/main/java/org/jabref/gui/entryeditor/EntryEditor.java +++ b/src/main/java/org/jabref/gui/entryeditor/EntryEditor.java @@ -31,6 +31,7 @@ import org.jabref.gui.StateManager; import org.jabref.gui.citationkeypattern.GenerateCitationKeySingleAction; import org.jabref.gui.cleanup.CleanupSingleAction; +import org.jabref.gui.entryeditor.aichattab.AiChatTab; import org.jabref.gui.entryeditor.citationrelationtab.CitationRelationsTab; import org.jabref.gui.entryeditor.fileannotationtab.FileAnnotationTab; import org.jabref.gui.entryeditor.fileannotationtab.FulltextSearchResultsTab; @@ -45,6 +46,7 @@ import org.jabref.gui.undo.CountingUndoManager; import org.jabref.gui.util.DefaultTaskExecutor; import org.jabref.gui.util.TaskExecutor; +import org.jabref.logic.ai.AiService; import org.jabref.logic.bibtex.TypedBibEntry; import org.jabref.logic.help.HelpFile; import org.jabref.logic.importer.EntryBasedFetcher; @@ -87,6 +89,9 @@ public class EntryEditor extends BorderPane { private final ExternalFilesEntryLinker fileLinker; private final DirectoryMonitorManager directoryMonitorManager; + // TODO: Move this out. + private final AiService aiService; + private Subscription typeSubscription; /* @@ -131,6 +136,8 @@ public EntryEditor(LibraryTab libraryTab) { .root(this) .load(); + this.aiService = new AiService(preferencesService.getAiPreferences()); + this.entryEditorPreferences = preferencesService.getEntryEditorPreferences(); this.fileLinker = new ExternalFilesEntryLinker(preferencesService.getFilePreferences(), databaseContext, dialogService); @@ -314,6 +321,7 @@ private List createTabs() { entryEditorTabs.add(sourceTab); entryEditorTabs.add(new LatexCitationsTab(databaseContext, preferencesService, dialogService, directoryMonitorManager)); entryEditorTabs.add(new FulltextSearchResultsTab(stateManager, preferencesService, dialogService, taskExecutor)); + entryEditorTabs.add(new AiChatTab(preferencesService, aiService, databaseContext, taskExecutor)); return entryEditorTabs; } diff --git a/src/main/java/org/jabref/gui/entryeditor/EntryEditorPreferences.java b/src/main/java/org/jabref/gui/entryeditor/EntryEditorPreferences.java index ae0fda0194c..7a0cf3efb1d 100644 --- a/src/main/java/org/jabref/gui/entryeditor/EntryEditorPreferences.java +++ b/src/main/java/org/jabref/gui/entryeditor/EntryEditorPreferences.java @@ -40,6 +40,7 @@ public static JournalPopupEnabled fromString(String status) { private final MapProperty> defaultEntryEditorTabList; private final BooleanProperty shouldOpenOnNewEntry; private final BooleanProperty shouldShowRecommendationsTab; + private final BooleanProperty shouldShowAiChatTab; private final BooleanProperty shouldShowLatexCitationsTab; private final BooleanProperty showSourceTabByDefault; private final BooleanProperty enableValidation; @@ -54,6 +55,7 @@ public EntryEditorPreferences(Map> entryEditorTabList, Map> defaultEntryEditorTabList, boolean shouldOpenOnNewEntry, boolean shouldShowRecommendationsTab, + boolean shouldShowAiChatTab, boolean shouldShowLatexCitationsTab, boolean showSourceTabByDefault, boolean enableValidation, @@ -68,6 +70,7 @@ public EntryEditorPreferences(Map> entryEditorTabList, this.defaultEntryEditorTabList = new SimpleMapProperty<>(FXCollections.observableMap(defaultEntryEditorTabList)); this.shouldOpenOnNewEntry = new SimpleBooleanProperty(shouldOpenOnNewEntry); this.shouldShowRecommendationsTab = new SimpleBooleanProperty(shouldShowRecommendationsTab); + this.shouldShowAiChatTab = new SimpleBooleanProperty(shouldShowAiChatTab); this.shouldShowLatexCitationsTab = new SimpleBooleanProperty(shouldShowLatexCitationsTab); this.showSourceTabByDefault = new SimpleBooleanProperty(showSourceTabByDefault); this.enableValidation = new SimpleBooleanProperty(enableValidation); @@ -119,6 +122,18 @@ public void setShouldShowRecommendationsTab(boolean shouldShowRecommendationsTab this.shouldShowRecommendationsTab.set(shouldShowRecommendationsTab); } + public boolean shouldShowAiChatTab() { + return shouldShowAiChatTab.get(); + } + + public BooleanProperty shouldShowAiChatTabProperty() { + return shouldShowAiChatTab; + } + + public void setShouldShowAiChatTab(boolean shouldShowAiChatTab) { + this.shouldShowAiChatTab.set(shouldShowAiChatTab); + } + public boolean shouldShowLatexCitationsTab() { return shouldShowLatexCitationsTab.get(); } diff --git a/src/main/java/org/jabref/gui/entryeditor/aichattab/AiChatTab.java b/src/main/java/org/jabref/gui/entryeditor/aichattab/AiChatTab.java new file mode 100644 index 00000000000..3674e6008ac --- /dev/null +++ b/src/main/java/org/jabref/gui/entryeditor/aichattab/AiChatTab.java @@ -0,0 +1,156 @@ +package org.jabref.gui.entryeditor.aichattab; + +import java.nio.file.Path; +import java.util.Optional; + +import javafx.scene.control.*; + +import org.jabref.gui.entryeditor.EntryEditorPreferences; +import org.jabref.gui.entryeditor.EntryEditorTab; +import org.jabref.gui.entryeditor.aichattab.components.AiChatComponentOld; +import org.jabref.gui.entryeditor.aichattab.components.aichat.AiChatComponent; +import org.jabref.gui.util.BackgroundTask; +import org.jabref.gui.util.TaskExecutor; +import org.jabref.logic.ai.AiChat; +import org.jabref.logic.ai.AiService; +import org.jabref.logic.ai.AiIngestor; +import org.jabref.logic.ai.ChatMessage; +import org.jabref.logic.l10n.Localization; +import org.jabref.logic.util.io.FileUtil; +import org.jabref.model.database.BibDatabaseContext; +import org.jabref.model.entry.BibEntry; +import org.jabref.model.entry.LinkedFile; +import org.jabref.preferences.AiPreferences; +import org.jabref.preferences.FilePreferences; +import org.jabref.preferences.PreferencesService; + +import com.tobiasdiez.easybind.EasyBind; +import dev.langchain4j.data.segment.TextSegment; +import dev.langchain4j.store.embedding.EmbeddingStore; +import dev.langchain4j.store.embedding.filter.Filter; +import dev.langchain4j.store.embedding.filter.MetadataFilterBuilder; +import dev.langchain4j.store.embedding.inmemory.InMemoryEmbeddingStore; +import org.slf4j.LoggerFactory; + +public class AiChatTab extends EntryEditorTab { + public static final String NAME = "AI chat"; + + private static final org.slf4j.Logger LOGGER = LoggerFactory.getLogger(AiChatTab.class.getName()); + + private static final String QA_SYSTEM_MESSAGE = """ + You are an AI research assistant. You read and analyze scientific articles. + The user will send you a question regarding a paper. You will be supplied also with the relevant information found in the article. + Answer the question only by using the relevant information. Don't make up the answer. + If you can't answer the user question using the provided information, then reply that you couldn't do it."""; + + private final FilePreferences filePreferences; + private final AiPreferences aiPreferences; + private final EntryEditorPreferences entryEditorPreferences; + private final BibDatabaseContext bibDatabaseContext; + private final TaskExecutor taskExecutor; + + private AiChatComponent aiChatComponent = null; + + private final AiService aiService; + + private AiChat aiChat = null; + + private BibEntry currentBibEntry = null; + + public AiChatTab(PreferencesService preferencesService, AiService aiService, + BibDatabaseContext bibDatabaseContext, TaskExecutor taskExecutor) { + this.filePreferences = preferencesService.getFilePreferences(); + this.aiPreferences = preferencesService.getAiPreferences(); + this.entryEditorPreferences = preferencesService.getEntryEditorPreferences(); + + this.aiService = aiService; + + this.bibDatabaseContext = bibDatabaseContext; + + this.taskExecutor = taskExecutor; + + setText(Localization.lang(NAME)); + setTooltip(new Tooltip(Localization.lang("AI chat with full-text article"))); + } + + @Override + public boolean shouldShow(BibEntry entry) { + return entryEditorPreferences.shouldShowAiChatTab(); + } + + @Override + protected void bindToEntry(BibEntry entry) { + if (!aiPreferences.getEnableChatWithFiles()) { + setContent(new Label(Localization.lang("JabRef uses OpenAI to enable \"chatting\" with PDF files. OpenAI is an external service. To enable JabRef chatgting with PDF files, the content of the PDF files need to be shared with OpenAI. As soon as you ask a question, the text content of all PDFs attached to the entry are send to OpenAI. The privacy policy of OpenAI applies. You find it at ."))); + } else if (entry.getCitationKey().isEmpty()) { + setContent(new Label(Localization.lang("Please provide a citation key for the entry in order to enable chatting with PDF files."))); + } else if (!checkIfCitationKeyIsUnique(bibDatabaseContext, entry.getCitationKey().get())) { + setContent(new Label(Localization.lang("Please provide a unique citation key for the entry in order to enable chatting with PDF files."))); + } else if (entry.getFiles().isEmpty()) { + setContent(new Label(Localization.lang("No files attached"))); + } else if (!entry.getFiles().stream().map(LinkedFile::getLink).map(Path::of).allMatch(FileUtil::isPDFFile)) { + setContent(new Label(Localization.lang("Only PDF files are supported"))); + } else { + bindToCorrectEntry(entry); + } + } + + private static boolean checkIfCitationKeyIsUnique(BibDatabaseContext bibDatabaseContext, String citationKey) { + return bibDatabaseContext.getDatabase().getEntries().stream() + .map(BibEntry::getCitationKey) + .filter(Optional::isPresent) + .map(Optional::get) + .filter(key -> key.equals(citationKey)) + .count() == 1; + } + + private void bindToCorrectEntry(BibEntry entry) { + currentBibEntry = entry; + + createAiChat(); + aiChat.restoreMessages(entry.getAiChatMessages()); + ingestFiles(entry); + buildChatUI(entry); + } + + private void createAiChat() { + aiChat = new AiChat(aiService, MetadataFilterBuilder.metadataKey("linkedFile").isIn(currentBibEntry.getFiles().stream().map(LinkedFile::getLink).toList())); + aiChat.setSystemMessage(QA_SYSTEM_MESSAGE); + } + + private void ingestFiles(BibEntry entry) { + AiIngestor aiIngestor = new AiIngestor(aiService.getEmbeddingStore(), aiService.getEmbeddingModel()); + entry.getFiles().forEach(file -> { + aiIngestor.ingestLinkedFile(file, bibDatabaseContext, filePreferences); + }); + } + + private void buildChatUI(BibEntry entry) { + aiChatComponent = new AiChatComponent((userPrompt) -> { + ChatMessage userMessage = ChatMessage.user(userPrompt); + aiChatComponent.addMessage(userMessage); + entry.getAiChatMessages().add(userMessage); + aiChatComponent.setLoading(true); + + BackgroundTask.wrap(() -> aiChat.execute(userPrompt)) + .onSuccess(aiMessageText -> { + aiChatComponent.setLoading(false); + + ChatMessage aiMessage = ChatMessage.assistant(aiMessageText); + aiChatComponent.addMessage(aiMessage); + entry.getAiChatMessages().add(aiMessage); + }) + .onFailure(e -> { + // TODO: User-friendly error message. + LOGGER.error("Got an error while sending a message to AI", e); + aiChatComponent.setLoading(false); + aiChatComponent.addError(e.getMessage()); + }) + .executeWith(taskExecutor); + }); + + entry.getAiChatMessages().forEach(aiChatComponent::addMessage); + + setContent(aiChatComponent); + } +} diff --git a/src/main/java/org/jabref/gui/entryeditor/aichattab/components/AiChatComponentOld.java b/src/main/java/org/jabref/gui/entryeditor/aichattab/components/AiChatComponentOld.java new file mode 100644 index 00000000000..2a1146dff89 --- /dev/null +++ b/src/main/java/org/jabref/gui/entryeditor/aichattab/components/AiChatComponentOld.java @@ -0,0 +1,89 @@ +package org.jabref.gui.entryeditor.aichattab.components; + +import javafx.geometry.Insets; +import javafx.geometry.Pos; +import javafx.scene.Node; +import javafx.scene.control.Button; +import javafx.scene.control.ScrollPane; +import javafx.scene.control.TextField; +import javafx.scene.layout.HBox; +import javafx.scene.layout.Priority; +import javafx.scene.layout.VBox; + +import org.jabref.gui.entryeditor.aichattab.components.chatmessage.ChatMessageComponent; +import org.jabref.logic.ai.ChatMessage; +import org.jabref.logic.l10n.Localization; + +import java.util.List; +import java.util.function.Consumer; + +public class AiChatComponentOld { + private final Consumer sendMessageCallback; + + private final VBox aiChatBox = new VBox(10); + private final VBox chatVBox = new VBox(10); + private final TextField userPromptTextField = new TextField(); + + public AiChatComponentOld(Consumer sendMessageCallback) { + this.sendMessageCallback = sendMessageCallback; + + buildUI(); + } + + private void buildUI() { + aiChatBox.setPadding(new Insets(10)); + + aiChatBox.getChildren().add(constructChatScrollPane()); + aiChatBox.getChildren().add(constructUserPromptBox()); + } + + private Node constructChatScrollPane() { + ScrollPane chatScrollPane = new ScrollPane(); + chatScrollPane.setFitToWidth(true); + chatScrollPane.setStyle("-fx-border-color: black;"); + VBox.setVgrow(chatScrollPane, Priority.ALWAYS); + + chatVBox.setPadding(new Insets(10)); + + chatScrollPane.setContent(chatVBox); + + chatScrollPane.vvalueProperty().bind(chatVBox.heightProperty()); + + return chatScrollPane; + } + + private Node constructUserPromptBox() { + HBox userPromptHBox = new HBox(10); + userPromptHBox.setAlignment(Pos.CENTER); + + HBox.setHgrow(userPromptTextField, Priority.ALWAYS); + userPromptTextField.setOnAction(e -> internalSendMessageEvent()); + + userPromptHBox.getChildren().add(userPromptTextField); + + Button userPromptSubmitButton = new Button(Localization.lang("Submit")); + userPromptSubmitButton.setOnAction(e -> internalSendMessageEvent()); + + userPromptHBox.getChildren().add(userPromptSubmitButton); + + return userPromptHBox; + } + + public void addMessage(ChatMessage chatMessage) { + chatVBox.getChildren().add(new ChatMessageComponent().withChatMessage(chatMessage)); + } + + private void internalSendMessageEvent() { + String userPrompt = userPromptTextField.getText(); + userPromptTextField.clear(); + sendMessageCallback.accept(userPrompt); + } + + public void restoreMessages(List messages) { + messages.forEach(this::addMessage); + } + + public Node getNode() { + return aiChatBox; + } +} diff --git a/src/main/java/org/jabref/gui/entryeditor/aichattab/components/aichat/AiChatComponent.fxml b/src/main/java/org/jabref/gui/entryeditor/aichattab/components/aichat/AiChatComponent.fxml new file mode 100644 index 00000000000..69f90d6ed94 --- /dev/null +++ b/src/main/java/org/jabref/gui/entryeditor/aichattab/components/aichat/AiChatComponent.fxml @@ -0,0 +1,30 @@ + + + + + + + + + + + + + + + + + + + + + + + + + +