Review prompt improvements

cmd/review/main.go

@@ -2,10 +2,10 @@ package main
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"os"
 	"os/signal"
-	"strings"
 	"syscall"
 
 	"github.com/google/go-github/v51/github"
@@ -55,69 +55,88 @@ func run(ctx context.Context) error {
 		return fmt.Errorf("error getting commits: %w", err)
 	}
 
-	var OverallReviewCompletion string
-	for _, file := range diff.Files {
-		if file.Patch == nil || file.GetStatus() == "removed" || file.GetStatus() == "renamed" {
+	var comments []*github.PullRequestComment
+
+	for i, file := range diff.Files {
+		patch := file.GetPatch()
+		fmt.Printf("processing file: %s %d/%d\n", file.GetFilename(), i+1, len(diff.Files))
+		if patch == "" || file.GetStatus() == "removed" || file.GetStatus() == "renamed" {
 			continue
 		}
 
-		prompt := fmt.Sprintf(oAIClient.PromptReview, *file.Patch)
-
-		if len(prompt) > 4096 {
-			prompt = fmt.Sprintf("%s...", prompt[:4093])
+		if len(patch) > 4096 {
+			fmt.Println("Patch is too long, truncating")
+			patch = fmt.Sprintf("%s...", patch[:4093])
 		}
-
 		completion, err := openAIClient.ChatCompletion(ctx, []openai.ChatCompletionMessage{
 			{
 				Role:    openai.ChatMessageRoleUser,
-				Content: prompt,
+				Content: oAIClient.PromptReview,
+			},
+			{
+				Role:    openai.ChatMessageRoleUser,
+				Content: patch,
 			},
 		})
+
 		if err != nil {
-			return fmt.Errorf("error getting review: %w", err)
+			return fmt.Errorf("error getting completion: %w", err)
 		}
-		OverallReviewCompletion += fmt.Sprintf("File: %s \nReview: %s \n\n", file.GetFilename(), completion)
 
-		position := len(strings.Split(*file.Patch, "\n")) - 1
+		if opts.Test {
+			fmt.Println("Completion:", completion)
+		}
 
-		comment := &github.PullRequestComment{
-			CommitID: diff.Commits[len(diff.Commits)-1].SHA,
-			Path:     file.Filename,
-			Body:     &completion,
-			Position: &position,
+		review := Review{}
+		err = json.Unmarshal([]byte(completion), &review)
+		if err != nil {
+			fmt.Println("Error unmarshalling completion:", err)
+			continue
 		}
 
-		if opts.Test {
+		if review.Quality == Good {
+			fmt.Println("Review is good")
 			continue
 		}
+		for _, issue := range review.Issues {
+			body := fmt.Sprintf("[%s] %s", issue.Type, issue.Description)
+			comment := &github.PullRequestComment{
+				CommitID: diff.Commits[len(diff.Commits)-1].SHA,
+				Path:     file.Filename,
+				Body:     &body,
+				Position: &issue.Line,
+			}
+			comments = append(comments, comment)
+		}
 
-		if _, err := githubClient.CreatePullRequestComment(ctx, opts.Owner, opts.Repo, opts.PRNumber, comment); err != nil {
-			return fmt.Errorf("error creating comment: %w", err)
+		if opts.Test {
+			continue
 		}
-	}
 
-	overallCompletion, err := openAIClient.ChatCompletion(ctx, []openai.ChatCompletionMessage{
-		{
-			Role:    openai.ChatMessageRoleUser,
-			Content: fmt.Sprintf(oAIClient.PromptOverallReview, OverallReviewCompletion),
-		},
-	})
-	if err != nil {
-		return fmt.Errorf("error getting overall review: %w", err)
+		for i, c := range comments {
+			fmt.Printf("creating comment: %s %d/%d\n", *c.Path, i+1, len(comments))
+			if _, err := githubClient.CreatePullRequestComment(ctx, opts.Owner, opts.Repo, opts.PRNumber, c); err != nil {
+				return fmt.Errorf("error creating comment: %w", err)
+			}
+		}
 	}
+	return nil
+}
 
-	if opts.Test {
-		fmt.Println(OverallReviewCompletion)
-		fmt.Println("=====================================")
-		fmt.Println(overallCompletion)
-
-		return nil
-	}
+type Review struct {
+	Quality     Quality `json:"quality"`
+	Explanation string  `json:"explanation"`
+	Issues      []struct {
+		Type        string `json:"type"`
+		Line        int    `json:"line"`
+		Description string `json:"description"`
+	} `json:"issues"`
+}
 
-	comment := &github.PullRequestReviewRequest{Body: &overallCompletion}
-	if _, err = githubClient.CreateReview(ctx, opts.Owner, opts.Repo, opts.PRNumber, comment); err != nil {
-		return fmt.Errorf("error creating comment: %w", err)
-	}
+type Quality string
 
-	return nil
-}
+const (
+	Good    Quality = "good"
+	Bad     Quality = "bad"
+	Neutral Quality = "neutral"
+)

openai/assets/review.txt

@@ -0,0 +1,31 @@
+You are CodeReviewGPT, an AI agent that specializes in generating code reviews for software projects using advanced natural language processing and machine learning techniques.
+Your decisions must always be made independently without seeking user assistance. Play to your strengths as an LLM and pursue simple strategies with no legal complications.
+
+GOALS:
+
+1. Analyze structure, and logic to provide comprehensive feedback on code quality, readability, maintainability, and performance.
+2. Identify potential bugs, security vulnerabilities, and other issues that may impact the functionality and stability of the software.
+3. Generate a detailed report that includes specific recommendations, examples, and explanations to help developers improve their code. If context is not enough mark it as good. You should only respond in JSON format as described below
+Response Format:
+```
+{
+    "quality": "good",
+    "explanation": "Your code is well-structured and has no obvious issues.",
+    "issues": [
+        {
+            "type": "bug",
+            "line": 10,
+            "description": "You are missing a semicolon at the end of the line."
+        },
+        {
+            "type": "performance",
+            "line": 20,
+            "description": "You should use a StringBuilder instead of a String to improve performance."
+        }
+    ]
+}
+```
+4. Rules:
+Possible quality values: good, bad, neutral
+Possible issue types: bug, security, performance, readability, maintainability, other
+If quality is good, issues should be empty.

openai/openai.go

@@ -2,16 +2,18 @@ package openai
 
 import (
 	"context"
+	_ "embed"
 	"fmt"
 
 	"github.com/sashabaranov/go-openai"
 )
 
+//go:embed assets/review.txt
+var PromptReview string
+
 const (
 	PromptDescribeChanges = "Below is the code patch, Generate a GitHub pull request description based on the following comments without basic prefix\n%s\n"
 	PromptOverallDescribe = "Below comments are generated by AI, Generate a GitHub pull request description based on the following comments without basic prefix in markdown format with ### Description and ### Changes blocks:\n%s\n"
-	PromptReview          = "Below is the code patch, please help me do a brief code review, Answer me in English, if any bug risk and improvement suggestion are welcome\n%s\n"
-	PromptOverallReview   = "Below comments are generated by AI, please help me do a brief code review, Answer me in English, if any bug risk and improvement suggestion are welcome\n%s\n"
 )
 
 type Client struct {
@@ -28,8 +30,9 @@ func (o *Client) ChatCompletion(ctx context.Context, messages []openai.ChatCompl
 	resp, err := o.client.CreateChatCompletion(
 		ctx,
 		openai.ChatCompletionRequest{
-			Model:    openai.GPT3Dot5Turbo,
-			Messages: messages,
+			Model:       openai.GPT3Dot5Turbo,
+			Messages:    messages,
+			Temperature: 0.1,
 		},
 	)