Review prompt improvements

cmd/review/main.go

@@ -2,10 +2,11 @@ package main
 
 import (
 	"context"
+	"encoding/json"
+	"errors"
 	"fmt"
 	"os"
 	"os/signal"
-	"strings"
 	"syscall"
 
 	"github.com/google/go-github/v51/github"
@@ -55,69 +56,122 @@ func run(ctx context.Context) error {
 		return fmt.Errorf("error getting commits: %w", err)
 	}
 
-	var OverallReviewCompletion string
-	for _, file := range diff.Files {
-		if file.Patch == nil || file.GetStatus() == "removed" || file.GetStatus() == "renamed" {
+	var comments []*github.PullRequestComment
+
+	for i, file := range diff.Files {
+		patch := file.GetPatch()
+		fmt.Printf("processing file: %s %d/%d\n", file.GetFilename(), i+1, len(diff.Files))
+		if patch == "" || file.GetStatus() == "removed" || file.GetStatus() == "renamed" {
 			continue
 		}
 
-		prompt := fmt.Sprintf(oAIClient.PromptReview, *file.Patch)
-
-		if len(prompt) > 4096 {
-			prompt = fmt.Sprintf("%s...", prompt[:4093])
+		if len(patch) > 3000 {
+			fmt.Println("Patch is too long, truncating")
+			patch = fmt.Sprintf("%s...", patch[:3000])
 		}
-
 		completion, err := openAIClient.ChatCompletion(ctx, []openai.ChatCompletionMessage{
 			{
 				Role:    openai.ChatMessageRoleUser,
-				Content: prompt,
+				Content: oAIClient.PromptReview,
+			},
+			{
+				Role:    openai.ChatMessageRoleUser,
+				Content: patch,
 			},
 		})
+
 		if err != nil {
-			return fmt.Errorf("error getting review: %w", err)
+			return fmt.Errorf("error getting completion: %w", err)
+		}
+
+		if opts.Test {
+			fmt.Println("Completion:", completion)
 		}
-		OverallReviewCompletion += fmt.Sprintf("File: %s \nReview: %s \n\n", file.GetFilename(), completion)
 
-		position := len(strings.Split(*file.Patch, "\n")) - 1
+		review, err := extractJSON(completion)
+		if err != nil {
+			fmt.Println("Error extracting JSON:", err)
+			continue
+		}
 
-		comment := &github.PullRequestComment{
-			CommitID: diff.Commits[len(diff.Commits)-1].SHA,
-			Path:     file.Filename,
-			Body:     &completion,
-			Position: &position,
+		if review.Quality == Good {
+			fmt.Println("Review is good")
+			continue
+		}
+		for _, issue := range review.Issues {
+			body := fmt.Sprintf("[%s] %s", issue.Type, issue.Description)
+			comment := &github.PullRequestComment{
+				CommitID: diff.Commits[len(diff.Commits)-1].SHA,
+				Path:     file.Filename,
+				Body:     &body,
+				Position: &issue.Line,
+			}
+			comments = append(comments, comment)
 		}
 
 		if opts.Test {
 			continue
 		}
 
-		if _, err := githubClient.CreatePullRequestComment(ctx, opts.Owner, opts.Repo, opts.PRNumber, comment); err != nil {
-			return fmt.Errorf("error creating comment: %w", err)
+		for i, c := range comments {
+			fmt.Printf("creating comment: %s %d/%d\n", *c.Path, i+1, len(comments))
+			if _, err := githubClient.CreatePullRequestComment(ctx, opts.Owner, opts.Repo, opts.PRNumber, c); err != nil {
+				return fmt.Errorf("error creating comment: %w", err)
+			}
 		}
 	}
+	return nil
+}
 
-	overallCompletion, err := openAIClient.ChatCompletion(ctx, []openai.ChatCompletionMessage{
-		{
-			Role:    openai.ChatMessageRoleUser,
-			Content: fmt.Sprintf(oAIClient.PromptOverallReview, OverallReviewCompletion),
-		},
-	})
-	if err != nil {
-		return fmt.Errorf("error getting overall review: %w", err)
-	}
+type Review struct {
+	Quality Quality `json:"quality"`
+	Issues  []struct {
+		Type        string `json:"type"`
+		Line        int    `json:"line"`
+		Description string `json:"description"`
+	} `json:"issues"`
+}
 
-	if opts.Test {
-		fmt.Println(OverallReviewCompletion)
-		fmt.Println("=====================================")
-		fmt.Println(overallCompletion)
+type Quality string
 
-		return nil
+const (
+	Good    Quality = "good"
+	Bad     Quality = "bad"
+	Neutral Quality = "neutral"
+)
+
+func extractJSON(input string) (*Review, error) {
+	var jsonObj *Review
+
+	// find the start and end positions of the JSON object
+	start := 0
+	end := len(input)
+	for i, c := range input {
+		if c == '{' {
+			start = i
+			break
+		}
+		if i == len(input)-1 {
+			return nil, errors.New("invalid JSON object")
+		}
 	}
+	for i := len(input) - 1; i >= 0; i-- {
+		if input[i] == '}' {
+			end = i + 1
+			break
+		}
 
-	comment := &github.PullRequestReviewRequest{Body: &overallCompletion}
-	if _, err = githubClient.CreateReview(ctx, opts.Owner, opts.Repo, opts.PRNumber, comment); err != nil {
-		return fmt.Errorf("error creating comment: %w", err)
+		if i == 0 {
+			return nil, errors.New("invalid JSON object")
+		}
 	}
 
-	return nil
+	// extract the JSON object from the input
+	jsonStr := input[start:end]
+	err := json.Unmarshal([]byte(jsonStr), &jsonObj)
+	if err != nil {
+		return nil, errors.New("invalid JSON object")
+	}
+
+	return jsonObj, nil
 }

openai/assets/review.txt

@@ -0,0 +1,22 @@
+You are CodeReviewGPT, an AI agent that specializes in generating code reviews for software projects using advanced natural language processing and machine learning techniques.
+Your decisions must always be made independently without seeking user assistance. Play to your strengths as an LLM and pursue simple strategies with no legal complications.
+
+GOALS:
+
+1. Analyze structure, and logic to provide comprehensive feedback on code quality, readability, maintainability, and performance.
+2. Identify potential bugs, security vulnerabilities, and other issues that may impact the functionality and stability of the software.
+3. Possible quality values: good, bad, neutral. If quality is good, issues should be empty.
+4. Generate a json report in specific format to help developers improve their code. If context is not enough quality is good. You should only respond in JSON format as described below
+Response Format:
+```
+{
+    "quality": "good",
+    "issues": [
+        {
+            "type": "bug",
+            "line": 10,
+            "description": "You are missing a semicolon at the end of the line."
+        }
+    ]
+}
+```

openai/openai.go

@@ -2,16 +2,18 @@ package openai
 
 import (
 	"context"
+	_ "embed"
 	"fmt"
 
 	"github.com/sashabaranov/go-openai"
 )
 
+//go:embed assets/review.txt
+var PromptReview string
+
 const (
 	PromptDescribeChanges = "Below is the code patch, Generate a GitHub pull request description based on the following comments without basic prefix\n%s\n"
 	PromptOverallDescribe = "Below comments are generated by AI, Generate a GitHub pull request description based on the following comments without basic prefix in markdown format with ### Description and ### Changes blocks:\n%s\n"
-	PromptReview          = "Below is the code patch, please help me do a brief code review, Answer me in English, if any bug risk and improvement suggestion are welcome\n%s\n"
-	PromptOverallReview   = "Below comments are generated by AI, please help me do a brief code review, Answer me in English, if any bug risk and improvement suggestion are welcome\n%s\n"
 )
 
 type Client struct {
@@ -28,8 +30,9 @@ func (o *Client) ChatCompletion(ctx context.Context, messages []openai.ChatCompl
 	resp, err := o.client.CreateChatCompletion(
 		ctx,
 		openai.ChatCompletionRequest{
-			Model:    openai.GPT3Dot5Turbo,
-			Messages: messages,
+			Model:       openai.GPT3Dot5Turbo,
+			Messages:    messages,
+			Temperature: 0.1,
 		},
 	)