[release tool] hashrelease promotion pipeline

.semaphore/release/hashrelease.yml

@@ -47,7 +47,6 @@ blocks:
       jobs:
         - name: Build and publish hashrelease
           commands:
-            - if [[ ${SEMAPHORE_WORKFLOW_TRIGGERED_BY_SCHEDULE} == "true" ]]; then export BUILD_CONTAINER_IMAGES=true; export PUBLISH_IMAGES=true; fi
             - make hashrelease
       prologue:
         commands:

release/cmd/flags.go

@@ -258,7 +258,7 @@ var (
 // External flags are flags used to interact with external services
 var (
 	// CI flags for interacting with CI services (Semaphore)
-	ciFlags     = []cli.Flag{ciFlag, ciBaseURLFlag, ciJobIDFlag}
+	ciFlags     = []cli.Flag{ciFlag, ciBaseURLFlag, ciJobIDFlag, ciPipelineIDFlag, ciTokenFlag}
 	semaphoreCI = "semaphore"
 	ciFlag      = &cli.BoolFlag{
 		Name:    "ci",
@@ -282,6 +282,16 @@ var (
 		Usage:   fmt.Sprintf("The job ID for the %s CI job", semaphoreCI),
 		EnvVars: []string{"SEMAPHORE_JOB_ID"},
 	}
+	ciPipelineIDFlag = &cli.StringFlag{
+		Name:    "ci-pipeline-id",
+		Usage:   fmt.Sprintf("The pipeline ID for the %s CI pipeline", semaphoreCI),
+		EnvVars: []string{"SEMAPHORE_PIPELINE_ID"},
+	}
+	ciTokenFlag = &cli.StringFlag{
+		Name:    "ci-token",
+		Usage:   fmt.Sprintf("The token for interacting with %s API", semaphoreCI),
+		EnvVars: []string{"SEMAPHORE_API_TOKEN"},
+	}
 
 	// Slack flags for posting messages to Slack
 	slackFlags     = []cli.Flag{slackTokenFlag, slackChannelFlag, notifyFlag}

release/cmd/hashrelease.go

@@ -21,6 +21,7 @@ import (
 	"github.com/sirupsen/logrus"
 	cli "github.com/urfave/cli/v2"
 
+	"github.com/projectcalico/calico/release/internal/ci"
 	"github.com/projectcalico/calico/release/internal/hashreleaseserver"
 	"github.com/projectcalico/calico/release/internal/imagescanner"
 	"github.com/projectcalico/calico/release/internal/outputs"
@@ -64,6 +65,10 @@ func hashreleaseSubCommands(cfg *Config) []*cli.Command {
 					return err
 				}
 
+				if err := validateCIBuildRequirements(c, cfg.RepoRootDir); err != nil {
+					return err
+				}
+
 				// Clone the operator repository.
 				operatorDir := filepath.Join(cfg.TmpDir, operator.DefaultRepoName)
 				err := operator.Clone(c.String(operatorOrgFlag.Name), c.String(operatorRepoFlag.Name), c.String(operatorBranchFlag.Name), operatorDir)
@@ -306,6 +311,9 @@ func validateHashreleaseBuildFlags(c *cli.Context) error {
 			return fmt.Errorf("missing hashrelease server configuration, must set %s, %s, %s, %s, and %s",
 				sshHostFlag, sshUserFlag, sshKeyFlag, sshPortFlag, sshKnownHostsFlag)
 		}
+		if c.String(ciTokenFlag.Name) == "" {
+			return fmt.Errorf("%s API token must be set when running on CI, either set \"SEMAPHORE_API_TOKEN\" or use %s flag", semaphoreCI, ciTokenFlag.Name)
+		}
 	} else {
 		// If building images, log a warning if no registry is specified.
 		if c.Bool(buildHashreleaseImageFlag.Name) && len(c.StringSlice(registryFlag.Name)) == 0 {
@@ -381,3 +389,24 @@ func imageScanningAPIConfig(c *cli.Context) *imagescanner.Config {
 		Scanner: c.String(imageScannerSelectFlag.Name),
 	}
 }
+
+func validateCIBuildRequirements(c *cli.Context, repoRootDir string) error {
+	if !c.Bool(ciFlag.Name) {
+		return nil
+	}
+	if c.Bool(buildImagesFlag.Name) {
+		logrus.Debug("Building images, skipping images promotions check...")
+		return nil
+	}
+	orgURL := c.String(ciBaseURLFlag.Name)
+	token := c.String(ciTokenFlag.Name)
+	pipelineID := c.String(ciPipelineIDFlag.Name)
+	promotionsDone, err := ci.ImagePromotionsDone(repoRootDir, orgURL, pipelineID, token)
+	if err != nil {
+		return fmt.Errorf("failed to check if images promotions are done: %v", err)
+	}
+	if !promotionsDone {
+		return fmt.Errorf("images promotions are not done, wait for all images promotions to pass before publishing the hashrelease")
+	}
+	return nil
+}

release/internal/ci/semaphore.go

@@ -0,0 +1,197 @@
+package ci
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"strconv"
+	"strings"
+
+	"github.com/sirupsen/logrus"
+
+	"github.com/projectcalico/calico/release/internal/command"
+)
+
+const passed = "passed"
+
+type promotion struct {
+	Status     string `json:"status"`
+	Name       string `json:"name"`
+	PipelineID string `json:"scheduled_pipeline_id"`
+}
+
+type pipeline struct {
+	Result      string `json:"result"`
+	PromotionOf string `json:"promotion_of"`
+}
+
+type pipelineDetails struct {
+	Pipeline pipeline `json:"pipeline"`
+}
+
+func apiURL(orgURL, path string) string {
+	orgURL = strings.TrimPrefix(orgURL, "/")
+	path = strings.TrimSuffix(path, "/")
+	return fmt.Sprintf("%s/api/v1alpha/%s", orgURL, path)
+}
+
+func fetchImagePromotions(orgURL, pipelineID, token string) ([]promotion, error) {
+	url := apiURL(orgURL, "/promotions")
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create %s request: %s", url, err.Error())
+	}
+	req.Header.Set("Authorization", fmt.Sprintf("Token %s", token))
+	q := req.URL.Query()
+	q.Add("pipeline_id", pipelineID)
+	req.URL.RawQuery = q.Encode()
+
+	logrus.WithField("url", req.URL.String()).Debug("get pipeline promotions")
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("failed to request promotions: %s", err.Error())
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("failed to fetch promotions")
+	}
+
+	var promotions []promotion
+	if err := json.NewDecoder(resp.Body).Decode(&promotions); err != nil {
+		return nil, fmt.Errorf("failed to parse promotions: %s", err.Error())
+	}
+
+	imagesPromotionsMap := make(map[string]promotion)
+	for _, p := range promotions {
+		if strings.HasPrefix(strings.ToLower(p.Name), "push ") {
+			if currentP, ok := imagesPromotionsMap[p.Name]; ok {
+				// If the promotion is already in the map,
+				// only if the staus for the promotion in the map is not passed.
+				if currentP.Status != passed {
+					imagesPromotionsMap[p.Name] = p
+				}
+			} else {
+				imagesPromotionsMap[p.Name] = p
+			}
+		}
+	}
+
+	imagesPromotions := make([]promotion, 0, len(imagesPromotionsMap))
+	for _, p := range imagesPromotionsMap {
+		imagesPromotions = append(imagesPromotions, p)
+	}
+	return imagesPromotions, nil
+}
+
+func getPipelineResult(orgURL, pipelineID, token string) (*pipeline, error) {
+	url := apiURL(orgURL, fmt.Sprintf("/pipelines/%s", pipelineID))
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create %s request: %s", url, err.Error())
+	}
+	req.Header.Set("Authorization", fmt.Sprintf("Token %s", token))
+
+	logrus.WithField("url", req.URL.String()).Debug("get pipeline details")
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("failed to request pipeline details: %s", err.Error())
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("failed to fetch pipeline details")
+	}
+
+	var p pipelineDetails
+	if err := json.NewDecoder(resp.Body).Decode(&p); err != nil {
+		return nil, fmt.Errorf("failed to parse pipeline: %s", err.Error())
+	}
+
+	return &p.Pipeline, err
+}
+
+func fetchParentPipelineID(orgURL, pipelineID, token string) (string, error) {
+	url := apiURL(orgURL, fmt.Sprintf("/pipelines/%s", pipelineID))
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return "", fmt.Errorf("failed to create %s request: %s", url, err.Error())
+	}
+	req.Header.Set("Authorization", fmt.Sprintf("Token %s", token))
+
+	logrus.WithField("url", req.URL.String()).Debug("get pipeline details")
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return "", fmt.Errorf("failed to request pipeline details: %s", err.Error())
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("failed to fetch pipeline details")
+	}
+
+	var p pipelineDetails
+	if err := json.NewDecoder(resp.Body).Decode(&p); err != nil {
+		return "", fmt.Errorf("failed to parse pipeline: %s", err.Error())
+	}
+
+	return p.Pipeline.PromotionOf, err
+}
+
+// ImagePromotionsDone checks if all the promotion pipelines have passed.
+//
+// As it is checking in the hashrelease pipeline, it tries to get the pipeline that triggered the hashrelease promotion.
+// If the pipeline that triggered the hashrelease promotion is not found,
+// this means that the hashrelease pipeline was not triggered by a promotion (likely triggered from a task).
+// In this case, it skips the image promotions check.
+//
+// Once the pipeline that triggered the hashrelease promotion is found, it checks if all the expected image promotions have passed.
+// Since the API only return promotions that have been triggered, it is possible that some promotions are not triggered.
+// This is why it checks that the number of promotions is equal or greater than the expected number from the semaphore.yml.
+func ImagePromotionsDone(repoRootDir, orgURL, pipelineID, token string) (bool, error) {
+	expectPromotionCountStr, err := command.Run("grep", []string{"-c", `"name: Push "`, fmt.Sprintf("%s/.semaphore/semaphore.yml.d/03-promotions.yml", repoRootDir)})
+	if err != nil {
+		return false, fmt.Errorf("failed to get expected image promotions")
+	}
+	expectedPromotionCount, err := strconv.Atoi(expectPromotionCountStr)
+	if err != nil {
+		return false, fmt.Errorf("unable to convert expected promotions to int")
+	}
+	logrus.WithField("count", expectedPromotionCount).Debug("expected number of image promotions")
+	parentPipelineID, err := fetchParentPipelineID(orgURL, pipelineID, token)
+	if err != nil {
+		return false, err
+	}
+	if parentPipelineID == "" {
+		logrus.Info("no parent pipeline found, skipping image promotions check")
+		return true, nil
+	}
+	logrus.WithField("pipeline_id", parentPipelineID).Debug("found pipeline that triggered image promotions")
+	promotions, err := fetchImagePromotions(orgURL, pipelineID, token)
+	if err != nil {
+		return false, err
+	}
+	promotionsCount := len(promotions)
+	logrus.WithFields(logrus.Fields{
+		"expected": expectedPromotionCount,
+		"actual":   promotionsCount,
+	}).Debug("number of image promotions")
+	if promotionsCount < expectedPromotionCount {
+		return false, fmt.Errorf("number of promotions do not match: expected %d, got %d", expectedPromotionCount, promotionsCount)
+	}
+	for _, promotion := range promotions {
+		if promotion.Status != passed {
+			logrus.WithField("promotion", promotion.Name).Error("triggering promotion failed")
+			return false, fmt.Errorf("triggering %q promotion failed, cannot check pipeline result", promotion.Name)
+		}
+		pipeline, err := getPipelineResult(orgURL, promotion.PipelineID, token)
+		if err != nil {
+			return false, fmt.Errorf("unable to get %q pipeline details", promotion.Name)
+		}
+		if pipeline.Result != passed {
+			logrus.WithField("promotion", promotion.Name).Error("promotion failed")
+			return false, fmt.Errorf("%q promotion failed", promotion.Name)
+		}
+	}
+	return true, nil
+}