Actions workflow to sync icons to the bucket

.github/workflows/lint.yml

@@ -1,14 +1,18 @@
-name: Linting
+name: Lint & Validate
 
 on:
   pull_request_target:
     branches:
       - main
-  workflow_dispatch:
+    paths-ignore:
+      - "src/technologies/*.json"
+      - "src/categories.json"
+      - "src/groups.json"
+      - "src/images/icons/**"
 
 jobs:
   lint:
-    name: Lint
+    name: Lint & Validate
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
@@ -20,9 +24,7 @@ jobs:
       - name: Install dependencies
         run: yarn install
 
-      - name: Validate
-        run: yarn run validate
-
-      - name: Run WebPageTest with unit tests
-        id: unit-test
-        run: yarn run lint
+      - name: Lint and Validate
+        run: |
+          yarn run lint
+          yarn run validate

.github/workflows/test.yml

@@ -9,6 +9,7 @@ on:
       - "src/categories.json"
       - "src/groups.json"
       - "tests/*.js"
+      - ".github/workflows/test.yml"
   workflow_dispatch:
 
 jobs:
@@ -25,8 +26,10 @@ jobs:
       - name: Install dependencies
         run: yarn install
 
-      - name: Validate
-        run: yarn run validate
+      - name: Lint & Validate
+        run: |
+          yarn run lint
+          yarn run validate
 
       - name: Run WebPageTest with unit tests
         id: unit-test

.github/workflows/upload.yml

@@ -1,45 +1,91 @@
-name: Tests
+name: Upload technologies to BQ and Cloud Storage
 
 on:
   push:
     branches:
       - main
     paths:
+      - ".github/workflows/upload.yml"
       - "src/technologies/*.json"
       - "src/categories.json"
       - "src/groups.json"
+      - "src/images/icons/**"
   workflow_dispatch:
+    inputs:
+      force_update:
+        description: 'Force update regardless of changes'
+        required: false
+        default: 'false'
 
 jobs:
   test:
     name: Test and upload to GCP
     runs-on: ubuntu-latest
     steps:
+
       - name: Checkout
         uses: actions/checkout@v4
         with:
           ref: ${{ github.event.pull_request.head.sha }}
-          fetch-depth: 0
 
       - name: Install dependencies
         run: yarn install
 
       - name: Validate
         run: yarn run validate
 
-      - name: Run WebPageTest with unit tests
-        id: unit-test
-        env:
-          WPT_SERVER: "webpagetest.httparchive.org"
-          WPT_API_KEY: ${{ secrets.HA_API_KEY }}
-          PR_NUMBER: ${{ github.event.pull_request.number }}
-        run: yarn run test
+      - name: Get changed technology files
+        run: |
+          git log -1 HEAD^
+          git log -1 HEAD
+
+          git fetch origin main --depth=1
+          TECH_FILES_CHANGED=$(git diff --name-only HEAD^ HEAD -- 'src/technologies/*.json' 'src/categories.json' 'src/groups.json' | wc -l)
+          echo $TECH_FILES_CHANGED
+          echo "TECH_FILES_CHANGED=${TECH_FILES_CHANGED}" >> $GITHUB_ENV
 
-      - name: Upload to GCP
-        id: upload
+      - name: Upload technology rules to BigQuery
+        if: env.TECH_FILES_CHANGED != '' || inputs.force_update == 'true'
         env:
-          GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
           GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
+        run: yarn run tech_upload
+
+      - name: Get changed icons
+        run: |
+          git fetch origin main --depth=1
+
+          if [[ "${{ inputs.force_update }}" == "true" ]]; then
+          CHANGED_ICONS=$(find src/images/icons -name "*.svg" -o -name "*.png" | xargs)
+          else
+          CHANGED_ICONS=$(git diff --name-only HEAD^ HEAD -- 'src/images/icons/*.svg' 'src/images/icons/*.png' | xargs)
+          fi
+
+          echo $CHANGED_ICONS
+          echo "CHANGED_ICONS=${CHANGED_ICONS}" >> $GITHUB_ENV
+
+      # PNG are generally smaller, and we have some SVG that are 1Mb, so conversion to PNG is necessary.
+      - name: Install Image Conversion Package
+        if: env.CHANGED_ICONS != ''
+        run: sudo apt-get install -y librsvg2-bin
+
+      - name: Convert changed SVGs to PNGs
+        if: env.CHANGED_ICONS != ''
+        run: |
+          for file in $CHANGED_ICONS; do
+            if [[ "$file" == *.svg ]]; then
+              rsvg-convert "$file" -o "${file%.svg}.png" -w 16 -h 16
+            fi
+          done
+
+      - uses: 'google-github-actions/auth@v2'
+        if: env.CHANGED_ICONS != '' || inputs.force_update == 'true'
+        with:
+          credentials_json: ${{ secrets.GCP_SA_KEY }}
+
+      - uses: 'google-github-actions/setup-gcloud@v2'
+        if: env.CHANGED_ICONS != '' || inputs.force_update == 'true'
+
+      - name: Sync new PNGs to Cloud Storage
+        if: env.CHANGED_ICONS != '' || inputs.force_update == 'true'
         run: |
-          echo $GCP_SA_KEY > /tmp/gcp_key.json
-          yarn run upload
+          gcloud storage rsync src/images/icons/ gs://httparchive/static/icons/ --exclude=".*.svg$" --cache-control="public, max-age=31536000, immutable"

package.json

@@ -23,11 +23,11 @@
   "scripts": {
     "lint": "eslint src/**/*.{js,json} tests/**/*.js scripts/**/*.js && jsonlint -jsV ./schema.json --trim-trailing-commas --enforce-double-quotes ./src/technologies/ && jsonlint -js --trim-trailing-commas --enforce-double-quotes ./src/categories.json",
     "lint:fix": "eslint --fix src/**/*.{js,json} tests/**/*.js scripts/**/*.js && jsonlint -isV ./schema.json --trim-trailing-commas --enforce-double-quotes ./src/technologies/ && jsonlint -is --trim-trailing-commas --enforce-double-quotes ./src/categories.json",
-    "validate": "yarn run lint && node ./scripts/validate.js",
+    "validate": "node ./scripts/validate.js",
     "test": "jest",
-    "upload": "node ./scripts/bigquery_upload.js",
-    "convert": "node --no-warnings ./scripts/convert.js",
-    "build": "yarn run validate && yarn run convert && node ./scripts/build.js"
+    "tech_upload": "node ./scripts/tech_upload.js",
+    "convert": "node ./scripts/convert.js",
+    "build": "yarn run lint && yarn run validate && yarn run convert && node ./scripts/build.js"
   },
   "jest": {
     "reporters": [

scripts/convert.js

@@ -164,12 +164,10 @@ function dateDiff(file) {
 })()
 
 /**
-
 cd  ; cp *.svg converted ; cd converted ; convert-svg-to-png *.svg --width 32 --height 32 ; rm *.svg
 (async() => {
   const inputFilePath = '/path/to/my-image.svg';
   const outputFilePath = await convertFile(inputFilePath);
-
   console.log(outputFilePath);
   //=> "/path/to/my-image.png"
 })();

scripts/bigquery_upload.js → scripts/tech_upload.js

@@ -6,7 +6,7 @@ const path = require('path')
 const { BigQuery } = require('@google-cloud/bigquery')
 
 const bigquery = new BigQuery({
-  keyFilename: '/tmp/gcp_key.json',
+  credentials: JSON.parse(process.env.GCP_SA_KEY),
 })
 
 const schemas = {
@@ -200,17 +200,16 @@ const main = async () => {
         app[field] = getRuleObject(technologies[key][field])
       }
     )
-    ;[
-      'website',
-      'description',
-      'icon',
-      'cpe',
-      'saas',
-      'oss',
-      'pricing',
-    ].forEach((field) => {
-      app[field] = technologies[key][field]
-    })
+    ;['website', 'description', 'cpe', 'saas', 'oss', 'pricing'].forEach(
+      (field) => {
+        app[field] = technologies[key][field]
+      }
+    )
+
+    // Handle icon field separately to ensure .png extension
+    app.icon = technologies[key].icon
+      ? `${technologies[key].icon.replace(/\.[^/.]+$/, '')}.png`
+      : technologies[key].icon
 
     return app
   })

scripts/validate.js

@@ -208,3 +208,5 @@ fs.readdirSync(iconPath).forEach((file) => {
     }
   }
 })
+
+console.log('Validation completed successfully.')

src/technologies/e.json

@@ -146,20 +146,6 @@
     ],
     "website": "https://erpnext.com"
   },
-  "Essential Blocks": {
-    "cats": [
-      87
-    ],
-    "description": "Essential Blocks is a Gutenberg Blocks page builder plugin that adds extra functionality to the WordPress Block Editor.",
-    "icon": "essential-blocks.png",
-    "requires": "WordPress",
-    "dom": [
-      "link[href*='/wp-content/uploads/eb-style/']",
-      "link#essential-blocks-frontend-style-css"
-    ],
-    "html": "<div[^>]+class=[\"']wp-block-essential-blocks-*",
-    "website": "https://essential-blocks.com"
-  },
   "ESW": {
     "cats": [
       106
@@ -1492,6 +1478,20 @@
     "scriptSrc": "/wp-content/uploads/essential-addons-elementor/",
     "website": "https://essential-addons.com/elementor/"
   },
+  "Essential Blocks": {
+    "cats": [
+      87
+    ],
+    "description": "Essential Blocks is a Gutenberg Blocks page builder plugin that adds extra functionality to the WordPress Block Editor.",
+    "dom": [
+      "link[href*='/wp-content/uploads/eb-style/']",
+      "link#essential-blocks-frontend-style-css"
+    ],
+    "html": "<div[^>]+class=[\"']wp-block-essential-blocks-*",
+    "icon": "essential-blocks.png",
+    "requires": "WordPress",
+    "website": "https://essential-blocks.com"
+  },
   "Essential JS 2": {
     "cats": [
       12,

src/technologies/g.json

@@ -448,13 +448,13 @@
       87
     ],
     "description": "Premium block library & block editor enhancements from StudioPress.",
-    "icon": "genesis-blocks.png",
-    "requires": "WordPress",
     "dom": [
-      "link[href*='/wp-content/plugins/atomic-blocks']", 
+      "link[href*='/wp-content/plugins/atomic-blocks']",
       "link[href*='/wp-content/plugins/genesis-blocks']"
     ],
     "html": "<div[^>]+class=[\"']wp-block-atomic-blocks-*",
+    "icon": "genesis-blocks.png",
+    "requires": "WordPress",
     "scriptSrc": "/wp-content/themes/genesis-blocks/lib/js/",
     "website": "https://www.studiopress.com/genesis-blocks/"
   },
@@ -463,12 +463,12 @@
       80
     ],
     "description": "Genesis theme is a WordPress theme that has been developed using the Genesis Framework from Studiopress.",
-    "icon": "Genesis theme.svg",
     "dom": [
       "body[class*='theme-genesis']",
       "body[class*='genesis-breadcrumbs-hidden']",
       "body[class*='genesis-title-hidden']"
     ],
+    "icon": "Genesis theme.svg",
     "requires": "WordPress",
     "scriptSrc": "/wp-content/themes/genesis/lib/js/",
     "website": "https://www.studiopress.com/themes/genesis"