diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
index ad6e05a73464..d60b123f257c 100644
--- a/.github/pull_request_template.md
+++ b/.github/pull_request_template.md
@@ -1,26 +1,12 @@
-<!--
-COMMENT BLOCKS WILL NOT BE INCLUDED IN THE PR.
-Feel free to delete sections of the template which do not apply to your PR, or add additional details
--->
-
 ###### Merge Checklist  <!-- REQUIRED -->
-<!-- You can set them now ([x]) or set them later using the Github UI -->
-<!-- **All** boxes should be checked before merging the PR *(just tick any boxes which don't apply to this PR)* -->
 - [ ] Followed patch format from upstream recommendation: https://github.com/kata-containers/community/blob/main/CONTRIBUTING.md#patch-format
   - [ ] Included a single commit in a given PR - at least unless there are related commits and each makes sense as a change on its own.
 - [ ] Aware about the PR to be merged using "create a merge commit" rather than "squash and merge" (or similar)
 - [ ] genPolicy only: Ensured the tool still builds on Windows
-- [ ] genPolicy only: Updated sample YAMLs' policy annotations, if applicable
-- [ ] The `upstream-missing` label (or `upstream-not-needed`) has been set on the PR. 
+- [ ] The `upstream/missing` label (or `upstream/not-needed`) has been set on the PR. 
 
 ###### Summary <!-- REQUIRED -->
 <!-- Quick explanation of WHAT changed and WHY. -->
 
-###### Associated issues  <!-- optional -->
-<!-- Link to Github issues if possible. -->
-
-###### Links to CVEs  <!-- optional -->
-<!-- https://nvd.nist.gov/vuln/detail/CVE-YYYY-XXXX -->
-
 ###### Test Methodology
 <!-- How was this test validated? i.e. local build, pipeline build etc. -->
diff --git a/.github/workflows/check-samples.yaml b/.github/workflows/check-samples.yaml
new file mode 100644
index 000000000000..92945a2fa2da
--- /dev/null
+++ b/.github/workflows/check-samples.yaml
@@ -0,0 +1,49 @@
+# Copyright (c) Microsoft Corporation.
+
+name: Check policy samples
+
+on:
+  pull_request:
+
+jobs:
+  check-policy-samples:
+    runs-on: ubuntu-latest
+
+    steps:
+
+    - name: Check out code
+      uses: actions/checkout@v4
+
+    - name: Install yq
+      env:
+        INSTALL_IN_GOPATH: false
+      run: |
+        ./ci/install_yq.sh
+
+    - name: Install Rust
+      run: |
+        ./tests/install_rust.sh
+        echo "${HOME}/.cargo/bin" >> $GITHUB_PATH
+
+    - name: Install protobuf-compiler
+      run: |
+        sudo apt-get -y install protobuf-compiler
+
+    - name: Configure containerd
+      run: |
+        sudo containerd config default | sudo dd of=/etc/containerd/config.toml
+        sudo systemctl restart containerd
+        sudo systemctl is-active containerd
+
+    - name: Update policy samples
+      working-directory: ./src/tools/genpolicy
+      run: |
+        python3 update_policy_samples.py
+
+    - name: Show diff
+      run: |
+        git diff
+
+    - name: Check policy samples
+      run: |
+        git diff-files --exit-code
diff --git a/src/tools/genpolicy/update_policy_samples.py b/src/tools/genpolicy/update_policy_samples.py
index e5ffcb082849..35c54993a156 100644
--- a/src/tools/genpolicy/update_policy_samples.py
+++ b/src/tools/genpolicy/update_policy_samples.py
@@ -1,4 +1,4 @@
-from concurrent.futures import ThreadPoolExecutor
+import concurrent.futures
 import os
 import subprocess
 import sys
@@ -19,42 +19,56 @@
 file_base_path = "../../agent/samples/policy/yaml"
 
 def runCmd(arg):
-    return subprocess.run([arg], stdout=sys.stdout, stderr=sys.stderr, universal_newlines=True, input="", shell=True)
+    return subprocess.run([arg], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True, input="", shell=True, check=True)
 
 def timeRunCmd(arg):
+    log = [f"========== COMMAND: {arg}"]
     start = time.time()
-    proc = runCmd(arg)
-    end = time.time()
 
-    log = f"COMMAND: {arg}\n"
-    if proc.returncode != 0:
-        log += f"`{arg}` failed with exit code {proc.returncode}. Stderr: {proc.stderr}, Stdout: {proc.stdout}\n"
-    log += f"Time taken: {round(end - start, 2)} seconds"
-    print(log)
+    try:
+        p = runCmd(arg)
+    except subprocess.CalledProcessError as e:
+        log.append(e.stdout)
+        log.append(f"+++++ Failed with exit code {e.returncode}")
+        raise
+    else:
+        if p.stdout:
+            log.append(p.stdout)
+    finally:
+        end = time.time()
+        log.append(f"Time taken: {round(end - start, 2)} seconds")
+        print("\n".join(log))
 
 # check we can access all files we are about to update
 for file in default_yamls + silently_ignored + no_policy:
     filepath = os.path.join(file_base_path, file)
     if not os.path.exists(filepath):
-        print(f"filepath does not exists: {filepath}")
+        sys.exit(f"filepath does not exists: {filepath}")
 
 # build tool
-print("COMMAND: cargo build")
-runCmd("cargo build")
+print("========== COMMAND: LIBC=gnu BUILD_TYPE= make")
+runCmd("LIBC=gnu BUILD_TYPE= make")
 
 # update files
-genpolicy_path = "target/debug/genpolicy"
+genpolicy_path = "./target/x86_64-unknown-linux-gnu/debug/genpolicy"
 
 total_start = time.time()
-executor = ThreadPoolExecutor(max_workers=os.cpu_count())
 
-for file in default_yamls + no_policy + needs_containerd_pull:
-    executor.submit(timeRunCmd, f"sudo {genpolicy_path} -d -y {os.path.join(file_base_path, file)}")
+with concurrent.futures.ThreadPoolExecutor(max_workers=os.cpu_count()) as executor:
+    futures = []
 
-for file in silently_ignored:
-    executor.submit(timeRunCmd, f"sudo {genpolicy_path} -d -s -y {os.path.join(file_base_path, file)}")
+    for file in default_yamls + no_policy + needs_containerd_pull:
+        cmd = f"sudo {genpolicy_path} -d -y {os.path.join(file_base_path, file)}"
+        futures.append(executor.submit(timeRunCmd, cmd))
+
+    for file in silently_ignored:
+        cmd = f"sudo {genpolicy_path} -d -s -y {os.path.join(file_base_path, file)}"
+        futures.append(executor.submit(timeRunCmd, cmd))
+
+    for future in concurrent.futures.as_completed(futures):
+        # Surface any potential exception thrown by the future.
+        future.result()
 
-executor.shutdown()
 total_end = time.time()
 
 print(f"Total time taken: {total_end - total_start} seconds")