WebNN: Don't run per-op WPTs in non-window contexts

The WebNN tests are written as ".any.js" tests which can run in both
Window and Worker contexts. This is great! But we running every test
on every combination of (CPU, GPU, NPU) x (Window, Worker) consumes
significant cycles for little benefit. While tests and various backend
and platform implementations are in flux, it also adds significant
burden to gardeners.

Scope down to running just the Window variations for op-specific
tests. Tests for things like inputs, constants, tensors, and object
lifecycles are still run in all contexts, and identity() is retained
as a "canary".

Change-Id: I1f1c749ea4d002a578d6d2ca89579cec6b0090e2
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/6299831
Reviewed-by: Weizhong Xia <[email protected]>
Commit-Queue: Weizhong Xia <[email protected]>
Auto-Submit: Joshua Bell <[email protected]>
Reviewed-by: Reilly Grant <[email protected]>
Cr-Commit-Position: refs/heads/main@{#1424187}

webnn/README.md

@@ -0,0 +1,54 @@
+# Web Neural Network API Test Suite
+
+Spec: https://www.w3.org/TR/webnn/
+
+Repo: https://github.com/webmachinelearning/webnn
+
+## Web IDL Tests
+
+The `idlharness.https.any.js` test validates that the API implements
+the interfaces as defined in the Web IDL in the specification. A
+snapshot of the spec's Web IDL resides in `../interfaces/webnn.idl`
+and is automatically daily by shared WPT infrastructure.
+
+## Validation Tests
+
+The tests in `validation_tests` go beyond what can be validated from
+the Web IDL definitions in the spec. They ensure that parameters to
+the various WebNN methods are validated according to the algorithms in
+the spec, and that invalid inputs are rejected as specified. For
+example, `validation_tests/matmul.https.any.js` verifies that an
+exception is thrown if the rank of either of the input tensors is less
+than 2, or if the input tensor shapes are not broadcastable, etc.
+
+## Conformance Tests
+
+The tests in `conformance_tests` exercise the various WebNN operators
+and ensure that they are behaving as expected. For example,
+`conformance_tests/matmul.https.any.js` verifies that multiplying
+N-dimensional matrices actually produces the expected output.
+
+## Variations
+
+WebNN supports execution on various compute devices, e.g. CPU, GPU and
+NPU. The various tests make use of `META: variant=?...` to enable the
+test harness to execute the same test multiple times, with parameters
+specifying each device type in turn.
+
+NOTE: The specific API for device selection is currently under debate.
+See:
+https://github.com/webmachinelearning/webnn/blob/main/device-selection-explainer.md
+
+## Globals
+
+WebNN interfaces are exposed to both Window and Worker global contexts
+via the `navigator.ml` member. These tests make use of the `.any.js`
+convention and `META: global=window,dedicatedworker` to allow the test
+harness to execute the same test multiple times, once for each global
+context type.
+
+Since executing a large number of tests consumes significant compute
+cycles, and since unexpected behavior differences across contexts
+(e.g. `window` vs. `dedicatedworker`) for individual WebNN operators
+(e.g. `matmul()`) is unlikely, most operator-specific tests only
+specify `global=window`.

webnn/conformance_tests/abs.https.any.js

@@ -1,5 +1,5 @@
 // META: title=test WebNN API element-wise abs operation
-// META: global=window,dedicatedworker
+// META: global=window
 // META: variant=?cpu
 // META: variant=?gpu
 // META: variant=?npu

webnn/conformance_tests/add.https.any.js

@@ -1,5 +1,5 @@
 // META: title=test WebNN API element-wise add operation
-// META: global=window,dedicatedworker
+// META: global=window
 // META: variant=?cpu
 // META: variant=?gpu
 // META: variant=?npu

webnn/conformance_tests/arg_min_max.https.any.js

@@ -1,5 +1,5 @@
 // META: title=test WebNN API argMin/Max operations
-// META: global=window,dedicatedworker
+// META: global=window
 // META: variant=?cpu
 // META: variant=?gpu
 // META: variant=?npu

webnn/conformance_tests/batch_normalization.https.any.js

@@ -1,5 +1,5 @@
 // META: title=test WebNN API batchNormalization operation
-// META: global=window,dedicatedworker
+// META: global=window
 // META: variant=?cpu
 // META: variant=?gpu
 // META: variant=?npu

webnn/conformance_tests/cast.https.any.js

@@ -1,5 +1,5 @@
 // META: title=test WebNN API cast operation
-// META: global=window,dedicatedworker
+// META: global=window
 // META: variant=?cpu
 // META: variant=?gpu
 // META: variant=?npu

webnn/conformance_tests/ceil.https.any.js

@@ -1,5 +1,5 @@
 // META: title=test WebNN API element-wise ceil operation
-// META: global=window,dedicatedworker
+// META: global=window
 // META: variant=?cpu
 // META: variant=?gpu
 // META: variant=?npu

webnn/conformance_tests/clamp.https.any.js

@@ -1,5 +1,5 @@
 // META: title=test WebNN API clamp operation
-// META: global=window,dedicatedworker
+// META: global=window
 // META: variant=?cpu
 // META: variant=?gpu
 // META: variant=?npu

webnn/conformance_tests/concat.https.any.js

@@ -1,5 +1,5 @@
 // META: title=test WebNN API concat operation
-// META: global=window,dedicatedworker
+// META: global=window
 // META: variant=?cpu
 // META: variant=?gpu
 // META: variant=?npu

webnn/conformance_tests/conv2d.https.any.js

@@ -1,5 +1,5 @@
 // META: title=test WebNN API conv2d operation
-// META: global=window,dedicatedworker
+// META: global=window
 // META: variant=?cpu
 // META: variant=?gpu
 // META: variant=?npu

webnn/conformance_tests/conv_transpose2d.https.any.js

@@ -1,5 +1,5 @@
 // META: title=test WebNN API convTranspose2d operation
-// META: global=window,dedicatedworker
+// META: global=window
 // META: variant=?cpu
 // META: variant=?gpu
 // META: variant=?npu

webnn/conformance_tests/cos.https.any.js

@@ -1,5 +1,5 @@
 // META: title=test WebNN API element-wise cos operation
-// META: global=window,dedicatedworker
+// META: global=window
 // META: variant=?cpu
 // META: variant=?gpu
 // META: variant=?npu

webnn/conformance_tests/cumulative_sum.https.any.js

@@ -1,5 +1,5 @@
 // META: title=test WebNN API cumulativeSum operation
-// META: global=window,dedicatedworker
+// META: global=window
 // META: variant=?cpu
 // META: variant=?gpu
 // META: variant=?npu

webnn/conformance_tests/dequantizeLinear.https.any.js

@@ -1,5 +1,5 @@
 // META: title=test WebNN API dequantizeLinear operation
-// META: global=window,dedicatedworker
+// META: global=window
 // META: variant=?cpu
 // META: variant=?gpu
 // META: variant=?npu

webnn/conformance_tests/div.https.any.js

@@ -1,5 +1,5 @@
 // META: title=test WebNN API element-wise div operation
-// META: global=window,dedicatedworker
+// META: global=window
 // META: variant=?cpu
 // META: variant=?gpu
 // META: variant=?npu

webnn/conformance_tests/elu.https.any.js

@@ -1,5 +1,5 @@
 // META: title=test WebNN API elu operation
-// META: global=window,dedicatedworker
+// META: global=window
 // META: variant=?cpu
 // META: variant=?gpu
 // META: variant=?npu

webnn/conformance_tests/equal.https.any.js

@@ -1,5 +1,5 @@
 // META: title=test WebNN API element-wise equal operation
-// META: global=window,dedicatedworker
+// META: global=window
 // META: variant=?cpu
 // META: variant=?gpu
 // META: variant=?npu

webnn/conformance_tests/erf.https.any.js

@@ -1,5 +1,5 @@
 // META: title=test WebNN API element-wise erf operation
-// META: global=window,dedicatedworker
+// META: global=window
 // META: variant=?cpu
 // META: variant=?gpu
 // META: variant=?npu

webnn/conformance_tests/exp.https.any.js

@@ -1,5 +1,5 @@
 // META: title=test WebNN API element-wise exp operation
-// META: global=window,dedicatedworker
+// META: global=window
 // META: variant=?cpu
 // META: variant=?gpu
 // META: variant=?npu

webnn/conformance_tests/expand.https.any.js

@@ -1,5 +1,5 @@
 // META: title=test WebNN API expand operation
-// META: global=window,dedicatedworker
+// META: global=window
 // META: variant=?cpu
 // META: variant=?gpu
 // META: variant=?npu

webnn/conformance_tests/floor.https.any.js

@@ -1,5 +1,5 @@
 // META: title=test WebNN API element-wise floor operation
-// META: global=window,dedicatedworker
+// META: global=window
 // META: variant=?cpu
 // META: variant=?gpu
 // META: variant=?npu

webnn/conformance_tests/gather.https.any.js

@@ -1,5 +1,5 @@
 // META: title=test WebNN API gather operation
-// META: global=window,dedicatedworker
+// META: global=window
 // META: variant=?cpu
 // META: variant=?gpu
 // META: variant=?npu

webnn/conformance_tests/gatherElements.https.any.js

@@ -1,5 +1,5 @@
 // META: title=test WebNN API gatherElements operation
-// META: global=window,dedicatedworker
+// META: global=window
 // META: variant=?cpu
 // META: variant=?gpu
 // META: variant=?npu

webnn/conformance_tests/gatherND.https.any.js

@@ -1,5 +1,5 @@
 // META: title=test WebNN API gatherND operation
-// META: global=window,dedicatedworker
+// META: global=window
 // META: variant=?cpu
 // META: variant=?gpu
 // META: variant=?npu

webnn/conformance_tests/gelu.https.any.js

@@ -1,5 +1,5 @@
 // META: title=test WebNN API gelu operation
-// META: global=window,dedicatedworker
+// META: global=window
 // META: variant=?cpu
 // META: variant=?gpu
 // META: variant=?npu

webnn/conformance_tests/gemm.https.any.js

@@ -1,5 +1,5 @@
 // META: title=test WebNN API gemm operation
-// META: global=window,dedicatedworker
+// META: global=window
 // META: variant=?cpu
 // META: variant=?gpu
 // META: variant=?npu

webnn/conformance_tests/greater.https.any.js

@@ -1,5 +1,5 @@
 // META: title=test WebNN API element-wise greater operation
-// META: global=window,dedicatedworker
+// META: global=window
 // META: variant=?cpu
 // META: variant=?gpu
 // META: variant=?npu

webnn/conformance_tests/greater_or_equal.https.any.js

@@ -1,5 +1,5 @@
 // META: title=test WebNN API element-wise greaterOrEqual operation
-// META: global=window,dedicatedworker
+// META: global=window
 // META: variant=?cpu
 // META: variant=?gpu
 // META: variant=?npu

webnn/conformance_tests/gru.https.any.js

@@ -1,5 +1,5 @@
 // META: title=test WebNN API gru operation
-// META: global=window,dedicatedworker
+// META: global=window
 // META: variant=?cpu
 // META: variant=?gpu
 // META: variant=?npu

webnn/conformance_tests/gru_cell.https.any.js

@@ -1,5 +1,5 @@
 // META: title=test WebNN API gruCell operation
-// META: global=window,dedicatedworker
+// META: global=window
 // META: variant=?cpu
 // META: variant=?gpu
 // META: variant=?npu

webnn/conformance_tests/hard_sigmoid.https.any.js

@@ -1,5 +1,5 @@
 // META: title=test WebNN API hardSigmoid operation
-// META: global=window,dedicatedworker
+// META: global=window
 // META: variant=?cpu
 // META: variant=?gpu
 // META: variant=?npu

webnn/conformance_tests/hard_swish.https.any.js

@@ -1,5 +1,5 @@
 // META: title=test WebNN API tanh operation
-// META: global=window,dedicatedworker
+// META: global=window
 // META: variant=?cpu
 // META: variant=?gpu
 // META: variant=?npu

webnn/conformance_tests/instance_normalization.https.any.js

@@ -1,5 +1,5 @@
 // META: title=test WebNN API instanceNormalization operation
-// META: global=window,dedicatedworker
+// META: global=window
 // META: variant=?cpu
 // META: variant=?gpu
 // META: variant=?npu

webnn/conformance_tests/layer_normalization.https.any.js

@@ -1,5 +1,5 @@
 // META: title=test WebNN API layerNormalization operation
-// META: global=window,dedicatedworker
+// META: global=window
 // META: variant=?cpu
 // META: variant=?gpu
 // META: variant=?npu

webnn/conformance_tests/leaky_relu.https.any.js

@@ -1,5 +1,5 @@
 // META: title=test WebNN API leakyRelu operation
-// META: global=window,dedicatedworker
+// META: global=window
 // META: variant=?cpu
 // META: variant=?gpu
 // META: variant=?npu

webnn/conformance_tests/lesser.https.any.js

@@ -1,5 +1,5 @@
 // META: title=test WebNN API element-wise lesser operation
-// META: global=window,dedicatedworker
+// META: global=window
 // META: variant=?cpu
 // META: variant=?gpu
 // META: variant=?npu

webnn/conformance_tests/lesser_or_equal.https.any.js

@@ -1,5 +1,5 @@
 // META: title=test WebNN API element-wise lesserOrEqual operation
-// META: global=window,dedicatedworker
+// META: global=window
 // META: variant=?cpu
 // META: variant=?gpu
 // META: variant=?npu

webnn/conformance_tests/linear.https.any.js

@@ -1,5 +1,5 @@
 // META: title=test WebNN API linear operation
-// META: global=window,dedicatedworker
+// META: global=window
 // META: variant=?cpu
 // META: variant=?gpu
 // META: variant=?npu

webnn/conformance_tests/log.https.any.js

@@ -1,5 +1,5 @@
 // META: title=test WebNN API element-wise log operation
-// META: global=window,dedicatedworker
+// META: global=window
 // META: variant=?cpu
 // META: variant=?gpu
 // META: variant=?npu

webnn/conformance_tests/logical_and.https.any.js

@@ -1,5 +1,5 @@
 // META: title=test WebNN API element-wise logicalAnd operation
-// META: global=window,dedicatedworker
+// META: global=window
 // META: variant=?cpu
 // META: variant=?gpu
 // META: variant=?npu

webnn/conformance_tests/logical_not.https.any.js

@@ -1,5 +1,5 @@
 // META: title=test WebNN API element-wise logicalNot operation
-// META: global=window,dedicatedworker
+// META: global=window
 // META: variant=?cpu
 // META: variant=?gpu
 // META: variant=?npu

webnn/conformance_tests/logical_or.https.any.js

@@ -1,5 +1,5 @@
 // META: title=test WebNN API element-wise logicalOr operation
-// META: global=window,dedicatedworker
+// META: global=window
 // META: variant=?cpu
 // META: variant=?gpu
 // META: variant=?npu

webnn/conformance_tests/logical_xor.https.any.js

@@ -1,5 +1,5 @@
 // META: title=test WebNN API element-wise logicalXor operation
-// META: global=window,dedicatedworker
+// META: global=window
 // META: variant=?cpu
 // META: variant=?gpu
 // META: variant=?npu

webnn/conformance_tests/lstm.https.any.js

@@ -1,5 +1,5 @@
 // META: title=test WebNN API lstm operation
-// META: global=window,dedicatedworker
+// META: global=window
 // META: variant=?cpu
 // META: variant=?gpu
 // META: variant=?npu

webnn/conformance_tests/lstm_cell.https.any.js

@@ -1,5 +1,5 @@
 // META: title=test WebNN API lstmCell operation
-// META: global=window,dedicatedworker
+// META: global=window
 // META: variant=?cpu
 // META: variant=?gpu
 // META: variant=?npu

webnn/conformance_tests/matmul.https.any.js

@@ -1,5 +1,5 @@
 // META: title=test WebNN API matmul operation
-// META: global=window,dedicatedworker
+// META: global=window
 // META: variant=?cpu
 // META: variant=?gpu
 // META: variant=?npu

webnn/conformance_tests/max.https.any.js

@@ -1,5 +1,5 @@
 // META: title=test WebNN API element-wise max operation
-// META: global=window,dedicatedworker
+// META: global=window
 // META: variant=?cpu
 // META: variant=?gpu
 // META: variant=?npu

webnn/conformance_tests/min.https.any.js

@@ -1,5 +1,5 @@
 // META: title=test WebNN API element-wise min operation
-// META: global=window,dedicatedworker
+// META: global=window
 // META: variant=?cpu
 // META: variant=?gpu
 // META: variant=?npu

webnn/conformance_tests/mul.https.any.js

@@ -1,5 +1,5 @@
 // META: title=test WebNN API element-wise mul operation
-// META: global=window,dedicatedworker
+// META: global=window
 // META: variant=?cpu
 // META: variant=?gpu
 // META: variant=?npu

webnn/conformance_tests/neg.https.any.js

@@ -1,5 +1,5 @@
 // META: title=test WebNN API element-wise neg operation
-// META: global=window,dedicatedworker
+// META: global=window
 // META: variant=?cpu
 // META: variant=?gpu
 // META: variant=?npu