Add test ops validation for validation workflows (#1650)

* Add test ops validation

* include workflows

Author: atalman

.github/scripts/validate_test_ops.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+set -eux -o pipefail
+
+retry () {
+    $*  || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*)
+}
+
+# Clone the Pytorch branch
+retry git clone --depth 1 https://github.com/pytorch/pytorch.git
+retry git submodule update --init --recursive
+pushd pytorch
+
+# Run test_ops validation
+export CUDA_LAUNCH_BLOCKING=1
+python3 test/test_ops.py

.github/workflows/validate-binaries.yml

@@ -27,6 +27,11 @@ on:
         default: false
         required: false
         type: boolean
+      include-test-ops:
+        description: 'Include Test Ops tests (only Linux)'
+        default: false
+        required: false
+        type: boolean
   workflow_dispatch:
     inputs:
       os:
@@ -65,6 +70,11 @@ on:
         default: ""
         required: false
         type: string
+      include-test-ops:
+        description: 'Include Test Ops tests (only Linux)'
+        default: false
+        required: false
+        type: boolean
 
 
 jobs:
@@ -94,6 +104,7 @@ jobs:
       torchonly: ${{ inputs.torchonly }}
       version: ${{ inputs.version }}
       release-matrix: ${{ needs.generate-release-matrix.outputs.matrix }}
+      include-test-ops: ${{ inputs.include-test-ops }}
 
   linux-aarch64:
     if:  inputs.os == 'linux-aarch64' || inputs.os == 'all'

.github/workflows/validate-linux-binaries.yml

@@ -27,6 +27,11 @@ on:
         default: ""
         required: false
         type: string
+      include-test-ops:
+        description: 'Include Test Ops tests (only Linux)'
+        default: false
+        required: false
+        type: boolean
   workflow_dispatch:
     inputs:
       channel:
@@ -58,6 +63,11 @@ on:
         default: ""
         required: false
         type: string
+      include-test-ops:
+        description: 'Include Test Ops tests (only Linux)'
+        default: false
+        required: false
+        type: boolean
 
 jobs:
   generate-linux-matrix:
@@ -84,6 +94,7 @@ jobs:
         set -ex
         export ENV_NAME="conda-env-${{ github.run_id }}"
         export TORCH_ONLY=${{ inputs.torchonly }}
+        export INCLUDE_TEST_OPS=${{ inputs.include-test-ops }}
         export RELEASE_VERSION=${{ inputs.version }}
         export TARGET_OS="linux"
         eval "$(conda shell.bash hook)"
@@ -97,3 +108,7 @@ jobs:
 
         # Standart case: Validate binaries
         source ./.github/scripts/validate_binaries.sh
+
+        if [[ ${INCLUDE_TEST_OPS} == 'true' ]]; then
+          source ./.github/scripts/validate_test_ops.sh
+        fi