From 27369505155503c838f00ddf538a9a1503c0489b Mon Sep 17 00:00:00 2001 From: Michael Peyton Jones Date: Tue, 30 Jan 2024 16:20:52 +0000 Subject: [PATCH 1/8] Turn off tasty-rerun --- .github/workflows/test.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f8619c683b..64fa68c28b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -107,8 +107,12 @@ jobs: # run the tests without parallelism, otherwise tasty will attempt to run # all functional test cases simultaneously which causes way too many hls # instances to be spun up for the poor github actions runner to handle + # TODO(MPJ): I removed the tasty-rerun options since it seemed to be + # triggering when it shouldn't. We should try and figure out why or + # just drop it entirely. + # --rerun-update --rerun-filter failures,exceptions run: | - echo "TEST_OPTS=-j1 --rerun-update --rerun-filter failures,exceptions" >> $GITHUB_ENV + echo "TEST_OPTS=-j1 " >> $GITHUB_ENV - name: Cache test log between attempts of the same run uses: actions/cache@v3 From 07e85e9a72b55426e8ba5ee27ae9875b5d0b0241 Mon Sep 17 00:00:00 2001 From: Michael Peyton Jones Date: Mon, 29 Jan 2024 15:32:08 +0000 Subject: [PATCH 2/8] Eval plugin test fixups --- plugins/hls-eval-plugin/test/Main.hs | 6 ++++-- .../test/testdata/T14.ghc98.expected.hs | 8 ++++++++ .../test/testdata/TFlags.ghc98.expected.hs | 4 +--- .../test/testdata/TPropertyError.ghc98.expected.hs | 13 +++++++++++++ 4 files changed, 26 insertions(+), 5 deletions(-) create mode 100644 plugins/hls-eval-plugin/test/testdata/T14.ghc98.expected.hs create mode 100644 plugins/hls-eval-plugin/test/testdata/TPropertyError.ghc98.expected.hs diff --git a/plugins/hls-eval-plugin/test/Main.hs b/plugins/hls-eval-plugin/test/Main.hs index 3b34c1130f..0f7c64e3f5 100644 --- a/plugins/hls-eval-plugin/test/Main.hs +++ b/plugins/hls-eval-plugin/test/Main.hs @@ -131,7 +131,7 @@ tests = , goldenWithEvalAndFs "Transitive local dependency" (FS.directProjectMulti ["TTransitive.hs", "TLocalImport.hs", "Util.hs"]) "TTransitive" "hs" -- , goldenWithEval "Local Modules can be imported in a test" "TLocalImportInTest" "hs" , goldenWithEval "Setting language option TupleSections" "TLanguageOptionsTupleSections" "hs" - , goldenWithEval' ":set accepts ghci flags" "TFlags" "hs" (if ghcVersion >= GHC92 then "ghc98.expected" else if ghcVersion >= GHC92 then "ghc92.expected" else "expected") + , goldenWithEval' ":set accepts ghci flags" "TFlags" "hs" (if ghcVersion >= GHC98 then "ghc98.expected" else if ghcVersion >= GHC92 then "ghc92.expected" else "expected") , testCase ":set -fprint-explicit-foralls works" $ do evalInFile "T8.hs" "-- >>> :t id" "-- id :: a -> a" evalInFile "T8.hs" "-- >>> :set -fprint-explicit-foralls\n-- >>> :t id" @@ -142,7 +142,9 @@ tests = , goldenWithEval "IO expressions are supported, stdout/stderr output is ignored" "TIO" "hs" , goldenWithEvalAndFs "Property checking" cabalProjectFS "TProperty" "hs" , goldenWithEvalAndFs' "Property checking with exception" cabalProjectFS "TPropertyError" "hs" ( - if ghcVersion >= GHC96 then + if ghcVersion >= GHC98 then + "ghc98.expected" + else if ghcVersion >= GHC96 then "ghc96.expected" else if ghcVersion >= GHC94 && hostOS == Windows then "windows-ghc94.expected" diff --git a/plugins/hls-eval-plugin/test/testdata/T14.ghc98.expected.hs b/plugins/hls-eval-plugin/test/testdata/T14.ghc98.expected.hs new file mode 100644 index 0000000000..61ee830fa1 --- /dev/null +++ b/plugins/hls-eval-plugin/test/testdata/T14.ghc98.expected.hs @@ -0,0 +1,8 @@ +{-# LANGUAGE TypeApplications #-} +module T14 where + +foo :: Show a => a -> String +foo = show + +-- >>> :type foo @Int +-- foo @Int :: Show Int => Int -> String diff --git a/plugins/hls-eval-plugin/test/testdata/TFlags.ghc98.expected.hs b/plugins/hls-eval-plugin/test/testdata/TFlags.ghc98.expected.hs index 2c8e0ef92a..2e4de4c0b7 100644 --- a/plugins/hls-eval-plugin/test/testdata/TFlags.ghc98.expected.hs +++ b/plugins/hls-eval-plugin/test/testdata/TFlags.ghc98.expected.hs @@ -21,7 +21,6 @@ Options apply only in the section where they are defined (unless they are in the >>> class L a b c Too many parameters for class `L' -(Enable MultiParamTypeClasses to allow multi-parameter classes) In the class declaration for `L' -} @@ -33,7 +32,6 @@ Not set yet: >>> class D No parameters for class `D' -(Enable MultiParamTypeClasses to allow no-parameter classes) In the class declaration for `D' Now it works: @@ -57,7 +55,7 @@ It still works {- Invalid option/flags are reported, but valid ones will be reflected >>> :set -XRank2Types -XAbsent -XDatatypeContexts -XWrong -fprint-nothing-at-all -: warning: +: warning: [GHC-53692] [-Wdeprecated-flags (in -Wdefault)] -XDatatypeContexts is deprecated: It was widely considered a misfeature, and has been removed from the Haskell language. Some flags have not been recognized: -XAbsent, -XWrong, -fprint-nothing-at-all diff --git a/plugins/hls-eval-plugin/test/testdata/TPropertyError.ghc98.expected.hs b/plugins/hls-eval-plugin/test/testdata/TPropertyError.ghc98.expected.hs new file mode 100644 index 0000000000..9fc0848785 --- /dev/null +++ b/plugins/hls-eval-plugin/test/testdata/TPropertyError.ghc98.expected.hs @@ -0,0 +1,13 @@ +-- Support for property checking +module TProperty where + +-- prop> \(l::[Bool]) -> head l +-- *** Failed! (after 1 test): +-- Exception: +-- Prelude.head: empty list +-- CallStack (from HasCallStack): +-- error, called at libraries/base/GHC/List.hs:1782:3 in base:GHC.List +-- errorEmptyList, called at libraries/base/GHC/List.hs:89:11 in base:GHC.List +-- badHead, called at libraries/base/GHC/List.hs:83:28 in base:GHC.List +-- head, called at :1:27 in interactive:Ghci2 +-- [] From a5760164bb6f7460dffd65a77b9e6ae15ec7c00c Mon Sep 17 00:00:00 2001 From: Michael Peyton Jones Date: Wed, 31 Jan 2024 10:53:44 +0000 Subject: [PATCH 3/8] Revert "Turn off tasty-rerun" This reverts commit 27369505155503c838f00ddf538a9a1503c0489b. --- .github/workflows/test.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 64fa68c28b..f8619c683b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -107,12 +107,8 @@ jobs: # run the tests without parallelism, otherwise tasty will attempt to run # all functional test cases simultaneously which causes way too many hls # instances to be spun up for the poor github actions runner to handle - # TODO(MPJ): I removed the tasty-rerun options since it seemed to be - # triggering when it shouldn't. We should try and figure out why or - # just drop it entirely. - # --rerun-update --rerun-filter failures,exceptions run: | - echo "TEST_OPTS=-j1 " >> $GITHUB_ENV + echo "TEST_OPTS=-j1 --rerun-update --rerun-filter failures,exceptions" >> $GITHUB_ENV - name: Cache test log between attempts of the same run uses: actions/cache@v3 From 86412324cc6f8ee895915e613645f5a31ad36694 Mon Sep 17 00:00:00 2001 From: Michael Peyton Jones Date: Wed, 31 Jan 2024 10:54:06 +0000 Subject: [PATCH 4/8] Instead just turn off the test log caching --- .github/workflows/test.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f8619c683b..2160e21b19 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -110,14 +110,6 @@ jobs: run: | echo "TEST_OPTS=-j1 --rerun-update --rerun-filter failures,exceptions" >> $GITHUB_ENV - - name: Cache test log between attempts of the same run - uses: actions/cache@v3 - env: - cache-name: cache-test-log - with: - path: "**/.tasty-rerun-log*" - key: v1-${{ runner.os }}-${{ matrix.ghc }}-test-log-${{ github.sha }} - - if: matrix.test name: Test hls-graph run: cabal test hls-graph --test-options="$TEST_OPTS" From 7f7f36355f1d869d6bce3088b1ffaab0af8c5752 Mon Sep 17 00:00:00 2001 From: Michael Peyton Jones Date: Wed, 31 Jan 2024 11:45:59 +0000 Subject: [PATCH 5/8] Try this --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2160e21b19..0d1cf54755 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -108,7 +108,7 @@ jobs: # all functional test cases simultaneously which causes way too many hls # instances to be spun up for the poor github actions runner to handle run: | - echo "TEST_OPTS=-j1 --rerun-update --rerun-filter failures,exceptions" >> $GITHUB_ENV + echo "TEST_OPTS=-j1 --rerun-update --rerun-filter failures,exceptions,new" >> $GITHUB_ENV - if: matrix.test name: Test hls-graph From 111c040ddc346be17b49f9fe7e93f8e73261b347 Mon Sep 17 00:00:00 2001 From: Michael Peyton Jones Date: Wed, 31 Jan 2024 13:43:09 +0000 Subject: [PATCH 6/8] More eval plugin --- plugins/hls-eval-plugin/test/Main.hs | 4 +--- .../TPropertyError.windows-ghc94.expected.hs | 13 ------------- 2 files changed, 1 insertion(+), 16 deletions(-) delete mode 100644 plugins/hls-eval-plugin/test/testdata/TPropertyError.windows-ghc94.expected.hs diff --git a/plugins/hls-eval-plugin/test/Main.hs b/plugins/hls-eval-plugin/test/Main.hs index 0f7c64e3f5..fb67a81062 100644 --- a/plugins/hls-eval-plugin/test/Main.hs +++ b/plugins/hls-eval-plugin/test/Main.hs @@ -141,13 +141,11 @@ tests = , goldenWithEval "The default language extensions for the eval plugin are the same as those for ghci" "TSameDefaultLanguageExtensionsAsGhci" "hs" , goldenWithEval "IO expressions are supported, stdout/stderr output is ignored" "TIO" "hs" , goldenWithEvalAndFs "Property checking" cabalProjectFS "TProperty" "hs" - , goldenWithEvalAndFs' "Property checking with exception" cabalProjectFS "TPropertyError" "hs" ( + , knownBrokenInEnv [HostOS Windows] "The output has path separators in it, which on Windows look different. Just skip it there" $ goldenWithEvalAndFs' "Property checking with exception" cabalProjectFS "TPropertyError" "hs" ( if ghcVersion >= GHC98 then "ghc98.expected" else if ghcVersion >= GHC96 then "ghc96.expected" - else if ghcVersion >= GHC94 && hostOS == Windows then - "windows-ghc94.expected" else if ghcVersion >= GHC94 then "ghc94.expected" else diff --git a/plugins/hls-eval-plugin/test/testdata/TPropertyError.windows-ghc94.expected.hs b/plugins/hls-eval-plugin/test/testdata/TPropertyError.windows-ghc94.expected.hs deleted file mode 100644 index 6c7813d776..0000000000 --- a/plugins/hls-eval-plugin/test/testdata/TPropertyError.windows-ghc94.expected.hs +++ /dev/null @@ -1,13 +0,0 @@ --- Support for property checking -module TProperty where - --- prop> \(l::[Bool]) -> head l --- *** Failed! (after 1 test): --- Exception: --- Prelude.head: empty list --- CallStack (from HasCallStack): --- error, called at libraries\base\GHC\List.hs:1646:3 in base:GHC.List --- errorEmptyList, called at libraries\base\GHC\List.hs:85:11 in base:GHC.List --- badHead, called at libraries\base\GHC\List.hs:81:28 in base:GHC.List --- head, called at :1:27 in interactive:Ghci2 --- [] From 43ad4623303d8288726f17c74a12decdf91768c3 Mon Sep 17 00:00:00 2001 From: Michael Peyton Jones Date: Wed, 31 Jan 2024 18:40:37 +0000 Subject: [PATCH 7/8] Add a comment --- .github/workflows/test.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0d1cf54755..00081642d9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -107,6 +107,10 @@ jobs: # run the tests without parallelism, otherwise tasty will attempt to run # all functional test cases simultaneously which causes way too many hls # instances to be spun up for the poor github actions runner to handle + # + # See https://github.com/ocharles/tasty-rerun/issues/22 for why we need + # to include 'new' in the filters, since many of our test suites are in the + # same package. run: | echo "TEST_OPTS=-j1 --rerun-update --rerun-filter failures,exceptions,new" >> $GITHUB_ENV From 2eead22053aa1fc6cddf1c3a176bed41d8667b23 Mon Sep 17 00:00:00 2001 From: Michael Peyton Jones Date: Thu, 1 Feb 2024 09:06:27 +0000 Subject: [PATCH 8/8] Fix flag name --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 00081642d9..11a634389e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -232,7 +232,7 @@ jobs: ## version needs to be limited since the tests depend on cabal-fmt which only builds using specific ghc versions - if: matrix.test && matrix.ghc == '9.2' name: Test hls-cabal-fmt-plugin test suite - run: cabal test hls-cabal-fmt-plugin-tests --flag=isolateTests --test-options="$TEST_OPTS" || cabal test hls-cabal-fmt-plugin-tests --flag=isolateTests --test-options="$TEST_OPTS" + run: cabal test hls-cabal-fmt-plugin-tests --flag=isolateCabalfmtTests --test-options="$TEST_OPTS" || cabal test hls-cabal-fmt-plugin-tests --flag=isolateCabalfmtTests --test-options="$TEST_OPTS" - if: matrix.test name: Test hls-cabal-plugin test suite