From 8be51b79c5a46b58a3366204f4c9c8fb1a2de613 Mon Sep 17 00:00:00 2001 From: ofekshenawa Date: Thu, 21 Nov 2024 15:04:52 +0200 Subject: [PATCH 1/9] Create workflow that tests go-redis against docker --- .github/workflows/test-against-docker.yml | 120 ++++++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 .github/workflows/test-against-docker.yml diff --git a/.github/workflows/test-against-docker.yml b/.github/workflows/test-against-docker.yml new file mode 100644 index 000000000..4d0ec7f33 --- /dev/null +++ b/.github/workflows/test-against-docker.yml @@ -0,0 +1,120 @@ +--- + + name: Build and Test using containerized environment + + on: + push: + paths-ignore: + - 'docs/**' + - '**/*.md' + - '**/*.rst' + branches: + - master + - '[0-9].*' + pull_request: + branches: + - master + - '[0-9].*' + schedule: + - cron: '0 1 * * *' # nightly build + + jobs: + + build: + name: Build and Test + runs-on: ubuntu-latest + env: + REDIS_ENV_WORK_DIR: ${{ github.workspace }}/redis-env-work + REDIS_ENV_CONF_DIR: ${{ github.workspace }}/src/test/resources/env + CLIENT_LIBS_IMAGE_PREFIX: "redislabs/client-libs-test" + strategy: + fail-fast: false + matrix: + redis_version: + - "7.4.1" + - "7.2.6" + - "6.2.16" + go_version: + - "1.19.x" + - "1.20.x" + - "1.21.x" + steps: + - name: Set up ${{ matrix.go_version }} + uses: actions/setup-go@v5 + with: + go-version: ${{ matrix.go_version }} + + - name: Checkout code + uses: actions/checkout@v4 + + # Set up Docker Compose environment + - name: Set up Docker Compose environment + run: | + mkdir -m 777 $REDIS_ENV_WORK_DIR + export REDIS_VERSION="${{ matrix.redis_version }}" + export COMPOSE_ENV_FILES="src/test/resources/env/.env" + if [[ "${{ matrix.redis_version }}" == "6.2.16" ]]; then + COMPOSE_ENV_FILES+=",src/test/resources/env/.env.v${{ matrix.redis_version }}" + fi + docker compose -f src/test/resources/env/docker-compose.yml up -d + + - name: Run tests + env: + GO_VERSION: ${{ matrix.go_version }} + run: | + set -e + GO_MOD_DIRS=$(find . -type d -name go.mod | xargs -n 1 dirname) + for dir in $GO_MOD_DIRS; do + if echo "$dir" | grep -q "./example" && [ "$GO_VERSION" = "19" ]; then + echo "Skipping go test in $dir due to Go version 1.19 and dir contains ./example" + continue + fi + echo "Running tests in $dir" + ( + cd "$dir" + go mod tidy -compat=1.18 + go test ./... -short -race + go test ./... -run=NONE -bench=. -benchmem + env GOOS=linux GOARCH=386 go test ./... + go test -coverprofile=coverage.txt -covermode=atomic ./... + go vet + ) + done + + - name: Build custom vet tool + run: | + cd internal/customvet && go build . + go vet -vettool ./internal/customvet/customvet + + # Collect logs on failure + - name: Collect logs on failure + if: failure() # This runs only if the previous steps failed + run: | + echo "Collecting logs from $WORK_DIR..." + ls -la $REDIS_ENV_WORK_DIR + # Upload logs as artifacts + - name: Upload logs on failure + if: failure() + uses: actions/upload-artifact@v3 + with: + name: redis-env-work-logs + path: ${{ env.REDIS_ENV_WORK_DIR }} + # Bring down the Docker Compose test environment + - name: Tear down Docker Compose environment + if: always() + run: | + docker compose $COMPOSE_ENV_FILES -f src/test/resources/env/docker-compose.yml down + continue-on-error: true + # Upload code coverage + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v4 + with: + fail_ci_if_error: false + token: ${{ secrets.CODECOV_TOKEN }} + - name: Upload test results to Codecov + if: ${{ github.event_name == 'schedule' || (github.event_name == 'push') || github.event_name == 'workflow_dispatch'}} + uses: codecov/test-results-action@v1 + with: + fail_ci_if_error: false + files: ./target/surefire-reports/TEST* + token: ${{ secrets.CODECOV_TOKEN }} \ No newline at end of file From 1dcda5e6a639d210e106563d9c68247f56a346a3 Mon Sep 17 00:00:00 2001 From: ofekshenawa Date: Sun, 15 Dec 2024 16:17:48 +0200 Subject: [PATCH 2/9] Add docker compose file --- .github/docker-compose.yml | 22 ++++ .github/workflows/build.yml | 41 +++++++- .github/workflows/test-against-docker.yml | 120 ---------------------- 3 files changed, 62 insertions(+), 121 deletions(-) create mode 100644 .github/docker-compose.yml delete mode 100644 .github/workflows/test-against-docker.yml diff --git a/.github/docker-compose.yml b/.github/docker-compose.yml new file mode 100644 index 000000000..4bc5949ba --- /dev/null +++ b/.github/docker-compose.yml @@ -0,0 +1,22 @@ +--- + +services: + + redis: + image: ${CLIENT_LIBS_TEST_IMAGE:-redislabs/client-libs-test:8.0-M01} + container_name: redis-standalone + environment: + - TLS_ENABLED=yes + - REDIS_CLUSTER=no + - PORT=6379 + - TLS_PORT=6666 + command: ${REDIS_EXTRA_ARGS:---enable-debug-command yes --enable-module-command yes --tls-auth-clients optional --save ""} + ports: + - 6379:6379 + - 6380:6379 + - 6666:6666 # TLS port + volumes: + - "./dockers/standalone:/redis/work" + profiles: + - standalone + - all \ No newline at end of file diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5007423a4..cc80fc3b9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -42,4 +42,43 @@ jobs: uses: codecov/codecov-action@v4 with: files: coverage.txt - token: ${{ secrets.CODECOV_TOKEN }} \ No newline at end of file + token: ${{ secrets.CODECOV_TOKEN }} + + test-redis-ce: + name: test-redis-ce + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + redis_version: + - "8.0-M01" + # - "7.4.1" + # - "7.2.6" + # - "6.2.16" + go-version: + - "1.19.x" + - "1.20.x" + - "1.21.x" + + steps: + - name: Set up ${{ matrix.go-version }} + uses: actions/setup-go@v5 + with: + go-version: ${{ matrix.go-version }} + + - name: Checkout code + uses: actions/checkout@v4 + + # Set up Docker Compose environment + - name: Set up Docker Compose environment + run: | + docker compose -f src/test/resources/env/docker-compose.yml up -d + + - name: Run tests + run: | + go test \ + --ginkgo.skip-file="ring_test.go" \ + --ginkgo.skip-file="sentinel_test.go" \ + --ginkgo.skip-file="osscluster_test.go" \ + --ginkgo.skip-file="pubsub_test.go" \ + --ginkgo.skip-file="gears_commands_test.go" \ \ No newline at end of file diff --git a/.github/workflows/test-against-docker.yml b/.github/workflows/test-against-docker.yml deleted file mode 100644 index 4d0ec7f33..000000000 --- a/.github/workflows/test-against-docker.yml +++ /dev/null @@ -1,120 +0,0 @@ ---- - - name: Build and Test using containerized environment - - on: - push: - paths-ignore: - - 'docs/**' - - '**/*.md' - - '**/*.rst' - branches: - - master - - '[0-9].*' - pull_request: - branches: - - master - - '[0-9].*' - schedule: - - cron: '0 1 * * *' # nightly build - - jobs: - - build: - name: Build and Test - runs-on: ubuntu-latest - env: - REDIS_ENV_WORK_DIR: ${{ github.workspace }}/redis-env-work - REDIS_ENV_CONF_DIR: ${{ github.workspace }}/src/test/resources/env - CLIENT_LIBS_IMAGE_PREFIX: "redislabs/client-libs-test" - strategy: - fail-fast: false - matrix: - redis_version: - - "7.4.1" - - "7.2.6" - - "6.2.16" - go_version: - - "1.19.x" - - "1.20.x" - - "1.21.x" - steps: - - name: Set up ${{ matrix.go_version }} - uses: actions/setup-go@v5 - with: - go-version: ${{ matrix.go_version }} - - - name: Checkout code - uses: actions/checkout@v4 - - # Set up Docker Compose environment - - name: Set up Docker Compose environment - run: | - mkdir -m 777 $REDIS_ENV_WORK_DIR - export REDIS_VERSION="${{ matrix.redis_version }}" - export COMPOSE_ENV_FILES="src/test/resources/env/.env" - if [[ "${{ matrix.redis_version }}" == "6.2.16" ]]; then - COMPOSE_ENV_FILES+=",src/test/resources/env/.env.v${{ matrix.redis_version }}" - fi - docker compose -f src/test/resources/env/docker-compose.yml up -d - - - name: Run tests - env: - GO_VERSION: ${{ matrix.go_version }} - run: | - set -e - GO_MOD_DIRS=$(find . -type d -name go.mod | xargs -n 1 dirname) - for dir in $GO_MOD_DIRS; do - if echo "$dir" | grep -q "./example" && [ "$GO_VERSION" = "19" ]; then - echo "Skipping go test in $dir due to Go version 1.19 and dir contains ./example" - continue - fi - echo "Running tests in $dir" - ( - cd "$dir" - go mod tidy -compat=1.18 - go test ./... -short -race - go test ./... -run=NONE -bench=. -benchmem - env GOOS=linux GOARCH=386 go test ./... - go test -coverprofile=coverage.txt -covermode=atomic ./... - go vet - ) - done - - - name: Build custom vet tool - run: | - cd internal/customvet && go build . - go vet -vettool ./internal/customvet/customvet - - # Collect logs on failure - - name: Collect logs on failure - if: failure() # This runs only if the previous steps failed - run: | - echo "Collecting logs from $WORK_DIR..." - ls -la $REDIS_ENV_WORK_DIR - # Upload logs as artifacts - - name: Upload logs on failure - if: failure() - uses: actions/upload-artifact@v3 - with: - name: redis-env-work-logs - path: ${{ env.REDIS_ENV_WORK_DIR }} - # Bring down the Docker Compose test environment - - name: Tear down Docker Compose environment - if: always() - run: | - docker compose $COMPOSE_ENV_FILES -f src/test/resources/env/docker-compose.yml down - continue-on-error: true - # Upload code coverage - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v4 - with: - fail_ci_if_error: false - token: ${{ secrets.CODECOV_TOKEN }} - - name: Upload test results to Codecov - if: ${{ github.event_name == 'schedule' || (github.event_name == 'push') || github.event_name == 'workflow_dispatch'}} - uses: codecov/test-results-action@v1 - with: - fail_ci_if_error: false - files: ./target/surefire-reports/TEST* - token: ${{ secrets.CODECOV_TOKEN }} \ No newline at end of file From e4f26476a8b707332225c6a1f6f8d74f400ab8c4 Mon Sep 17 00:00:00 2001 From: ofekshenawa Date: Sun, 15 Dec 2024 17:45:37 +0200 Subject: [PATCH 3/9] Add docker compose file --- .github/workflows/build.yml | 8 ++++++-- .github/docker-compose.yml => docker-compose.yml | 5 ++--- main_test.go | 6 ++++-- 3 files changed, 12 insertions(+), 7 deletions(-) rename .github/docker-compose.yml => docker-compose.yml (58%) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index cc80fc3b9..592440587 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -72,13 +72,17 @@ jobs: # Set up Docker Compose environment - name: Set up Docker Compose environment run: | - docker compose -f src/test/resources/env/docker-compose.yml up -d + docker compose --profile all up -d --build - name: Run tests + env: + USE_PREBUILT_REDIS: "true" + RE_CLUSTER: "true" run: | go test \ --ginkgo.skip-file="ring_test.go" \ --ginkgo.skip-file="sentinel_test.go" \ --ginkgo.skip-file="osscluster_test.go" \ --ginkgo.skip-file="pubsub_test.go" \ - --ginkgo.skip-file="gears_commands_test.go" \ \ No newline at end of file + --ginkgo.skip-file="gears_commands_test.go" \ + --ginkgo.label-filter='!NonRedisEnterprise' \ No newline at end of file diff --git a/.github/docker-compose.yml b/docker-compose.yml similarity index 58% rename from .github/docker-compose.yml rename to docker-compose.yml index 4bc5949ba..fc6775d92 100644 --- a/.github/docker-compose.yml +++ b/docker-compose.yml @@ -3,14 +3,13 @@ services: redis: - image: ${CLIENT_LIBS_TEST_IMAGE:-redislabs/client-libs-test:8.0-M01} + image: redislabs/client-libs-test:8.0-M01 container_name: redis-standalone environment: - - TLS_ENABLED=yes - REDIS_CLUSTER=no - PORT=6379 - TLS_PORT=6666 - command: ${REDIS_EXTRA_ARGS:---enable-debug-command yes --enable-module-command yes --tls-auth-clients optional --save ""} + command: --enable-module-command yes ports: - 6379:6379 - 6380:6379 diff --git a/main_test.go b/main_test.go index 19e944446..38c6173ee 100644 --- a/main_test.go +++ b/main_test.go @@ -66,6 +66,7 @@ var cluster = &clusterScenario{ } var RECluster = false +var USE_PREBUILT_REDIS = false func registerProcess(port string, p *redisProcess) { if processes == nil { @@ -82,8 +83,9 @@ var _ = BeforeSuite(func() { } var err error RECluster, _ = strconv.ParseBool(os.Getenv("RE_CLUSTER")) - - if !RECluster { + USE_PREBUILT_REDIS, _ = strconv.ParseBool(os.Getenv("USE_PREBUILT_REDIS")) + // panic("RECluster: " + strconv.FormatBool(RECluster) + " USE_PREBUILT_REDIS: " + strconv.FormatBool(USE_PREBUILT_REDIS)) + if !RECluster || !USE_PREBUILT_REDIS { redisMain, err = startRedis(redisPort) Expect(err).NotTo(HaveOccurred()) From 0a1ee8f2a5f38b1ad703612340de908e9d8a5c3f Mon Sep 17 00:00:00 2001 From: ofekshenawa Date: Mon, 16 Dec 2024 10:30:05 +0200 Subject: [PATCH 4/9] Change command in docker compose --- docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index fc6775d92..2c782ec3e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,13 +3,13 @@ services: redis: - image: redislabs/client-libs-test:8.0-M01 + image: redislabs/client-libs-test:8.0-M02 container_name: redis-standalone environment: - REDIS_CLUSTER=no - PORT=6379 - TLS_PORT=6666 - command: --enable-module-command yes + command: --enable-module-command yes ports: - 6379:6379 - 6380:6379 From 4247e6253cbcd5343e20c2a9f969e900e2bfb6c9 Mon Sep 17 00:00:00 2001 From: ofekshenawa Date: Mon, 16 Dec 2024 10:36:44 +0200 Subject: [PATCH 5/9] Load modules locally --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 2c782ec3e..f16aca70d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,7 +9,7 @@ services: - REDIS_CLUSTER=no - PORT=6379 - TLS_PORT=6666 - command: --enable-module-command yes + command: --loadmodule /usr/local/lib/redis/modules/redisbloom.so --loadmodule /usr/local/lib/redis/modules/redisearch.so --loadmodule /usr/local/lib/redis/modules/redistimeseries.so --loadmodule /usr/local/lib/redis/modules/rejson.so ports: - 6379:6379 - 6380:6379 From 1f9608051f494d9c5247362614c1d3c1fa024f60 Mon Sep 17 00:00:00 2001 From: ofekshenawa Date: Mon, 16 Dec 2024 10:41:32 +0200 Subject: [PATCH 6/9] test varios redis versions --- .github/workflows/build.yml | 8 ++++---- docker-compose.yml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6f0ba9750..e58e039ca 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -52,9 +52,9 @@ jobs: matrix: redis_version: - "8.0-M01" - # - "7.4.1" - # - "7.2.6" - # - "6.2.16" + - "7.4.1" + - "7.2.6" + - "6.2.16" go-version: - "1.19.x" - "1.20.x" @@ -85,4 +85,4 @@ jobs: --ginkgo.skip-file="osscluster_test.go" \ --ginkgo.skip-file="pubsub_test.go" \ --ginkgo.skip-file="gears_commands_test.go" \ - --ginkgo.label-filter='!NonRedisEnterprise' \ No newline at end of file + --ginkgo.label-filter='!NonRedisEnterprise' diff --git a/docker-compose.yml b/docker-compose.yml index f16aca70d..f76f1101b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -18,4 +18,4 @@ services: - "./dockers/standalone:/redis/work" profiles: - standalone - - all \ No newline at end of file + - all From 676399024a10a136e36fd9d11a6d8ead4f49e8e7 Mon Sep 17 00:00:00 2001 From: ofekshenawa Date: Mon, 16 Dec 2024 10:56:17 +0200 Subject: [PATCH 7/9] add env var to test-redis-enterprise action --- .github/workflows/test-redis-enterprise.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test-redis-enterprise.yml b/.github/workflows/test-redis-enterprise.yml index 940f0eae7..625eab6df 100644 --- a/.github/workflows/test-redis-enterprise.yml +++ b/.github/workflows/test-redis-enterprise.yml @@ -47,6 +47,7 @@ jobs: - name: Test env: RE_CLUSTER: "1" + USE_PREBUILT_REDIS: "1" run: | go test \ --ginkgo.skip-file="ring_test.go" \ From 545e950f35bb0ee3d8acb594850336940b9e4138 Mon Sep 17 00:00:00 2001 From: ofekshenawa Date: Mon, 16 Dec 2024 16:21:17 +0200 Subject: [PATCH 8/9] cleaning code --- .github/workflows/build.yml | 4 ++-- .github/workflows/test-redis-enterprise.yml | 2 +- docker-compose.yml | 2 +- main_test.go | 7 +++---- 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e58e039ca..7578e962e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -72,11 +72,11 @@ jobs: # Set up Docker Compose environment - name: Set up Docker Compose environment run: | - docker compose --profile all up -d --build + docker compose --profile all up -d - name: Run tests env: - USE_PREBUILT_REDIS: "true" + USE_CONTAINERIZED_REDIS: "true" RE_CLUSTER: "true" run: | go test \ diff --git a/.github/workflows/test-redis-enterprise.yml b/.github/workflows/test-redis-enterprise.yml index 625eab6df..1cb36b8d2 100644 --- a/.github/workflows/test-redis-enterprise.yml +++ b/.github/workflows/test-redis-enterprise.yml @@ -47,7 +47,7 @@ jobs: - name: Test env: RE_CLUSTER: "1" - USE_PREBUILT_REDIS: "1" + USE_CONTAINERIZED_REDIS: "1" run: | go test \ --ginkgo.skip-file="ring_test.go" \ diff --git a/docker-compose.yml b/docker-compose.yml index f76f1101b..815019bba 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,7 @@ services: - redis: + redis-stanalone: image: redislabs/client-libs-test:8.0-M02 container_name: redis-standalone environment: diff --git a/main_test.go b/main_test.go index 38c6173ee..44f8e6829 100644 --- a/main_test.go +++ b/main_test.go @@ -66,7 +66,7 @@ var cluster = &clusterScenario{ } var RECluster = false -var USE_PREBUILT_REDIS = false +var USE_CONTAINERIZED_REDIS = false func registerProcess(port string, p *redisProcess) { if processes == nil { @@ -83,9 +83,8 @@ var _ = BeforeSuite(func() { } var err error RECluster, _ = strconv.ParseBool(os.Getenv("RE_CLUSTER")) - USE_PREBUILT_REDIS, _ = strconv.ParseBool(os.Getenv("USE_PREBUILT_REDIS")) - // panic("RECluster: " + strconv.FormatBool(RECluster) + " USE_PREBUILT_REDIS: " + strconv.FormatBool(USE_PREBUILT_REDIS)) - if !RECluster || !USE_PREBUILT_REDIS { + USE_CONTAINERIZED_REDIS, _ = strconv.ParseBool(os.Getenv("USE_CONTAINERIZED_REDIS")) + if !RECluster || !USE_CONTAINERIZED_REDIS { redisMain, err = startRedis(redisPort) Expect(err).NotTo(HaveOccurred()) From 99fc68eaa448ff7e0e2a0b4e7cded0f2a2686d3a Mon Sep 17 00:00:00 2001 From: ofekshenawa Date: Mon, 16 Dec 2024 16:22:21 +0200 Subject: [PATCH 9/9] cleaning code --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 815019bba..a641e4d3b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,7 +15,7 @@ services: - 6380:6379 - 6666:6666 # TLS port volumes: - - "./dockers/standalone:/redis/work" + - "./dockers/redis-standalone:/redis/work" profiles: - standalone - all