From 23b96bbfc01985075d8bb3748ea8d74615926d28 Mon Sep 17 00:00:00 2001 From: CryptoNinjaGeek <86222482+CryptoNinjaGeek@users.noreply.github.com> Date: Thu, 5 Oct 2023 11:49:12 +0200 Subject: [PATCH 01/34] Add .circleci/config.yml --- .circleci/config.yml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 .circleci/config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 00000000..4175da6c --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,26 @@ +# Use the latest 2.1 version of CircleCI pipeline process engine. +# See: https://circleci.com/docs/configuration-reference +version: 2.1 + +# Define a job to be invoked later in a workflow. +# See: https://circleci.com/docs/configuration-reference/#jobs +jobs: + say-hello: + # Specify the execution environment. You can specify an image from Docker Hub or use one of our convenience images from CircleCI's Developer Hub. + # See: https://circleci.com/docs/configuration-reference/#executor-job + docker: + - image: cimg/base:stable + # Add steps to the job + # See: https://circleci.com/docs/configuration-reference/#steps + steps: + - checkout + - run: + name: "Say hello" + command: "echo Hello, World!" + +# Orchestrate jobs using workflows +# See: https://circleci.com/docs/configuration-reference/#workflows +workflows: + say-hello-workflow: + jobs: + - say-hello From 505e085d7333ce6e30a14bba5e76bfe7cad99d0c Mon Sep 17 00:00:00 2001 From: Alex Petenchea Date: Fri, 6 Oct 2023 17:44:42 +0300 Subject: [PATCH 02/34] adding circleci configuration --- .circleci/config.yml | 170 +++++++++++++++++++++++++++++---- .github/workflows/build.yaml | 22 +---- .github/workflows/codeql.yaml | 2 +- CONTRIBUTING.md | 4 +- arango/client.py | 20 +++- arango/formatter.py | 6 ++ arango/resolver.py | 58 +++++++++++ starter.sh | 82 ++++++++++++++++ tester.sh | 128 ------------------------- tests/conftest.py | 22 ++++- tests/static/cluster-3.10.conf | 12 +++ tests/static/cluster.conf | 4 +- tests/static/setup.sh | 7 ++ tests/static/single-3.10.conf | 10 ++ tests/static/single.conf | 3 +- tests/test_async.py | 13 ++- tests/test_client.py | 8 +- tests/test_cluster.py | 12 ++- tests/test_database.py | 3 +- tests/test_pregel.py | 13 ++- tests/test_resolver.py | 29 ++++++ 21 files changed, 435 insertions(+), 193 deletions(-) create mode 100755 starter.sh delete mode 100755 tester.sh create mode 100644 tests/static/cluster-3.10.conf create mode 100644 tests/static/setup.sh create mode 100644 tests/static/single-3.10.conf diff --git a/.circleci/config.yml b/.circleci/config.yml index 4175da6c..8f8c641f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,26 +1,164 @@ -# Use the latest 2.1 version of CircleCI pipeline process engine. -# See: https://circleci.com/docs/configuration-reference version: 2.1 -# Define a job to be invoked later in a workflow. -# See: https://circleci.com/docs/configuration-reference/#jobs +parameters: + preview: + type: boolean + default: true + 3-10: + type: boolean + default: true + 3-11: + type: boolean + default: true + jobs: - say-hello: - # Specify the execution environment. You can specify an image from Docker Hub or use one of our convenience images from CircleCI's Developer Hub. - # See: https://circleci.com/docs/configuration-reference/#executor-job + run-tests: + resource_class: small + parameters: + python-version: + type: string + default: "latest" + arangodb-version: + type: string + default: "arangodb:latest" + arangodb-config: + type: string + default: "single.conf" + cluster: + type: boolean + default: false + enterprise: + type: boolean + default: false docker: - - image: cimg/base:stable - # Add steps to the job - # See: https://circleci.com/docs/configuration-reference/#steps + - image: python:<< parameters.python-version >> + command: ["/bin/sh", "-c", "python -m http.server"] + - image: arangodb/<< parameters.arangodb-version >> + environment: + ARANGODB_CONF: << parameters.arangodb-config >> + PROJECT: /root/project + command: + - "/bin/sh" + - "-c" + - > + while ! wget -q -O /dev/null http://localhost:8000/$PROJECT/tests/static/setup.sh; do sleep 1; done && + wget -O - http://localhost:8000/$PROJECT/tests/static/setup.sh | + /bin/sh steps: - checkout - run: - name: "Say hello" - command: "echo Hello, World!" + name: "Install Dependencies" + command: | + pip install -e .[dev] + - run: + name: "Wait for ArangoDB starter" + command: | + wget --quiet --waitretry=1 --tries=120 -O - http://localhost:8528/version + if [ $? -eq 0 ]; then + echo "starter ready" + exit 0 + else + echo "starter not ready, giving up" + exit 1 + fi + - run: + name: "Run pytest" + command: | + mkdir test-results + args=("--log-cli-level=DEBUG" "--host" "localhost" "--junitxml=./test-results/junit.xml") + if [ << parameters.cluster >> = true ]; then + args+=("--cluster" "--port=8529" "--port=8539" "--port=8549") + else + args+=("--port=8529") + fi + if [ << parameters.enterprise >> = true ]; then + args+=("--enterprise") + fi + echo "Running py.test with args: ${args[@]}" + py.test "${args[@]}" + - store_test_results: + path: ./test-results/junit.xml + - store_artifacts: + path: ./test-results -# Orchestrate jobs using workflows -# See: https://circleci.com/docs/configuration-reference/#workflows workflows: - say-hello-workflow: + python-3.8-community-single-3.10: + when: << pipeline.parameters.3-10 >> + jobs: + - run-tests: + name: python-3.8-community-single-3.10 + python-version: "3.8.2" + arangodb-version: "arangodb:3.10.10" + arangodb-config: "single-3.10.conf" + cluster: false + enterprise: false + python-3.8-enterprise-cluster-3.10: + when: << pipeline.parameters.3-10 >> + jobs: + - run-tests: + name: python-3.8-enterprise-cluster-3.10 + python-version: "3.8.2" + arangodb-version: "enterprise:3.10.10" + arangodb-config: "cluster-3.10.conf" + cluster: true + enterprise: true + python-3.10-community-single-3.11: + when: << pipeline.parameters.3-11 >> + jobs: + - run-tests: + name: python-3.10-community-single-3.11 + python-version: "3.10.6" + arangodb-version: "arangodb:3.11.4" + arangodb-config: "single.conf" + cluster: false + enterprise: false + python-3.10-community-cluster-3.11: + when: << pipeline.parameters.3-11 >> + jobs: + - run-tests: + name: python-3.10-community-cluster-3.11 + python-version: "3.10.6" + arangodb-version: "arangodb:3.11.4" + arangodb-config: "cluster.conf" + cluster: true + enterprise: false + python-3.10-enterprise-single-3.11: + when: << pipeline.parameters.3-11 >> + jobs: + - run-tests: + name: python-3.10-enterprise-single-3.11 + python-version: "3.10.6" + arangodb-version: "enterprise:3.11.4" + arangodb-config: "single.conf" + cluster: false + enterprise: true + python-3.10-enterprise-cluster-3.11: + when: << pipeline.parameters.3-11 >> + jobs: + - run-tests: + name: python-3.10-enterprise-cluster-3.11 + python-version: "3.10.6" + arangodb-version: "enterprise:3.11.4" + arangodb-config: "cluster.conf" + cluster: true + enterprise: true + python-latest-enterprise-single-preview: + when: << pipeline.parameters.preview >> + jobs: + - run-tests: + name: python-latest-enterprise-single-preview + python-version: "latest" + arangodb-version: "enterprise-preview:latest" + arangodb-config: "single.conf" + cluster: false + enterprise: true + python-latest-enterprise-cluster-preview: + when: << pipeline.parameters.preview >> jobs: - - say-hello + - run-tests: + name: python-latest-enterprise-cluster-preview + python-version: "latest" + arangodb-version: "enterprise-preview:latest" + arangodb-config: "cluster.conf" + cluster: true + enterprise: true diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index cdd55d25..a542f28b 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -2,7 +2,7 @@ name: Build on: pull_request: - branches: [main, dev] + branches: [main] workflow_dispatch: inputs: debug_enabled: @@ -12,13 +12,9 @@ on: default: false jobs: - build: + docs: runs-on: ubuntu-22.04 - strategy: - matrix: - python-version: ["3.8", "3.9", "3.10", "3.11"] - steps: - name: Checkout repository uses: actions/checkout@v3 @@ -29,7 +25,7 @@ jobs: - name: Create ArangoDB Docker container run: > docker create --name arango -p 8529:8529 -e ARANGO_ROOT_PASSWORD=passwd -v "$(pwd)/tests/static/":/tests/static - arangodb/arangodb:3.10.9 --server.jwt-secret-keyfile=/tests/static/keyfile + arangodb/arangodb:3.11.4 --server.jwt-secret-keyfile=/tests/static/keyfile - name: Start ArangoDB Docker container run: docker start arango @@ -37,7 +33,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: ${{ matrix.python-version }} + python-version: '3.10' - name: Debug with tmate uses: mxschmitt/action-tmate@v3 @@ -49,18 +45,8 @@ jobs: - name: Install dependencies run: pip install .[dev] - - name: Run unit tests - run: py.test --complete --cov=arango --cov-report=xml - - name: Run Sphinx doctest run: python -m sphinx -b doctest docs docs/_build - name: Generate Sphinx HTML run: python -m sphinx -b html -W docs docs/_build - - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v3 - if: matrix.python-version == '3.10' - with: - fail_ci_if_error: false - token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/codeql.yaml b/.github/workflows/codeql.yaml index 23450ff4..b182474f 100644 --- a/.github/workflows/codeql.yaml +++ b/.github/workflows/codeql.yaml @@ -2,7 +2,7 @@ name: CodeQL on: pull_request: - branches: [main, dev] + branches: [main] schedule: - cron: '21 2 * * 3' diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1d25746c..6f698c30 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -13,10 +13,10 @@ Run unit tests with coverage: py.test --cov=arango --cov-report=html # Open htmlcov/index.html in your browser ``` -For a more comprehensive test suite, run: +To start and ArangoDB instance locally, run: ```shell -./tester.sh # Requires docker +./starter.sh # Requires docker ``` Build and test documentation: diff --git a/arango/client.py b/arango/client.py index a6bcab6d..d26352f3 100644 --- a/arango/client.py +++ b/arango/client.py @@ -15,7 +15,9 @@ from arango.exceptions import ServerConnectionError from arango.http import DEFAULT_REQUEST_TIMEOUT, DefaultHTTPClient, HTTPClient from arango.resolver import ( + FallbackHostResolver, HostResolver, + PeriodicHostResolver, RandomHostResolver, RoundRobinHostResolver, SingleHostResolver, @@ -52,9 +54,9 @@ class ArangoClient: :param hosts: Host URL or list of URLs (coordinators in a cluster). :type hosts: str | [str] :param host_resolver: Host resolver. This parameter used for clusters (when - multiple host URLs are provided). Accepted values are "roundrobin" and - "random". Any other value defaults to round robin. - :type host_resolver: str + multiple host URLs are provided). Accepted values are "fallback", + "roundrobin", "random" and "periodic". The default value is "fallback". + :type host_resolver: str | arango.resolver.HostResolver :param resolver_max_tries: Number of attempts to process an HTTP request before throwing a ConnectionAbortedError. Must not be lower than the number of hosts. @@ -88,7 +90,7 @@ class ArangoClient: def __init__( self, hosts: Union[str, Sequence[str]] = "http://127.0.0.1:8529", - host_resolver: str = "roundrobin", + host_resolver: Union[str, HostResolver] = "fallback", resolver_max_tries: Optional[int] = None, http_client: Optional[HTTPClient] = None, serializer: Callable[..., str] = default_serializer, @@ -106,10 +108,18 @@ def __init__( if host_count == 1: self._host_resolver = SingleHostResolver(1, resolver_max_tries) + elif host_resolver == "fallback": + self._host_resolver = FallbackHostResolver(host_count, resolver_max_tries) elif host_resolver == "random": self._host_resolver = RandomHostResolver(host_count, resolver_max_tries) - else: + elif host_resolver == "roundrobin": self._host_resolver = RoundRobinHostResolver(host_count, resolver_max_tries) + elif host_resolver == "periodic": + self._host_resolver = PeriodicHostResolver(host_count, resolver_max_tries) + else: + if not isinstance(host_resolver, HostResolver): + raise ValueError("Invalid host resolver") + self._host_resolver = host_resolver # Initializes the http client self._http = http_client or DefaultHTTPClient(request_timeout=request_timeout) diff --git a/arango/formatter.py b/arango/formatter.py index a74cd3ea..2058b1d6 100644 --- a/arango/formatter.py +++ b/arango/formatter.py @@ -1194,6 +1194,12 @@ def format_pregel_job_data(body: Json) -> Json: if "useMemoryMaps" in body: result["use_memory_maps"] = body["useMemoryMaps"] + # Introduced in 3.11 + if "user" in body: + result["user"] = body["user"] + if "graphLoaded" in body: + result["graph_loaded"] = body["graphLoaded"] + return verify_format(body, result) diff --git a/arango/resolver.py b/arango/resolver.py index 06a7aa77..714d76c1 100644 --- a/arango/resolver.py +++ b/arango/resolver.py @@ -1,11 +1,15 @@ __all__ = [ "HostResolver", + "FallbackHostResolver", + "PeriodicHostResolver", "SingleHostResolver", "RandomHostResolver", "RoundRobinHostResolver", ] +import logging import random +import time from abc import ABC, abstractmethod from typing import Optional, Set @@ -66,3 +70,57 @@ def __init__(self, host_count: int, max_tries: Optional[int] = None) -> None: def get_host_index(self, indexes_to_filter: Optional[Set[int]] = None) -> int: self._index = (self._index + 1) % self.host_count return self._index + + +class PeriodicHostResolver(HostResolver): + """ + Changes the host every N requests. + An optional timeout may be applied between host changes, + such that all coordinators get a chance to update their view of the agency. + For example, if one coordinator creates a database, the others may not be + immediately aware of it. If the timeout is set to 1 second, then the host + resolver waits for 1 second before changing the host. + """ + + def __init__( + self, + host_count: int, + max_tries: Optional[int] = None, + requests_period: int = 100, + switch_timeout: float = 0, + ) -> None: + super().__init__(host_count, max_tries) + self._requests_period = requests_period + self._switch_timeout = switch_timeout + self._req_count = 0 + self._index = 0 + + def get_host_index(self, indexes_to_filter: Optional[Set[int]] = None) -> int: + indexes_to_filter = indexes_to_filter or set() + self._req_count = (self._req_count + 1) % self._requests_period + if self._req_count == 0 or self._index in indexes_to_filter: + self._index = (self._index + 1) % self.host_count + while self._index in indexes_to_filter: + self._index = (self._index + 1) % self.host_count + self._req_count = 0 + time.sleep(self._switch_timeout) + return self._index + + +class FallbackHostResolver(HostResolver): + """ + Fallback host resolver. + If the current host fails, the next one is used. + """ + + def __init__(self, host_count: int, max_tries: Optional[int] = None) -> None: + super().__init__(host_count, max_tries) + self._index = 0 + self._logger = logging.getLogger(self.__class__.__name__) + + def get_host_index(self, indexes_to_filter: Optional[Set[int]] = None) -> int: + indexes_to_filter = indexes_to_filter or set() + while self._index in indexes_to_filter: + self._index = (self._index + 1) % self.host_count + self._logger.debug(f"Trying fallback on host {self._index}") + return self._index diff --git a/starter.sh b/starter.sh new file mode 100755 index 00000000..e69ce24b --- /dev/null +++ b/starter.sh @@ -0,0 +1,82 @@ +#!/bin/bash + +# Starts a local ArangoDB server or cluster (community or enterprise). +# Useful for testing the python-arango driver against a local ArangoDB setup. + +# Usage: +# ./starter.sh [single|cluster] [community|enterprise] [version] +# Example: +# ./starter.sh cluster enterprise 3.11.4 + +setup="${1:-single}" +if [[ "$setup" != "single" && "$setup" != "cluster" ]]; then + echo "Invalid argument. Please provide either 'single' or 'cluster'." + exit 1 +fi + +license="${2:-all}" +if [[ "$license" != "community" && "$license" != "enterprise" ]]; then + echo "Invalid argument. Please provide either 'community' or 'enterprise'." + exit 1 +fi + +version="${3:-3.11.4}" + +if [ "$setup" == "single" ]; then + if [ "$license" == "community" ]; then + echo "Starting community single server..." + docker run -d --rm \ + --name arango \ + -p 8528:8528 \ + -p 8529:8529 \ + -v "$(pwd)/tests/static/":/tests/static \ + -v /tmp:/tmp \ + arangodb/arangodb:"$version" \ + /bin/sh -c "arangodb --configuration=/tests/static/single.conf" + elif [ "$license" == "enterprise" ]; then + echo "Starting enterprise single server..." + docker run -d --rm \ + --name arango \ + -p 8528:8528 \ + -p 8529:8529 \ + -v "$(pwd)/tests/static/":/tests/static \ + -v /tmp:/tmp \ + arangodb/enterprise:"$version" \ + /bin/sh -c "arangodb --configuration=/tests/static/single.conf" + fi +elif [ "$setup" == "cluster" ]; then + if [ "$license" == "community" ]; then + echo "Starting community cluster..." + docker run -d --rm \ + --name arango \ + -p 8528:8528 \ + -p 8529:8529 \ + -p 8539:8539 \ + -p 8549:8549 \ + -v "$(pwd)/tests/static/":/tests/static \ + -v /tmp:/tmp \ + arangodb/arangodb:"$version" \ + /bin/sh -c "arangodb --configuration=/tests/static/cluster.conf" + elif [ "$license" == "enterprise" ]; then + echo "Starting enterprise cluster..." + docker run -d --rm \ + --name arango \ + -p 8528:8528 \ + -p 8529:8529 \ + -p 8539:8539 \ + -p 8549:8549 \ + -v "$(pwd)/tests/static/":/tests/static \ + -v /tmp:/tmp \ + arangodb/enterprise:"$version" \ + /bin/sh -c "arangodb --configuration=/tests/static/cluster.conf" + fi +fi + +wget --quiet --waitretry=1 --tries=120 -O - http://localhost:8528/version | jq +if [ $? -eq 0 ]; then + echo "OK starter ready" + exit 0 +else + echo "ERROR starter not ready, giving up" + exit 1 +fi diff --git a/tester.sh b/tester.sh deleted file mode 100755 index 00371890..00000000 --- a/tester.sh +++ /dev/null @@ -1,128 +0,0 @@ -#!/bin/bash - -# Tests python-arango driver against a local ArangoDB single server or cluster setup. -# 1. Starts a local ArangoDB server or cluster (community). -# 2. Runs the python-arango tests for the community edition. -# 3. Starts a local ArangoDB server or cluster (enterprise). -# 4. Runs all python-arango tests, including enterprise tests. - -# Usage: -# ./tester.sh [all|single|cluster] [all|community|enterprise] [version] ["notest"] - -setup="${1:-all}" -if [[ "$setup" != "all" && "$setup" != "single" && "$setup" != "cluster" ]]; then - echo "Invalid argument. Please provide either 'all', 'single' or 'cluster'." - exit 1 -fi - -tests="${2:-all}" -if [[ "$tests" != "all" && "$tests" != "community" && "$tests" != "enterprise" ]]; then - echo "Invalid argument. Please provide either 'all', 'community', or 'enterprise'." - exit 1 -fi - -# 3.11.2 -# 3.10.9 -# 3.9.11 -version="${3:-3.11.1}" - -if [[ -n "$4" && "$4" != "notest" ]]; then - echo "Invalid argument. Use 'notest' to only start the docker container, without running the tests." - exit 1 -fi -mode="${4:-test}" - -if [ "$setup" == "all" ] || [ "$setup" == "single" ]; then - if [ "$tests" == "all" ] || [ "$tests" == "community" ]; then - echo "Starting single server community setup..." - docker run -d --rm \ - --name arango \ - -p 8529:8529 \ - -v "$(pwd)/tests/static/":/tests/static \ - -v /tmp:/tmp \ - arangodb/arangodb:"$version" \ - /bin/sh -c "arangodb --configuration=/tests/static/single.conf" - - if [[ "$mode" == "notest" ]]; then - exit 0 - fi - - echo "Running python-arango tests for single server community setup..." - sleep 3 - py.test --complete --cov=arango --cov-report=html | tee single_community_results.txt - echo "Stopping single server community setup..." - docker stop arango - docker wait arango - sleep 3 - fi - - if [ "$tests" == "all" ] || [ "$tests" == "enterprise" ]; then - echo "Starting single server enterprise setup..." - docker run -d --rm \ - --name arango \ - -p 8529:8529 \ - -v "$(pwd)/tests/static/":/tests/static \ - -v /tmp:/tmp \ - arangodb/enterprise:"$version" \ - /bin/sh -c "arangodb --configuration=/tests/static/single.conf" - - if [[ "$mode" == "notest" ]]; then - exit 0 - fi - - echo "Running python-arango tests for single server enterprise setup..." - sleep 3 - py.test --complete --enterprise --cov=arango --cov-report=html --cov-append | tee single_enterprise_results.txt - echo "Stopping single server enterprise setup..." - docker stop arango - docker wait arango - sleep 3 - fi -fi - -if [ "$setup" == "all" ] || [ "$setup" == "cluster" ]; then - if [ "$tests" == "all" ] || [ "$tests" == "community" ]; then - echo "Starting community cluster setup..." - docker run -d --rm \ - --name arango \ - -p 8529:8529 \ - -v "$(pwd)/tests/static/":/tests/static \ - -v /tmp:/tmp \ - arangodb/arangodb:"$version" \ - /bin/sh -c "arangodb --configuration=/tests/static/cluster.conf" - - if [[ "$mode" == "notest" ]]; then - exit 0 - fi - - echo "Running python-arango tests for community cluster setup..." - sleep 15 - py.test --cluster --complete --cov=arango --cov-report=html | tee cluster_community_results.txt - echo "Stopping community cluster setup..." - docker stop arango - docker wait arango - sleep 3 - fi - - if [ "$tests" == "all" ] || [ "$tests" == "enterprise" ]; then - echo "Starting enterprise cluster setup..." - docker run -d --rm \ - --name arango \ - -p 8529:8529 \ - -v "$(pwd)/tests/static/":/tests/static \ - -v /tmp:/tmp \ - arangodb/enterprise:"$version" \ - /bin/sh -c "arangodb --configuration=/tests/static/cluster.conf" - - if [[ "$mode" == "notest" ]]; then - exit 0 - fi - - echo "Running python-arango tests for enterprise cluster setup..." - sleep 15 - py.test --cluster --enterprise --complete --cov=arango --cov-report=html | tee cluster_enterprise_results.txt - echo "Stopping enterprise cluster setup..." - docker stop arango - docker wait arango - fi -fi diff --git a/tests/conftest.py b/tests/conftest.py index 7ee60339..da95e2ef 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,6 +5,7 @@ from arango import ArangoClient, formatter from arango.database import StandardDatabase +from arango.http import DefaultHTTPClient from arango.typings import Json from tests.executors import TestAsyncApiExecutor, TestTransactionApiExecutor from tests.helpers import ( @@ -49,7 +50,7 @@ class GlobalData: def pytest_addoption(parser): parser.addoption("--host", action="store", default="127.0.0.1") - parser.addoption("--port", action="store", default="8529") + parser.addoption("--port", action="append", default=None) parser.addoption("--passwd", action="store", default="passwd") parser.addoption("--complete", action="store_true") parser.addoption("--cluster", action="store_true") @@ -59,15 +60,28 @@ def pytest_addoption(parser): def pytest_configure(config): - url = f"http://{config.getoption('host')}:{config.getoption('port')}" + ports = config.getoption("port") + if ports is None: + ports = ["8529"] + hosts = [f"http://{config.getoption('host')}:{p}" for p in ports] + url = hosts[0] secret = config.getoption("secret") - client = ArangoClient(hosts=[url, url, url]) + cluster = config.getoption("cluster") + + host_resolver = "fallback" + http_client = DefaultHTTPClient(request_timeout=120) + + client = ArangoClient( + hosts=hosts, host_resolver=host_resolver, http_client=http_client + ) sys_db = client.db( name="_system", username="root", password=config.getoption("passwd"), superuser_token=generate_jwt(secret), + verify=True, ) + db_version = sys_db.version() # Create a user and non-system database for testing. @@ -131,7 +145,7 @@ def pytest_configure(config): global_data.ecol_name = ecol_name global_data.fvcol_name = fvcol_name global_data.tvcol_name = tvcol_name - global_data.cluster = config.getoption("cluster") + global_data.cluster = cluster global_data.complete = config.getoption("complete") global_data.replication = config.getoption("replication") global_data.enterprise = config.getoption("enterprise") diff --git a/tests/static/cluster-3.10.conf b/tests/static/cluster-3.10.conf new file mode 100644 index 00000000..573c030a --- /dev/null +++ b/tests/static/cluster-3.10.conf @@ -0,0 +1,12 @@ +[starter] +mode = cluster +local = true +address = 0.0.0.0 +port = 8528 + +[auth] +jwt-secret = /tests/static/keyfile + +[args] +all.database.password = passwd +all.log.api-enabled = true diff --git a/tests/static/cluster.conf b/tests/static/cluster.conf index 60cfb844..182f3d17 100644 --- a/tests/static/cluster.conf +++ b/tests/static/cluster.conf @@ -2,12 +2,12 @@ mode = cluster local = true address = 0.0.0.0 +port = 8528 [auth] jwt-secret = /tests/static/keyfile [args] all.database.password = passwd -# Extended names can be used starting with 3.11 -# all.database.extended-names = true +all.database.extended-names = true all.log.api-enabled = true diff --git a/tests/static/setup.sh b/tests/static/setup.sh new file mode 100644 index 00000000..0d2189ba --- /dev/null +++ b/tests/static/setup.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +mkdir -p /tests/static +wget -O /tests/static/service.zip "http://localhost:8000/$PROJECT/tests/static/service.zip" +wget -O /tests/static/keyfile "http://localhost:8000/$PROJECT/tests/static/keyfile" +wget -O /tests/static/arangodb.conf "http://localhost:8000/$PROJECT/tests/static/$ARANGODB_CONF" +arangodb --configuration=/tests/static/arangodb.conf diff --git a/tests/static/single-3.10.conf b/tests/static/single-3.10.conf new file mode 100644 index 00000000..c982303b --- /dev/null +++ b/tests/static/single-3.10.conf @@ -0,0 +1,10 @@ +[starter] +mode = single +address = 0.0.0.0 +port = 8528 + +[auth] +jwt-secret = /tests/static/keyfile + +[args] +all.database.password = passwd diff --git a/tests/static/single.conf b/tests/static/single.conf index db6022af..e880f9d5 100644 --- a/tests/static/single.conf +++ b/tests/static/single.conf @@ -8,5 +8,4 @@ jwt-secret = /tests/static/keyfile [args] all.database.password = passwd -# Extended names can be used starting with 3.11 -# all.database.extended-names = true +all.database.extended-names = true diff --git a/tests/test_async.py b/tests/test_async.py index 61335777..0b7ca0a6 100644 --- a/tests/test_async.py +++ b/tests/test_async.py @@ -16,6 +16,12 @@ from tests.helpers import extract +@pytest.fixture(autouse=True) +def teardown(db): + yield + db.clear_async_jobs() + + def wait_on_job(job): """Block until the async job is done.""" while job.status() != "done": @@ -65,7 +71,10 @@ def test_async_execute_without_result(db, col, docs): assert async_col.insert(docs[2]) is None # Ensure that the operations went through - wait_on_jobs(db) + for _ in range(10): + if col.count() == 3: + break + time.sleep(0.5) assert extract("_key", col.all()) == ["1", "2", "3"] @@ -242,7 +251,7 @@ def test_async_list_jobs(db, col, docs): assert job3.id in job_ids # Test list async jobs that are pending - job4 = async_db.aql.execute("RETURN SLEEP(0.3)") + job4 = async_db.aql.execute("RETURN SLEEP(3)") assert db.async_jobs(status="pending") == [job4.id] wait_on_job(job4) # Make sure the job is done diff --git a/tests/test_client.py b/tests/test_client.py index c5018df2..5faa84db 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -10,11 +10,7 @@ from arango.database import StandardDatabase from arango.exceptions import ServerConnectionError from arango.http import DefaultHTTPClient -from arango.resolver import ( - RandomHostResolver, - RoundRobinHostResolver, - SingleHostResolver, -) +from arango.resolver import FallbackHostResolver, RandomHostResolver, SingleHostResolver from tests.helpers import generate_db_name, generate_string, generate_username @@ -40,7 +36,7 @@ def test_client_attributes(): assert client.version == importlib_metadata.version("python-arango") assert client.hosts == client_hosts assert repr(client) == client_repr - assert isinstance(client._host_resolver, RoundRobinHostResolver) + assert isinstance(client._host_resolver, FallbackHostResolver) client = ArangoClient( hosts=client_hosts, diff --git a/tests/test_cluster.py b/tests/test_cluster.py index 5702f877..5c4b8b4f 100644 --- a/tests/test_cluster.py +++ b/tests/test_cluster.py @@ -1,3 +1,5 @@ +import time + import pytest from packaging import version @@ -174,7 +176,15 @@ def test_cluster_rebalance(sys_db, bad_db, cluster, db_version): assert err.value.error_code == FORBIDDEN # Test rebalance execution - assert sys_db.cluster.execute_rebalance_plan(rebalance["moves"]) is True + tries = 0 + while sys_db.cluster.execute_rebalance_plan(rebalance["moves"]) is False: + if tries < 10: + tries += 1 + time.sleep(1) + else: + tries = -1 + break + assert tries != -1 with assert_raises(ClusterRebalanceError) as err: bad_db.cluster.execute_rebalance_plan(rebalance["moves"]) assert err.value.error_code == FORBIDDEN diff --git a/tests/test_database.py b/tests/test_database.py index 9ccec2c6..4fa0c0ed 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -209,7 +209,7 @@ def test_database_misc_methods(sys_db, db, bad_db, cluster): assert err.value.error_code in {11, 1228} # Test set log levels - new_levels = {"agency": "DEBUG", "collector": "INFO", "threads": "WARNING"} + new_levels = {"agency": "DEBUG", "engines": "INFO", "threads": "WARNING"} result = sys_db.set_log_levels(**new_levels) for key, value in new_levels.items(): assert result[key] == value @@ -344,7 +344,6 @@ def test_license(sys_db, enterprise): assert set(license.keys()) == { "upgrading", "features", - "hash", "license", "version", "status", diff --git a/tests/test_pregel.py b/tests/test_pregel.py index a99bddce..e17da72b 100644 --- a/tests/test_pregel.py +++ b/tests/test_pregel.py @@ -1,6 +1,7 @@ import time import pytest +from packaging import version from arango.exceptions import ( PregelJobCreateError, @@ -17,7 +18,7 @@ def test_pregel_attributes(db, username): assert repr(db.pregel) == f"" -def test_pregel_management(db, graph, cluster): +def test_pregel_management(db, db_version, graph, cluster): if cluster: pytest.skip("Not tested in a cluster setup") @@ -51,9 +52,13 @@ def test_pregel_management(db, graph, cluster): # Test delete existing pregel job assert db.pregel.delete_job(job_id) is True time.sleep(0.2) - with assert_raises(PregelJobGetError) as err: - db.pregel.job(job_id) - assert err.value.error_code in {4, 10, 1600} + if db_version < version.parse("3.11.0"): + with assert_raises(PregelJobGetError) as err: + db.pregel.job(job_id) + assert err.value.error_code in {4, 10, 1600} + else: + job = db.pregel.job(job_id) + assert job["state"] == "canceled" # Test delete missing pregel job with assert_raises(PregelJobDeleteError) as err: diff --git a/tests/test_resolver.py b/tests/test_resolver.py index ff5630a1..af58bb99 100644 --- a/tests/test_resolver.py +++ b/tests/test_resolver.py @@ -3,6 +3,8 @@ import pytest from arango.resolver import ( + FallbackHostResolver, + PeriodicHostResolver, RandomHostResolver, RoundRobinHostResolver, SingleHostResolver, @@ -54,3 +56,30 @@ def test_resolver_round_robin(): assert resolver.get_host_index() == 8 assert resolver.get_host_index() == 9 assert resolver.get_host_index() == 0 + + +def test_resolver_periodic(): + resolver = PeriodicHostResolver(3, requests_period=3) + assert resolver.get_host_index() == 0 + assert resolver.get_host_index() == 0 + assert resolver.get_host_index() == 1 + assert resolver.get_host_index() == 1 + assert resolver.get_host_index() == 1 + assert resolver.get_host_index() == 2 + assert resolver.get_host_index() == 2 + assert resolver.get_host_index() == 2 + assert resolver.get_host_index() == 0 + assert resolver.get_host_index() == 0 + assert resolver.get_host_index({0}) == 1 + assert resolver.get_host_index() == 1 + + +def test_resolver_fallback(): + resolver = FallbackHostResolver(4) + assert resolver.get_host_index() == 0 + assert resolver.get_host_index() == 0 + assert resolver.get_host_index({0, 1, 3}) == 2 + assert resolver.get_host_index({1, 2, 3}) == 0 + assert resolver.get_host_index({0}) == 1 + assert resolver.get_host_index({0}) == 1 + assert resolver.get_host_index() == 1 From 872546cbf963527cbd716c4852f3c81f4cc77d5c Mon Sep 17 00:00:00 2001 From: Alex Petenchea Date: Tue, 17 Oct 2023 14:01:41 +0300 Subject: [PATCH 03/34] uploading test results --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8f8c641f..d536da2f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -65,7 +65,7 @@ jobs: name: "Run pytest" command: | mkdir test-results - args=("--log-cli-level=DEBUG" "--host" "localhost" "--junitxml=./test-results/junit.xml") + args=("--log-cli-level=DEBUG" "--host" "localhost" "--junitxml=test-results/junit.xml") if [ << parameters.cluster >> = true ]; then args+=("--cluster" "--port=8529" "--port=8539" "--port=8549") else @@ -77,9 +77,9 @@ jobs: echo "Running py.test with args: ${args[@]}" py.test "${args[@]}" - store_test_results: - path: ./test-results/junit.xml + path: test-results - store_artifacts: - path: ./test-results + path: test-results workflows: python-3.8-community-single-3.10: From 37297bb788314683b1d85f8505a2534f799e0fcd Mon Sep 17 00:00:00 2001 From: Alex Petenchea Date: Tue, 17 Oct 2023 14:55:00 +0300 Subject: [PATCH 04/34] emitting warning instead of failing --- .circleci/config.yml | 2 +- tests/test_cluster.py | 16 ++++++---------- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d536da2f..fd740314 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -65,7 +65,7 @@ jobs: name: "Run pytest" command: | mkdir test-results - args=("--log-cli-level=DEBUG" "--host" "localhost" "--junitxml=test-results/junit.xml") + args=("--junitxml=test-results/junit.xml" "--log-cli-level=DEBUG" "--host" "localhost") if [ << parameters.cluster >> = true ]; then args+=("--cluster" "--port=8529" "--port=8539" "--port=8549") else diff --git a/tests/test_cluster.py b/tests/test_cluster.py index 5c4b8b4f..bbc31778 100644 --- a/tests/test_cluster.py +++ b/tests/test_cluster.py @@ -1,4 +1,4 @@ -import time +import warnings import pytest from packaging import version @@ -176,15 +176,11 @@ def test_cluster_rebalance(sys_db, bad_db, cluster, db_version): assert err.value.error_code == FORBIDDEN # Test rebalance execution - tries = 0 - while sys_db.cluster.execute_rebalance_plan(rebalance["moves"]) is False: - if tries < 10: - tries += 1 - time.sleep(1) - else: - tries = -1 - break - assert tries != -1 + if sys_db.cluster.execute_rebalance_plan(rebalance["moves"]) is False: + warnings.warn( + "Rebalance plan was not executed." + "This may happen independent of the driver." + ) with assert_raises(ClusterRebalanceError) as err: bad_db.cluster.execute_rebalance_plan(rebalance["moves"]) assert err.value.error_code == FORBIDDEN From 00c1795fc280de4acc59f2379201918f1ba5a560 Mon Sep 17 00:00:00 2001 From: Alex Petenchea Date: Tue, 17 Oct 2023 15:09:13 +0300 Subject: [PATCH 05/34] disabling log --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index fd740314..82819322 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -65,7 +65,7 @@ jobs: name: "Run pytest" command: | mkdir test-results - args=("--junitxml=test-results/junit.xml" "--log-cli-level=DEBUG" "--host" "localhost") + args=("--junitxml=test-results/junit.xml" "--host" "localhost") if [ << parameters.cluster >> = true ]; then args+=("--cluster" "--port=8529" "--port=8539" "--port=8549") else From 10782d1bb8a3ccdcb5b77939db86f454d1013fce Mon Sep 17 00:00:00 2001 From: Alex Petenchea Date: Tue, 17 Oct 2023 15:26:25 +0300 Subject: [PATCH 06/34] activate venv --- .circleci/config.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 82819322..13a256d6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -64,8 +64,9 @@ jobs: - run: name: "Run pytest" command: | + . venv/bin/activate mkdir test-results - args=("--junitxml=test-results/junit.xml" "--host" "localhost") + args=("--junitxml=test-results/junit.xml" "--log-cli-level=DEBUG" "--host" "localhost") if [ << parameters.cluster >> = true ]; then args+=("--cluster" "--port=8529" "--port=8539" "--port=8549") else @@ -76,10 +77,10 @@ jobs: fi echo "Running py.test with args: ${args[@]}" py.test "${args[@]}" - - store_test_results: - path: test-results - store_artifacts: path: test-results + - store_test_results: + path: test-results workflows: python-3.8-community-single-3.10: From 8fa6e8bbe752570c5ea332f2505017b0cb303267 Mon Sep 17 00:00:00 2001 From: Alex Petenchea Date: Tue, 17 Oct 2023 15:28:48 +0300 Subject: [PATCH 07/34] running without venv --- .circleci/config.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 13a256d6..921a3bf7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -64,7 +64,6 @@ jobs: - run: name: "Run pytest" command: | - . venv/bin/activate mkdir test-results args=("--junitxml=test-results/junit.xml" "--log-cli-level=DEBUG" "--host" "localhost") if [ << parameters.cluster >> = true ]; then From 592c2555fbbe0cd6874b6714b677fc1fd6cf1cce Mon Sep 17 00:00:00 2001 From: Anthony Mahanna Date: Thu, 9 Nov 2023 18:30:24 -0500 Subject: [PATCH 08/34] Update config.yml --- .circleci/config.yml | 217 +++++++++++++++---------------------------- 1 file changed, 75 insertions(+), 142 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 921a3bf7..f08294d2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,164 +1,97 @@ version: 2.1 -parameters: - preview: - type: boolean - default: true - 3-10: - type: boolean - default: true - 3-11: - type: boolean - default: true +# orbs: +# coveralls: coveralls/coveralls@2.2.1 + +workflows: + ci: + jobs: + - lint + - test: + matrix: + parameters: + python_version: ["3.10"] # ["3.8", "3.9", "3.10", "3.11", "3.12"] + arangodb_config: ["single"] # ["single", "cluster"] + arangodb_license: ["community"] # ["community", "enterprise"] + arangodb_version: ["latest"] # ["3.10.10", "3.11.4", "latest"] jobs: - run-tests: + lint: + docker: + - image: python:latest + steps: + - checkout + - run: + name: Install Dependencies + command: pip install .[dev] + + - run: + name: Run black + command: black --check --verbose --diff --color --config=pyproject.toml ./arango ./tests/ + + - run: + name: Run flake8 + command: flake8 ./arango ./tests + + - run: + name: Run isort + command: isort --check ./arango ./tests + + - run: + name: Run mypy + command: mypy ./arango + + test: resource_class: small parameters: - python-version: + python_version: type: string - default: "latest" - arangodb-version: + arangodb_config: type: string - default: "arangodb:latest" - arangodb-config: + arangodb_license: + type: string + arangodb_version: type: string - default: "single.conf" - cluster: - type: boolean - default: false - enterprise: - type: boolean - default: false docker: - - image: python:<< parameters.python-version >> - command: ["/bin/sh", "-c", "python -m http.server"] - - image: arangodb/<< parameters.arangodb-version >> - environment: - ARANGODB_CONF: << parameters.arangodb-config >> - PROJECT: /root/project - command: - - "/bin/sh" - - "-c" - - > - while ! wget -q -O /dev/null http://localhost:8000/$PROJECT/tests/static/setup.sh; do sleep 1; done && - wget -O - http://localhost:8000/$PROJECT/tests/static/setup.sh | - /bin/sh + - image: cimg/python:<< parameters.python_version >> steps: - checkout + - setup_remote_docker + - run: - name: "Install Dependencies" + name: Set Up ArangoDB command: | - pip install -e .[dev] + chmod +x starter.sh + ./starter.sh << parameters.arangodb_config >> << parameters.arangodb_license >> << parameters.arangodb_version >> + - run: - name: "Wait for ArangoDB starter" - command: | - wget --quiet --waitretry=1 --tries=120 -O - http://localhost:8528/version - if [ $? -eq 0 ]; then - echo "starter ready" - exit 0 - else - echo "starter not ready, giving up" - exit 1 - fi + name: "Install Dependencies" + command: pip install -e .[dev] pytest + + - run: + name: "Sleep" + command: sleep 15 + + - run: docker ps -a + + # - run: docker logs arango + + - run: curl -v http://localhost:8529/ + - run: name: "Run pytest" command: | - mkdir test-results - args=("--junitxml=test-results/junit.xml" "--log-cli-level=DEBUG" "--host" "localhost") - if [ << parameters.cluster >> = true ]; then - args+=("--cluster" "--port=8529" "--port=8539" "--port=8549") - else - args+=("--port=8529") + args=("--junitxml=test-results/junit.xml" "--log-cli-level=DEBUG" "--host" "arango" "--port=8529") + + if [ << parameters.arangodb_config >> = "cluster" ]; then + args+=("--cluster" "--port=8539" "--port=8549") fi - if [ << parameters.enterprise >> = true ]; then + + if [ << parameters.arangodb_license >> = "enterprise" ]; then args+=("--enterprise") fi - echo "Running py.test with args: ${args[@]}" - py.test "${args[@]}" - - store_artifacts: - path: test-results - - store_test_results: - path: test-results -workflows: - python-3.8-community-single-3.10: - when: << pipeline.parameters.3-10 >> - jobs: - - run-tests: - name: python-3.8-community-single-3.10 - python-version: "3.8.2" - arangodb-version: "arangodb:3.10.10" - arangodb-config: "single-3.10.conf" - cluster: false - enterprise: false - python-3.8-enterprise-cluster-3.10: - when: << pipeline.parameters.3-10 >> - jobs: - - run-tests: - name: python-3.8-enterprise-cluster-3.10 - python-version: "3.8.2" - arangodb-version: "enterprise:3.10.10" - arangodb-config: "cluster-3.10.conf" - cluster: true - enterprise: true - python-3.10-community-single-3.11: - when: << pipeline.parameters.3-11 >> - jobs: - - run-tests: - name: python-3.10-community-single-3.11 - python-version: "3.10.6" - arangodb-version: "arangodb:3.11.4" - arangodb-config: "single.conf" - cluster: false - enterprise: false - python-3.10-community-cluster-3.11: - when: << pipeline.parameters.3-11 >> - jobs: - - run-tests: - name: python-3.10-community-cluster-3.11 - python-version: "3.10.6" - arangodb-version: "arangodb:3.11.4" - arangodb-config: "cluster.conf" - cluster: true - enterprise: false - python-3.10-enterprise-single-3.11: - when: << pipeline.parameters.3-11 >> - jobs: - - run-tests: - name: python-3.10-enterprise-single-3.11 - python-version: "3.10.6" - arangodb-version: "enterprise:3.11.4" - arangodb-config: "single.conf" - cluster: false - enterprise: true - python-3.10-enterprise-cluster-3.11: - when: << pipeline.parameters.3-11 >> - jobs: - - run-tests: - name: python-3.10-enterprise-cluster-3.11 - python-version: "3.10.6" - arangodb-version: "enterprise:3.11.4" - arangodb-config: "cluster.conf" - cluster: true - enterprise: true - python-latest-enterprise-single-preview: - when: << pipeline.parameters.preview >> - jobs: - - run-tests: - name: python-latest-enterprise-single-preview - python-version: "latest" - arangodb-version: "enterprise-preview:latest" - arangodb-config: "single.conf" - cluster: false - enterprise: true - python-latest-enterprise-cluster-preview: - when: << pipeline.parameters.preview >> - jobs: - - run-tests: - name: python-latest-enterprise-cluster-preview - python-version: "latest" - arangodb-version: "enterprise-preview:latest" - arangodb-config: "cluster.conf" - cluster: true - enterprise: true + echo "Running pytest with args: ${args[@]}" + pytest --cov=arango --cov-report=xml "${args[@]}" + + # - coveralls/upload From e3d7a52c1cb21b60b07a4b6aaad9fa16f1a91dc1 Mon Sep 17 00:00:00 2001 From: Anthony Mahanna Date: Thu, 9 Nov 2023 18:30:26 -0500 Subject: [PATCH 09/34] Update build.yaml --- .github/workflows/build.yaml | 109 ++++++++++++++++++++++++++++++----- 1 file changed, 96 insertions(+), 13 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index a542f28b..7f2c8e52 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -1,23 +1,106 @@ name: Build -on: - pull_request: - branches: [main] - workflow_dispatch: - inputs: - debug_enabled: - type: boolean - description: Debug with tmate - required: false - default: false +on: push +# on: +# pull_request: +# branches: [main] +# workflow_dispatch: +# inputs: +# debug_enabled: +# type: boolean +# description: Debug with tmate +# required: false +# default: false jobs: + lint: + runs-on: ubuntu-latest + + steps: + - name: Check out code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.10' + + - name: Install Dependencies + run: pip install .[dev] + + - name: Run black + run: black --check --verbose --diff --color --config=pyproject.toml ./arango ./tests/ + + - name: Run flake8 + run: flake8 ./arango ./tests + + - name: Run isort + run: isort --check ./arango ./tests + + - name: Run mypy + run: mypy ./arango + + test: + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + python_version: ["3.10"] #["3.8", "3.9", "3.10", "3.11", "3.12"] + arangodb_config: ["single"] #["single", "cluster"] + arangodb_license: ["community"] #["community", "enterprise"] + arangodb_version: ["latest"] #["3.10.10", "3.11.4", "latest"] + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python_version }} + + - name: Setup ArangoDB + run: | + chmod +x starter.sh + ./starter.sh ${{ matrix.arangodb_config }} ${{ matrix.arangodb_license }} ${{ matrix.arangodb_version }} + + - name: Install Dependencies + run: pip install -e .[dev] + + # - name: Sleep + # run: sleep 15 + + - name: List Docker Containers + run: docker ps -a + + # - name: Show Docker Logs + # run: docker logs arango + + # - name: Curl Request + # run: curl -v http://localhost:8529/ + + - name: Pytest + run: | + args=("--host" "localhost" "--port=8529") + + if [ ${{ matrix.arangodb_config }} = "cluster" ]; then + args+=("--cluster" "--port=8539" "--port=8549") + fi + + if [ ${{ matrix.arangodb_license }} = "enterprise" ]; then + args+=("--enterprise") + fi + + echo "Running pytest with args: ${args[@]}" + pytest -k "test_client_good_connection" --cov=arango --cov-report=xml "${args[@]}" + docs: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Fetch all tags and branches run: git fetch --prune --unshallow @@ -25,7 +108,7 @@ jobs: - name: Create ArangoDB Docker container run: > docker create --name arango -p 8529:8529 -e ARANGO_ROOT_PASSWORD=passwd -v "$(pwd)/tests/static/":/tests/static - arangodb/arangodb:3.11.4 --server.jwt-secret-keyfile=/tests/static/keyfile + arangodb/arangodb:latest --server.jwt-secret-keyfile=/tests/static/keyfile - name: Start ArangoDB Docker container run: docker start arango From 030a7710f3311b9ff867fe99effe22376fbd71aa Mon Sep 17 00:00:00 2001 From: Anthony Mahanna Date: Thu, 9 Nov 2023 18:30:27 -0500 Subject: [PATCH 10/34] Update starter.sh --- starter.sh | 91 ++++++++++++++++++++---------------------------------- 1 file changed, 34 insertions(+), 57 deletions(-) diff --git a/starter.sh b/starter.sh index e69ce24b..b0374c58 100755 --- a/starter.sh +++ b/starter.sh @@ -9,74 +9,51 @@ # ./starter.sh cluster enterprise 3.11.4 setup="${1:-single}" -if [[ "$setup" != "single" && "$setup" != "cluster" ]]; then +license="${2:-community}" +version="${3:-latest}" + +extra_ports="" +if [ "$setup" == "single" ]; then + echo "" +elif [ "$setup" == "cluster" ]; then + extra_ports="-p 8539:8539 -p 8549:8549" +else echo "Invalid argument. Please provide either 'single' or 'cluster'." exit 1 fi -license="${2:-all}" -if [[ "$license" != "community" && "$license" != "enterprise" ]]; then +image_name="" +if [ "$license" == "community" ]; then + image_name="arangodb" +elif [ "$license" == "enterprise" ]; then + image_name="enterprise" +else echo "Invalid argument. Please provide either 'community' or 'enterprise'." exit 1 fi -version="${3:-3.11.4}" - -if [ "$setup" == "single" ]; then - if [ "$license" == "community" ]; then - echo "Starting community single server..." - docker run -d --rm \ - --name arango \ - -p 8528:8528 \ - -p 8529:8529 \ - -v "$(pwd)/tests/static/":/tests/static \ - -v /tmp:/tmp \ - arangodb/arangodb:"$version" \ - /bin/sh -c "arangodb --configuration=/tests/static/single.conf" - elif [ "$license" == "enterprise" ]; then - echo "Starting enterprise single server..." - docker run -d --rm \ - --name arango \ - -p 8528:8528 \ - -p 8529:8529 \ - -v "$(pwd)/tests/static/":/tests/static \ - -v /tmp:/tmp \ - arangodb/enterprise:"$version" \ - /bin/sh -c "arangodb --configuration=/tests/static/single.conf" - fi -elif [ "$setup" == "cluster" ]; then - if [ "$license" == "community" ]; then - echo "Starting community cluster..." - docker run -d --rm \ - --name arango \ - -p 8528:8528 \ - -p 8529:8529 \ - -p 8539:8539 \ - -p 8549:8549 \ - -v "$(pwd)/tests/static/":/tests/static \ - -v /tmp:/tmp \ - arangodb/arangodb:"$version" \ - /bin/sh -c "arangodb --configuration=/tests/static/cluster.conf" - elif [ "$license" == "enterprise" ]; then - echo "Starting enterprise cluster..." - docker run -d --rm \ - --name arango \ - -p 8528:8528 \ - -p 8529:8529 \ - -p 8539:8539 \ - -p 8549:8549 \ - -v "$(pwd)/tests/static/":/tests/static \ - -v /tmp:/tmp \ - arangodb/enterprise:"$version" \ - /bin/sh -c "arangodb --configuration=/tests/static/cluster.conf" - fi +conf_file="" +if [[ "${version%.*}" == "3.10" ]]; then + conf_file="${setup}-3.10" +else + conf_file="${setup}" fi +docker run -d --rm \ + --name arango \ + -p 8528:8528 \ + -p 8529:8529 \ + $extra_ports \ + -v "$(pwd)/tests/static/":/tests/static \ + -v /tmp:/tmp \ + "arangodb/$image_name:$version" \ + /bin/sh -c "arangodb --configuration=/tests/static/$conf_file.conf" + wget --quiet --waitretry=1 --tries=120 -O - http://localhost:8528/version | jq if [ $? -eq 0 ]; then - echo "OK starter ready" - exit 0 + echo "OK starter ready" + exit 0 else - echo "ERROR starter not ready, giving up" - exit 1 + echo "ERROR starter not ready, giving up" + exit 1 fi From 99f26210bb276aa625e5dd4207662db0a5416a8a Mon Sep 17 00:00:00 2001 From: Anthony Mahanna Date: Thu, 9 Nov 2023 18:31:51 -0500 Subject: [PATCH 11/34] remove `pytest -k` flag --- .github/workflows/build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 7f2c8e52..ad74e62b 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -93,7 +93,7 @@ jobs: fi echo "Running pytest with args: ${args[@]}" - pytest -k "test_client_good_connection" --cov=arango --cov-report=xml "${args[@]}" + pytest --cov=arango --cov-report=xml "${args[@]}" docs: runs-on: ubuntu-latest From 29733a2797f67291e77eebbc39796c42e6b9c116 Mon Sep 17 00:00:00 2001 From: Anthony Mahanna Date: Thu, 9 Nov 2023 18:40:21 -0500 Subject: [PATCH 12/34] include all arangodb variations --- .github/workflows/build.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index ad74e62b..f4ea665a 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -47,9 +47,9 @@ jobs: fail-fast: false matrix: python_version: ["3.10"] #["3.8", "3.9", "3.10", "3.11", "3.12"] - arangodb_config: ["single"] #["single", "cluster"] - arangodb_license: ["community"] #["community", "enterprise"] - arangodb_version: ["latest"] #["3.10.10", "3.11.4", "latest"] + arangodb_config: ["single", "cluster"] + arangodb_license: ["community", "enterprise"] + arangodb_version: ["3.10.10", "3.11.4", "latest"] steps: - name: Checkout code From ae433fa97f2bbf344909d15e1ab0613461a0212d Mon Sep 17 00:00:00 2001 From: Anthony Mahanna Date: Thu, 9 Nov 2023 18:54:35 -0500 Subject: [PATCH 13/34] cleanup --- .github/workflows/build.yaml | 31 +++++++++++-------------------- 1 file changed, 11 insertions(+), 20 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index f4ea665a..dac043d4 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -1,16 +1,14 @@ name: Build -on: push -# on: -# pull_request: -# branches: [main] -# workflow_dispatch: -# inputs: -# debug_enabled: -# type: boolean -# description: Debug with tmate -# required: false -# default: false +on: + pull_request: + workflow_dispatch: + inputs: + debug_enabled: + type: boolean + description: Debug with tmate + required: false + default: false jobs: lint: @@ -51,6 +49,8 @@ jobs: arangodb_license: ["community", "enterprise"] arangodb_version: ["3.10.10", "3.11.4", "latest"] + name: Python ${{ matrix.python_version }} - ArangoDB ${{ matrix.arangodb_config }}:${{ matrix.arangodb_license }}:${{ matrix.arangodb_version }} + steps: - name: Checkout code uses: actions/checkout@v4 @@ -68,18 +68,9 @@ jobs: - name: Install Dependencies run: pip install -e .[dev] - # - name: Sleep - # run: sleep 15 - - name: List Docker Containers run: docker ps -a - # - name: Show Docker Logs - # run: docker logs arango - - # - name: Curl Request - # run: curl -v http://localhost:8529/ - - name: Pytest run: | args=("--host" "localhost" "--port=8529") From e1979bf7e3c5d541dc41cb1ea6267c3ee52a28b4 Mon Sep 17 00:00:00 2001 From: Anthony Mahanna Date: Thu, 9 Nov 2023 19:00:23 -0500 Subject: [PATCH 14/34] update job names --- .github/workflows/build.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index dac043d4..8cde4c9d 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -14,6 +14,8 @@ jobs: lint: runs-on: ubuntu-latest + name: Lint + steps: - name: Check out code uses: actions/checkout@v4 @@ -49,7 +51,7 @@ jobs: arangodb_license: ["community", "enterprise"] arangodb_version: ["3.10.10", "3.11.4", "latest"] - name: Python ${{ matrix.python_version }} - ArangoDB ${{ matrix.arangodb_config }}:${{ matrix.arangodb_license }}:${{ matrix.arangodb_version }} + name: Test (${{ matrix.python_version }}:${{ matrix.arangodb_config }}:${{ matrix.arangodb_license }}:${{ matrix.arangodb_version }}) steps: - name: Checkout code @@ -89,6 +91,8 @@ jobs: docs: runs-on: ubuntu-latest + name: Sphinx Docs + steps: - name: Checkout repository uses: actions/checkout@v4 From 80867ad79197427f7526040301f1f74abe91008b Mon Sep 17 00:00:00 2001 From: Anthony Mahanna Date: Thu, 9 Nov 2023 19:01:00 -0500 Subject: [PATCH 15/34] Update build.yaml --- .github/workflows/build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 8cde4c9d..73e015ce 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -91,7 +91,7 @@ jobs: docs: runs-on: ubuntu-latest - name: Sphinx Docs + name: Docs steps: - name: Checkout repository From fcb3ff81bf9ca86950cfe3d1835ba8bb8262f774 Mon Sep 17 00:00:00 2001 From: Anthony Mahanna Date: Fri, 10 Nov 2023 10:24:59 -0500 Subject: [PATCH 16/34] Update config.yml --- .circleci/config.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f08294d2..91f27a24 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -52,11 +52,11 @@ jobs: type: string arangodb_version: type: string - docker: - - image: cimg/python:<< parameters.python_version >> + machine: + image: ubuntu-2004:202101-01 steps: - checkout - - setup_remote_docker + # - setup_remote_docker - run: name: Set Up ArangoDB @@ -74,7 +74,7 @@ jobs: - run: docker ps -a - # - run: docker logs arango + - run: docker logs arango - run: curl -v http://localhost:8529/ From 1d00cab339f0251b6ba2daf6fb1bfa9a41cfc732 Mon Sep 17 00:00:00 2001 From: Anthony Mahanna Date: Fri, 10 Nov 2023 10:28:08 -0500 Subject: [PATCH 17/34] Update config.yml --- .circleci/config.yml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 91f27a24..c5c1e347 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -56,7 +56,6 @@ jobs: image: ubuntu-2004:202101-01 steps: - checkout - # - setup_remote_docker - run: name: Set Up ArangoDB @@ -64,13 +63,18 @@ jobs: chmod +x starter.sh ./starter.sh << parameters.arangodb_config >> << parameters.arangodb_license >> << parameters.arangodb_version >> + - restore_cache: + key: pip-and-local-cache + - run: - name: "Install Dependencies" - command: pip install -e .[dev] pytest + name: Setup Python + command: | + pyenv install << parameters.python_version >> + pyenv global << parameters.python_version >> - run: - name: "Sleep" - command: sleep 15 + name: "Install Dependencies" + command: pip install -e .[dev] pytest - run: docker ps -a From e660872eb4332b01d402e57d0db4a7b61e5500c6 Mon Sep 17 00:00:00 2001 From: Anthony Mahanna Date: Fri, 10 Nov 2023 10:29:24 -0500 Subject: [PATCH 18/34] Update config.yml --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c5c1e347..cf1e2865 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -53,7 +53,7 @@ jobs: arangodb_version: type: string machine: - image: ubuntu-2004:202101-01 + image: ubuntu-2204:2023.10.1 steps: - checkout From c096095c63962c9cf51109ec8c9dbd48c44f6e41 Mon Sep 17 00:00:00 2001 From: Anthony Mahanna Date: Fri, 10 Nov 2023 10:39:35 -0500 Subject: [PATCH 19/34] Update config.yml --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index cf1e2865..fc1e30fb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -96,6 +96,6 @@ jobs: fi echo "Running pytest with args: ${args[@]}" - pytest --cov=arango --cov-report=xml "${args[@]}" + py.test --cov=arango --cov-report=xml "${args[@]}" # - coveralls/upload From 82696f4070d2df043bdc2964ac4755b09362c62f Mon Sep 17 00:00:00 2001 From: Anthony Mahanna Date: Fri, 10 Nov 2023 10:49:53 -0500 Subject: [PATCH 20/34] Update config.yml --- .circleci/config.yml | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index fc1e30fb..264fce10 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -74,7 +74,7 @@ jobs: - run: name: "Install Dependencies" - command: pip install -e .[dev] pytest + command: pip install -e .[dev] - run: docker ps -a @@ -85,17 +85,24 @@ jobs: - run: name: "Run pytest" command: | - args=("--junitxml=test-results/junit.xml" "--log-cli-level=DEBUG" "--host" "arango" "--port=8529") + mkdir test-results + args=("--junitxml=test-results/junit.xml" "--log-cli-level=DEBUG" "--host" "localhost" "--port=8529") if [ << parameters.arangodb_config >> = "cluster" ]; then - args+=("--cluster" "--port=8539" "--port=8549") + args+=("--cluster" "--port=8529" "--port=8539" "--port=8549") fi - + if [ << parameters.arangodb_license >> = "enterprise" ]; then args+=("--enterprise") fi - echo "Running pytest with args: ${args[@]}" - py.test --cov=arango --cov-report=xml "${args[@]}" + echo "Running py.test with args: ${args[@]}" + pytest "${args[@]}" + + - store_artifacts: + path: test-results + + - store_test_results: + path: test-results # - coveralls/upload From ce12ebdded97b74a0ce2a5fd68cdaa4120800fd0 Mon Sep 17 00:00:00 2001 From: Anthony Mahanna Date: Fri, 10 Nov 2023 10:55:52 -0500 Subject: [PATCH 21/34] fix --port flag --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 264fce10..7d8e76e0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -89,14 +89,14 @@ jobs: args=("--junitxml=test-results/junit.xml" "--log-cli-level=DEBUG" "--host" "localhost" "--port=8529") if [ << parameters.arangodb_config >> = "cluster" ]; then - args+=("--cluster" "--port=8529" "--port=8539" "--port=8549") + args+=("--cluster" "--port=8539" "--port=8549") fi if [ << parameters.arangodb_license >> = "enterprise" ]; then args+=("--enterprise") fi - echo "Running py.test with args: ${args[@]}" + echo "Running pytest with args: ${args[@]}" pytest "${args[@]}" - store_artifacts: From ced72f2468b089c3b5f1b6fd85482adfd219b7fc Mon Sep 17 00:00:00 2001 From: Anthony Mahanna Date: Fri, 10 Nov 2023 10:55:57 -0500 Subject: [PATCH 22/34] fail test on purpose --- tests/test_client.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_client.py b/tests/test_client.py index 5faa84db..eafc3485 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -55,6 +55,7 @@ def test_client_attributes(): def test_client_good_connection(db, username, password): + assert False client = ArangoClient(hosts="http://127.0.0.1:8529") # Test connection with verify flag on and off From 8dcfeae90dbe4db0148ea685432140969df4aff8 Mon Sep 17 00:00:00 2001 From: Anthony Mahanna Date: Fri, 10 Nov 2023 11:07:05 -0500 Subject: [PATCH 23/34] remove test failure --- tests/test_client.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_client.py b/tests/test_client.py index eafc3485..5faa84db 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -55,7 +55,6 @@ def test_client_attributes(): def test_client_good_connection(db, username, password): - assert False client = ArangoClient(hosts="http://127.0.0.1:8529") # Test connection with verify flag on and off From b07de3dcdf15dafaaf78adb83137c5d8b3ef6961 Mon Sep 17 00:00:00 2001 From: Anthony Mahanna Date: Fri, 10 Nov 2023 11:08:00 -0500 Subject: [PATCH 24/34] Update config.yml --- .circleci/config.yml | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7d8e76e0..711e2339 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -11,9 +11,9 @@ workflows: matrix: parameters: python_version: ["3.10"] # ["3.8", "3.9", "3.10", "3.11", "3.12"] - arangodb_config: ["single"] # ["single", "cluster"] - arangodb_license: ["community"] # ["community", "enterprise"] - arangodb_version: ["latest"] # ["3.10.10", "3.11.4", "latest"] + arangodb_config: ["single", "cluster"] + arangodb_license: ["community", "enterprise"] + arangodb_version: ["3.10.10", "3.11.4", "latest"] jobs: lint: @@ -78,10 +78,6 @@ jobs: - run: docker ps -a - - run: docker logs arango - - - run: curl -v http://localhost:8529/ - - run: name: "Run pytest" command: | @@ -97,7 +93,7 @@ jobs: fi echo "Running pytest with args: ${args[@]}" - pytest "${args[@]}" + pytest --cov=arango --cov-report=xml "${args[@]}" - store_artifacts: path: test-results From 9af147c94258b6f46d505aa942db88ea4c09d6fc Mon Sep 17 00:00:00 2001 From: Anthony Mahanna Date: Fri, 10 Nov 2023 11:30:40 -0500 Subject: [PATCH 25/34] rename `build.yaml` to `docs.yaml` --- .github/workflows/build.yaml | 130 ----------------------------------- .github/workflows/docs.yaml | 102 +++++++++++++++++++++++++++ 2 files changed, 102 insertions(+), 130 deletions(-) delete mode 100644 .github/workflows/build.yaml create mode 100644 .github/workflows/docs.yaml diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml deleted file mode 100644 index 73e015ce..00000000 --- a/.github/workflows/build.yaml +++ /dev/null @@ -1,130 +0,0 @@ -name: Build - -on: - pull_request: - workflow_dispatch: - inputs: - debug_enabled: - type: boolean - description: Debug with tmate - required: false - default: false - -jobs: - lint: - runs-on: ubuntu-latest - - name: Lint - - steps: - - name: Check out code - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.10' - - - name: Install Dependencies - run: pip install .[dev] - - - name: Run black - run: black --check --verbose --diff --color --config=pyproject.toml ./arango ./tests/ - - - name: Run flake8 - run: flake8 ./arango ./tests - - - name: Run isort - run: isort --check ./arango ./tests - - - name: Run mypy - run: mypy ./arango - - test: - runs-on: ubuntu-latest - - strategy: - fail-fast: false - matrix: - python_version: ["3.10"] #["3.8", "3.9", "3.10", "3.11", "3.12"] - arangodb_config: ["single", "cluster"] - arangodb_license: ["community", "enterprise"] - arangodb_version: ["3.10.10", "3.11.4", "latest"] - - name: Test (${{ matrix.python_version }}:${{ matrix.arangodb_config }}:${{ matrix.arangodb_license }}:${{ matrix.arangodb_version }}) - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python_version }} - - - name: Setup ArangoDB - run: | - chmod +x starter.sh - ./starter.sh ${{ matrix.arangodb_config }} ${{ matrix.arangodb_license }} ${{ matrix.arangodb_version }} - - - name: Install Dependencies - run: pip install -e .[dev] - - - name: List Docker Containers - run: docker ps -a - - - name: Pytest - run: | - args=("--host" "localhost" "--port=8529") - - if [ ${{ matrix.arangodb_config }} = "cluster" ]; then - args+=("--cluster" "--port=8539" "--port=8549") - fi - - if [ ${{ matrix.arangodb_license }} = "enterprise" ]; then - args+=("--enterprise") - fi - - echo "Running pytest with args: ${args[@]}" - pytest --cov=arango --cov-report=xml "${args[@]}" - - docs: - runs-on: ubuntu-latest - - name: Docs - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Fetch all tags and branches - run: git fetch --prune --unshallow - - - name: Create ArangoDB Docker container - run: > - docker create --name arango -p 8529:8529 -e ARANGO_ROOT_PASSWORD=passwd -v "$(pwd)/tests/static/":/tests/static - arangodb/arangodb:latest --server.jwt-secret-keyfile=/tests/static/keyfile - - - name: Start ArangoDB Docker container - run: docker start arango - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.10' - - - name: Debug with tmate - uses: mxschmitt/action-tmate@v3 - if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }} - - - name: Run pre-commit checks - uses: pre-commit/action@v3.0.0 - - - name: Install dependencies - run: pip install .[dev] - - - name: Run Sphinx doctest - run: python -m sphinx -b doctest docs docs/_build - - - name: Generate Sphinx HTML - run: python -m sphinx -b html -W docs docs/_build diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml new file mode 100644 index 00000000..bc06e12e --- /dev/null +++ b/.github/workflows/docs.yaml @@ -0,0 +1,102 @@ +name: Docs + +on: + pull_request: + workflow_dispatch: + inputs: + debug_enabled: + type: boolean + description: Debug with tmate + required: false + default: false + +jobs: + # This has been migrated to CircleCI + # test: + # runs-on: ubuntu-latest + + # strategy: + # fail-fast: false + # matrix: + # python_version: ["3.10"] #["3.8", "3.9", "3.10", "3.11", "3.12"] + # arangodb_config: ["single", "cluster"] + # arangodb_license: ["community", "enterprise"] + # arangodb_version: ["3.10.10", "3.11.4", "latest"] + + # name: Test (${{ matrix.python_version }}:${{ matrix.arangodb_config }}:${{ matrix.arangodb_license }}:${{ matrix.arangodb_version }}) + + # steps: + # - name: Checkout code + # uses: actions/checkout@v4 + + # - name: Set up Python + # uses: actions/setup-python@v4 + # with: + # python-version: ${{ matrix.python_version }} + + # - name: Setup ArangoDB + # run: | + # chmod +x starter.sh + # ./starter.sh ${{ matrix.arangodb_config }} ${{ matrix.arangodb_license }} ${{ matrix.arangodb_version }} + + # - name: Install Dependencies + # run: pip install -e .[dev] + + # - name: List Docker Containers + # run: docker ps -a + + # - name: Pytest + # run: | + # args=("--host" "localhost" "--port=8529") + + # if [ ${{ matrix.arangodb_config }} = "cluster" ]; then + # args+=("--cluster" "--port=8539" "--port=8549") + # fi + + # if [ ${{ matrix.arangodb_license }} = "enterprise" ]; then + # args+=("--enterprise") + # fi + + # echo "Running pytest with args: ${args[@]}" + # pytest --cov=arango --cov-report=xml "${args[@]}" + + docs: + runs-on: ubuntu-latest + + name: Docs + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Fetch all tags and branches + run: git fetch --prune --unshallow + + - name: Create ArangoDB Docker container + run: > + docker create --name arango -p 8529:8529 -e ARANGO_ROOT_PASSWORD=passwd -v "$(pwd)/tests/static/":/tests/static + arangodb/arangodb:latest --server.jwt-secret-keyfile=/tests/static/keyfile + + - name: Start ArangoDB Docker container + run: docker start arango + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.10' + + - name: Debug with tmate + uses: mxschmitt/action-tmate@v3 + if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }} + + - name: Run pre-commit checks + uses: pre-commit/action@v3.0.0 + + - name: Install dependencies + run: pip install .[dev] + + - name: Run Sphinx doctest + run: python -m sphinx -b doctest docs docs/_build + + - name: Generate Sphinx HTML + run: python -m sphinx -b html -W docs docs/_build From f9e3e9e01f6dfc6bcb003ed752550d009a1b7c95 Mon Sep 17 00:00:00 2001 From: Anthony Mahanna Date: Fri, 10 Nov 2023 11:30:48 -0500 Subject: [PATCH 26/34] use all python versions --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 711e2339..5ef4eaca 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -10,7 +10,7 @@ workflows: - test: matrix: parameters: - python_version: ["3.10"] # ["3.8", "3.9", "3.10", "3.11", "3.12"] + python_version: ["3.8", "3.9", "3.10", "3.11", "3.12"] arangodb_config: ["single", "cluster"] arangodb_license: ["community", "enterprise"] arangodb_version: ["3.10.10", "3.11.4", "latest"] From 561c4e35356fdc8a123f1a4c816796503b3479c2 Mon Sep 17 00:00:00 2001 From: Anthony Mahanna Date: Fri, 10 Nov 2023 12:28:29 -0500 Subject: [PATCH 27/34] Update config.yml --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5ef4eaca..eaab9378 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -42,7 +42,7 @@ jobs: command: mypy ./arango test: - resource_class: small + resource_class: medium parameters: python_version: type: string @@ -53,7 +53,7 @@ jobs: arangodb_version: type: string machine: - image: ubuntu-2204:2023.10.1 + image: ubuntu-2004 steps: - checkout @@ -87,7 +87,7 @@ jobs: if [ << parameters.arangodb_config >> = "cluster" ]; then args+=("--cluster" "--port=8539" "--port=8549") fi - + if [ << parameters.arangodb_license >> = "enterprise" ]; then args+=("--enterprise") fi From b0f4b7d001d4a96a0aaa8761aef2bd165601bb06 Mon Sep 17 00:00:00 2001 From: Anthony Mahanna Date: Fri, 10 Nov 2023 12:29:35 -0500 Subject: [PATCH 28/34] Update config.yml --- .circleci/config.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index eaab9378..45288ee8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -42,7 +42,6 @@ jobs: command: mypy ./arango test: - resource_class: medium parameters: python_version: type: string From 082715ee4920ed25bd30809c9fac456b78aa1970 Mon Sep 17 00:00:00 2001 From: Anthony Mahanna Date: Fri, 10 Nov 2023 12:30:28 -0500 Subject: [PATCH 29/34] Update config.yml --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 45288ee8..bfb8e842 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -52,7 +52,7 @@ jobs: arangodb_version: type: string machine: - image: ubuntu-2004 + image: ubuntu-2004:current steps: - checkout From d617d256b59b5f84cb9d8273dff01a83606e654f Mon Sep 17 00:00:00 2001 From: Anthony Mahanna Date: Fri, 10 Nov 2023 12:53:19 -0500 Subject: [PATCH 30/34] Update config.yml --- .circleci/config.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index bfb8e842..dbc3aa5a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -65,10 +65,16 @@ jobs: - restore_cache: key: pip-and-local-cache + # TODO: Revisit this bottleneck - run: name: Setup Python command: | - pyenv install << parameters.python_version >> + pyenv update + if ! pyenv versions | grep -q << parameters.python_version >>; then + pyenv install << parameters.python_version >> + else + echo "<< parameters.python_version >> already installed" + fi pyenv global << parameters.python_version >> - run: From 31c44f546d56e257320ea6d88c873e2682fc3b96 Mon Sep 17 00:00:00 2001 From: Anthony Mahanna Date: Fri, 10 Nov 2023 12:59:17 -0500 Subject: [PATCH 31/34] Update config.yml --- .circleci/config.yml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index dbc3aa5a..ac0a86ae 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -52,7 +52,7 @@ jobs: arangodb_version: type: string machine: - image: ubuntu-2004:current + image: ubuntu-2204:current steps: - checkout @@ -69,12 +69,8 @@ jobs: - run: name: Setup Python command: | - pyenv update - if ! pyenv versions | grep -q << parameters.python_version >>; then - pyenv install << parameters.python_version >> - else - echo "<< parameters.python_version >> already installed" - fi + pyenv --version + pyenv install -f << parameters.python_version >> pyenv global << parameters.python_version >> - run: From cf491409b60ccf99e9fe80a3872440989f6fc2f8 Mon Sep 17 00:00:00 2001 From: Anthony Mahanna Date: Fri, 10 Nov 2023 13:05:17 -0500 Subject: [PATCH 32/34] Update config.yml --- .circleci/config.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ac0a86ae..c47f4f7d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -10,7 +10,8 @@ workflows: - test: matrix: parameters: - python_version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + # TODO: Revisit why pyenv doesn't recognize 3.12 + python_version: ["3.8", "3.9", "3.10", "3.11"] # "3.12" arangodb_config: ["single", "cluster"] arangodb_license: ["community", "enterprise"] arangodb_version: ["3.10.10", "3.11.4", "latest"] @@ -102,4 +103,9 @@ jobs: - store_test_results: path: test-results - # - coveralls/upload + - run: + name: Upload to Coveralls + command: | + if [ "<< parameters.python_version >>" = "3.11" && "<< parameters.arangodb_config >>" == "single" && "<< parameters.arangodb_license >>" == "community" ]; then + coveralls/upload + fi From 44961dbff6ec25368a273aa5fa8d22e0a3b11727 Mon Sep 17 00:00:00 2001 From: Anthony Mahanna Date: Fri, 10 Nov 2023 13:16:18 -0500 Subject: [PATCH 33/34] Update config.yml --- .circleci/config.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c47f4f7d..ff6b5e4b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,7 +1,7 @@ version: 2.1 -# orbs: -# coveralls: coveralls/coveralls@2.2.1 +orbs: + coveralls: coveralls/coveralls@2.2.1 workflows: ci: @@ -52,6 +52,8 @@ jobs: type: string arangodb_version: type: string + # TODO: Reconsider using a docker image instead of a machine + # i.e cimg/python:<< parameters.python_version >> machine: image: ubuntu-2204:current steps: @@ -106,6 +108,6 @@ jobs: - run: name: Upload to Coveralls command: | - if [ "<< parameters.python_version >>" = "3.11" && "<< parameters.arangodb_config >>" == "single" && "<< parameters.arangodb_license >>" == "community" ]; then + if [ "<< parameters.python_version >>" = "3.11" && "<< parameters.arangodb_config >>" = "single" && "<< parameters.arangodb_license >>" = "community" && "<< parameters.arangodb_version >>" = "latest" ]; then coveralls/upload fi From 3c26c4715a3c6a808e712f32a9ca4d6d0d4c47b2 Mon Sep 17 00:00:00 2001 From: Anthony Mahanna Date: Fri, 10 Nov 2023 13:18:19 -0500 Subject: [PATCH 34/34] Update config.yml --- .circleci/config.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ff6b5e4b..a75aba3e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,7 +1,7 @@ version: 2.1 -orbs: - coveralls: coveralls/coveralls@2.2.1 +# orbs: +# coveralls: coveralls/coveralls@2.2.1 workflows: ci: @@ -105,9 +105,9 @@ jobs: - store_test_results: path: test-results - - run: - name: Upload to Coveralls - command: | - if [ "<< parameters.python_version >>" = "3.11" && "<< parameters.arangodb_config >>" = "single" && "<< parameters.arangodb_license >>" = "community" && "<< parameters.arangodb_version >>" = "latest" ]; then - coveralls/upload - fi + # - run: + # name: Upload to Coveralls + # command: | + # if [ "<< parameters.python_version >>" = "3.11" && "<< parameters.arangodb_config >>" = "single" && "<< parameters.arangodb_license >>" = "community" && "<< parameters.arangodb_version >>" = "latest" ]; then + # coveralls/upload + # fi