diff --git a/.github/workflows/automate-metrics.yaml b/.github/workflows/automate-metrics.yaml deleted file mode 100644 index 5d37daca..00000000 --- a/.github/workflows/automate-metrics.yaml +++ /dev/null @@ -1,34 +0,0 @@ -name: Update User Metrics - -on: - workflow_call: - -env: - PORTAL_ID: ${{ secrets.PORTAL_ID }} - FOUNDATIONS_ID: ${{ secrets.FOUNDATIONS_ID }} - COOKBOOKS_ID: ${{ secrets.COOKBOOKS_ID }} - PRIVATE_KEY: ${{ secrets.PRIVATE_KEY }} - PRIVATE_KEY_ID: ${{ secrets.PRIVATE_KEY_ID }} - -jobs: - main: - runs-on: macos-latest - steps: - - uses: actions/checkout@v3 - - name: Set-Up Virtual Environment - run: | - python -m venv analytics-api - source analytics-api/bin/activate - pip install google-analytics-data - - - name: Write Metrics to JSON - run: | - source analytics-api/bin/activate - curl -O https://raw.githubusercontent.com/jukent/projectpythia.github.io/main/.github/workflows/get-metrics.py - python get-metrics.py - - - name: Write Markdown File - run: | - source analytics-api/bin/activate - curl -O https://raw.githubusercontent.com/jukent/projectpythia.github.io/main/.github/workflows/write-metrics-md.py - python write-metrics-md.py diff --git a/.github/workflows/get-metrics.py b/.github/workflows/get-metrics.py index c519c6b3..38ca9f41 100644 --- a/.github/workflows/get-metrics.py +++ b/.github/workflows/get-metrics.py @@ -1,36 +1,44 @@ import json +import math import os -import base64 -import hashlib +import cartopy +import matplotlib.cm as cm +import matplotlib.colors as colors +import matplotlib.pyplot as plt +import numpy as np from google.analytics.data_v1beta import BetaAnalyticsDataClient -from google.analytics.data_v1beta.types import DateRange, Metric, RunReportRequest +from google.analytics.data_v1beta.types import DateRange, Dimension, Metric, RunReportRequest -PORTAL_ID = os.environ.get('PORTAL_ID') -FOUNDATIONS_ID = os.environ.get('FOUNDATIONS_ID') -COOKBOOKS_ID = os.environ.get('COOKBOOKS_ID') +PORTAL_ID = os.environ['PORTAL_ID'] +FOUNDATIONS_ID = os.environ['FOUNDATIONS_ID'] +COOKBOOKS_ID = os.environ['COOKBOOKS_ID'] PRIVATE_KEY_ID = os.environ.get('PRIVATE_KEY_ID') -PRIVATE_KEY = os.environ.get('PRIVATE_KEY').replace('$','\n') +PRIVATE_KEY = os.environ.get('PRIVATE_KEY').replace('$', '\n') credentials_dict = { - "type": "service_account", - "project_id": "cisl-vast-pythia", - "private_key_id": PRIVATE_KEY_ID, - "private_key": PRIVATE_KEY, - "client_email": "pythia-metrics-api@cisl-vast-pythia.iam.gserviceaccount.com", - "client_id": "113402578114110723940", - "auth_uri": "https://accounts.google.com/o/oauth2/auth", - "token_uri": "https://oauth2.googleapis.com/token", - "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", - "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/pythia-metrics-api%40cisl-vast-pythia.iam.gserviceaccount.com", - "universe_domain": "googleapis.com" + 'type': 'service_account', + 'project_id': 'cisl-vast-pythia', + 'private_key_id': PRIVATE_KEY_ID, + 'private_key': PRIVATE_KEY, + 'client_email': 'pythia-metrics-api@cisl-vast-pythia.iam.gserviceaccount.com', + 'client_id': '113402578114110723940', + 'auth_uri': 'https://accounts.google.com/o/oauth2/auth', + 'token_uri': 'https://oauth2.googleapis.com/token', + 'auth_provider_x509_cert_url': 'https://www.googleapis.com/oauth2/v1/certs', + 'client_x509_cert_url': 'https://www.googleapis.com/robot/v1/metadata/x509/pythia-metrics-api%40cisl-vast-pythia.iam.gserviceaccount.com', + 'universe_domain': 'googleapis.com', } client = BetaAnalyticsDataClient.from_service_account_info(credentials_dict) -def _run_total_users_report(property_id): +def _format_rounding(value): + return f"{round(value / 1000, 1):.1f}K" + + +def run_total_users_report(property_id): request = RunReportRequest( property=f'properties/{property_id}', dimensions=[], @@ -44,18 +52,163 @@ def _run_total_users_report(property_id): for row in response.rows: total_users += int(row.metric_values[0].value) - return total_users + return _format_rounding(total_users) + + +def _run_top_pages_report(property_id): + request = RunReportRequest( + property=f'properties/{property_id}', + dimensions=[Dimension(name='pageTitle')], + date_ranges=[DateRange(start_date='2020-03-31', end_date='today')], + metrics=[Metric(name='screenPageViews')], + ) + + response = client.run_report(request) + + page_views = {} + for row in response.rows: + page = row.dimension_values[0].value + views = int(row.metric_values[0].value) + page_views[page] = views + + top_10_pages = sorted(page_views.items(), key=lambda item: item[1], reverse=True)[:10] + top_10_pages_dict = {page: views for page, views in top_10_pages} + + return top_10_pages_dict + + +def plot_top_pages(foundations_id, cookbooks_id): + foundations_page_views = _run_top_pages_report(foundations_id) + foundations_pages = [] + foundations_sorted = {k: v for k, v in sorted(foundations_page_views.items(), key=lambda item: item[1])} + foundations_views = foundations_sorted.values() + for key in foundations_sorted: + newkey = key.split('β€”')[0] + foundations_pages.append(newkey) + + cookbooks_page_views = _run_top_pages_report(cookbooks_id) + cookbooks_pages = [] + cookbooks_sorted = {k: v for k, v in sorted(cookbooks_page_views.items(), key=lambda item: item[1])} + cookbooks_views = cookbooks_sorted.values() + for key in cookbooks_page_views: + newkey = key.split('β€”')[0] + cookbooks_pages.insert(0, newkey) + + pages = cookbooks_pages + foundations_pages + + fig, ax = plt.subplots() + plt.title('Most Popular Pages') + + views_max = int(math.ceil(max(foundations_views) / 10000.0)) * 10000 + ax.set_xlim([0, views_max]) + + y = np.arange(10) + y2 = np.arange(11, 21) + y3 = np.append(y, y2) + + bar1 = ax.barh(y2, foundations_views, align='center', label='Foundations', color='royalblue') + bar2 = ax.barh(y, cookbooks_views, align='center', label='Cookbooks', color='indianred') + + ax.set_yticks(y3, labels=pages) + + ax.bar_label(bar1, fmt=_format_rounding) + ax.bar_label(bar2, fmt=_format_rounding) + + plt.legend() + plt.savefig('bypage.png', bbox_inches='tight') + + +def _run_usersXcountry_report(foundations_id): + request = RunReportRequest( + property=f'properties/{foundations_id}', + dimensions=[Dimension(name='country')], + metrics=[Metric(name='activeUsers')], + date_ranges=[DateRange(start_date='2020-03-31', end_date='today')], + ) + + response = client.run_report(request) + + user_by_country = {} + for row in response.rows: + country = row.dimension_values[0].value + users = int(row.metric_values[0].value) + user_by_country[country] = user_by_country.get(country, 0) + users + + return user_by_country + + +def plot_usersXcountry(foundations_id): + users_by_country = _run_usersXcountry_report(foundations_id) + + dict_api2cartopy = { + 'Tanzania': 'United Republic of Tanzania', + 'United States': 'United States of America', + 'Congo - Kinshasa': 'Democratic Republic of the Congo', + 'Bahamas': 'The Bahamas', + 'Timor-Leste': 'East Timor', + 'C\u00f4te d\u2019Ivoire': 'Ivory Coast', + 'Bosnia & Herzegovina': 'Bosnia and Herzegovina', + 'Serbia': 'Republic of Serbia', + 'Trinidad & Tobago': 'Trinidad and Tobago', + } + + for key in dict_api2cartopy: + users_by_country[dict_api2cartopy[key]] = users_by_country.pop(key) + + top_10_countries = sorted(users_by_country.items(), key=lambda item: item[1], reverse=True)[:10] + top_10_text = '\n'.join( + f"{country}: {_format_rounding(value)}" for i, (country, value) in enumerate(top_10_countries) + ) + + fig = plt.figure(figsize=(10, 4)) + ax = plt.axes(projection=cartopy.crs.PlateCarree(), frameon=False) + ax.set_title('Pythia Foundations Unique Users by Country') + + shapefile = cartopy.io.shapereader.natural_earth(category='cultural', resolution='110m', name='admin_0_countries') + reader = cartopy.io.shapereader.Reader(shapefile) + countries = reader.records() + + colormap = plt.get_cmap('Blues') + colormap.set_extremes(under='grey') + vmax = int(math.ceil(max(users_by_country.values()) / 100.0)) * 100 + norm = colors.LogNorm(vmin=1, vmax=vmax) + mappable = cm.ScalarMappable(norm=norm, cmap=colormap) + + for country in countries: + country_name = country.attributes['SOVEREIGNT'] + if country_name in users_by_country.keys(): + facecolor = colormap((users_by_country[country_name] / 105)) + + ax.add_geometries( + [country.geometry], cartopy.crs.PlateCarree(), facecolor=facecolor, edgecolor='white', linewidth=0.7 + ) + else: + ax.add_geometries( + [country.geometry], cartopy.crs.PlateCarree(), facecolor='grey', edgecolor='white', linewidth=0.7 + ) + + cax = fig.add_axes([0.1, -0.015, 0.67, 0.03]) + fig.colorbar(mappable=mappable, cax=cax, spacing='uniform', orientation='horizontal', extend='min') + + props = dict(boxstyle='round', facecolor='white', edgecolor='white') + ax.text(1.01, 0.5, top_10_text, transform=ax.transAxes, fontsize=9, verticalalignment='center', bbox=props) + + plt.tight_layout() + plt.savefig('bycountry.png', bbox_inches='tight') def get_metrics(): metrics_dict = {} - metrics_dict['Portal'] = _run_total_users_report(str(PORTAL_ID)) - metrics_dict['Foundations'] = _run_total_users_report(str(FOUNDATIONS_ID)) - metrics_dict['Cookbooks'] = _run_total_users_report(str(COOKBOOKS_ID)) - + metrics_dict['Portal'] = run_total_users_report(str(PORTAL_ID)) + metrics_dict['Foundations'] = run_total_users_report(str(FOUNDATIONS_ID)) + metrics_dict['Cookbooks'] = run_total_users_report(str(COOKBOOKS_ID)) with open('user_metrics.json', 'w') as outfile: json.dump(metrics_dict, outfile) + plot_top_pages(str(FOUNDATIONS_ID), str(COOKBOOKS_ID)) + + plot_usersXcountry(str(FOUNDATIONS_ID)) + if __name__ == '__main__': get_metrics() diff --git a/.github/workflows/nightly-build.yaml b/.github/workflows/nightly-build.yaml index c46222c6..321411ae 100644 --- a/.github/workflows/nightly-build.yaml +++ b/.github/workflows/nightly-build.yaml @@ -5,9 +5,30 @@ on: schedule: - cron: '0 0 * * *' # Daily β€œAt 00:00” +env: + PORTAL_ID: ${{ secrets.PORTAL_ID }} + FOUNDATIONS_ID: ${{ secrets.FOUNDATIONS_ID }} + COOKBOOKS_ID: ${{ secrets.COOKBOOKS_ID }} + PRIVATE_KEY: ${{ secrets.PRIVATE_KEY }} + PRIVATE_KEY_ID: ${{ secrets.PRIVATE_KEY_ID }} + jobs: - update-metrics: - uses: ./.github/workflows/automate-metrics.yaml + automate-metrics: + runs-on: macos-latest + steps: + - uses: actions/checkout@v3 + - name: Automate Metrics + run: | + python -m venv analytics-api + source analytics-api/bin/activate + pip install google-analytics-data + conda install cartopy matplotlib + + curl -O https://raw.githubusercontent.com/jukent/projectpythia.github.io/main/.github/workflows/get-metrics.py + curl -O https://raw.githubusercontent.com/jukent/projectpythia.github.io/main/.github/workflows/write-metrics-md.py + + python get-metrics.py + python write-metrics-md.py build: if: ${{ github.repository_owner == 'ProjectPythia' }} diff --git a/.github/workflows/publish-site.yaml b/.github/workflows/publish-site.yaml index 4c06ca47..b5308b33 100644 --- a/.github/workflows/publish-site.yaml +++ b/.github/workflows/publish-site.yaml @@ -7,9 +7,30 @@ on: - main workflow_dispatch: +env: + PORTAL_ID: ${{ secrets.PORTAL_ID }} + FOUNDATIONS_ID: ${{ secrets.FOUNDATIONS_ID }} + COOKBOOKS_ID: ${{ secrets.COOKBOOKS_ID }} + PRIVATE_KEY: ${{ secrets.PRIVATE_KEY }} + PRIVATE_KEY_ID: ${{ secrets.PRIVATE_KEY_ID }} + jobs: - update-metrics: - uses: ./.github/workflows/automate-metrics.yaml + automate-metrics: + runs-on: macos-latest + steps: + - uses: actions/checkout@v3 + - name: Automate Metrics + run: | + python -m venv analytics-api + source analytics-api/bin/activate + pip install google-analytics-data + conda install cartopy matplotlib + + curl -O https://raw.githubusercontent.com/jukent/projectpythia.github.io/main/.github/workflows/get-metrics.py + curl -O https://raw.githubusercontent.com/jukent/projectpythia.github.io/main/.github/workflows/write-metrics-md.py + + python get-metrics.py + python write-metrics-md.py build: uses: ProjectPythia/cookbook-actions/.github/workflows/build-book.yaml@main diff --git a/.github/workflows/trigger-preview.yaml b/.github/workflows/trigger-preview.yaml index fda6c9a2..dcb866db 100644 --- a/.github/workflows/trigger-preview.yaml +++ b/.github/workflows/trigger-preview.yaml @@ -7,9 +7,30 @@ on: - requested - completed +env: + PORTAL_ID: ${{ secrets.PORTAL_ID }} + FOUNDATIONS_ID: ${{ secrets.FOUNDATIONS_ID }} + COOKBOOKS_ID: ${{ secrets.COOKBOOKS_ID }} + PRIVATE_KEY: ${{ secrets.PRIVATE_KEY }} + PRIVATE_KEY_ID: ${{ secrets.PRIVATE_KEY_ID }} + jobs: - update-metrics: - uses: ./.github/workflows/automate-metrics.yaml + automate-metrics: + runs-on: macos-latest + steps: + - uses: actions/checkout@v3 + - name: Automate Metrics + run: | + python -m venv analytics-api + source analytics-api/bin/activate + pip install google-analytics-data + conda install cartopy matplotlib + + curl -O https://raw.githubusercontent.com/jukent/projectpythia.github.io/main/.github/workflows/get-metrics.py + curl -O https://raw.githubusercontent.com/jukent/projectpythia.github.io/main/.github/workflows/write-metrics-md.py + + python get-metrics.py + python write-metrics-md.py find-pull-request: uses: ProjectPythia/cookbook-actions/.github/workflows/find-pull-request.yaml@main diff --git a/.github/workflows/trigger-site-build.yaml b/.github/workflows/trigger-site-build.yaml index 8902e634..4d49dfe3 100644 --- a/.github/workflows/trigger-site-build.yaml +++ b/.github/workflows/trigger-site-build.yaml @@ -2,9 +2,30 @@ name: trigger-site-build on: pull_request: +env: + PORTAL_ID: ${{ secrets.PORTAL_ID }} + FOUNDATIONS_ID: ${{ secrets.FOUNDATIONS_ID }} + COOKBOOKS_ID: ${{ secrets.COOKBOOKS_ID }} + PRIVATE_KEY: ${{ secrets.PRIVATE_KEY }} + PRIVATE_KEY_ID: ${{ secrets.PRIVATE_KEY_ID }} + jobs: - update-metrics: - uses: ./.github/workflows/automate-metrics.yaml + automate-metrics: + runs-on: macos-latest + steps: + - uses: actions/checkout@v3 + - name: Automate Metrics + run: | + python -m venv analytics-api + source analytics-api/bin/activate + pip install google-analytics-data + conda install cartopy matplotlib + + curl -O https://raw.githubusercontent.com/jukent/projectpythia.github.io/main/.github/workflows/get-metrics.py + curl -O https://raw.githubusercontent.com/jukent/projectpythia.github.io/main/.github/workflows/write-metrics-md.py + + python get-metrics.py + python write-metrics-md.py build: uses: ProjectPythia/cookbook-actions/.github/workflows/build-book.yaml@main diff --git a/.github/workflows/write-metrics-md.py b/.github/workflows/write-metrics-md.py index c20d7b59..961feb10 100644 --- a/.github/workflows/write-metrics-md.py +++ b/.github/workflows/write-metrics-md.py @@ -2,7 +2,7 @@ import json -def process_user_data(user_data_file, markdown_file): +def process_user_data(user_data_file, bar_plot_file, map_plot_file, markdown_file): """ Reads user data from a JSON file and writes it to a markdown file. @@ -18,15 +18,21 @@ def process_user_data(user_data_file, markdown_file): # Write processed data to markdown file with open(markdown_file, 'w') as f: f.write('# Metrics \n\n') + f.write(f'Last Updated: {now}\n\n') f.write('Total Users:\n\n') for key in user_data: - f.write(f'{key}: {user_data[key]}\n') + f.write(f'{key}: {(user_data[key])}\n') + f.write('\n') + f.write('![Top Pages](bar_plot_file)') + f.write('\n\n') + f.write('![Users by Country](../.github/workflows/map_plot_file)') f.write('\n') - f.write(f'Last Updated: {now}\n') f.close() if __name__ == '__main__': user_data_file = 'user_metrics.json' + bar_plot_file = '../.github/workflows/bypage.png' + map_plot_file = '../.github/workflows/bycountry.png' markdown_file = 'portal/metrics.md' - process_user_data(user_data_file, markdown_file) + process_user_data(user_data_file, bar_plot_file, map_plot_file, markdown_file) diff --git a/.gitignore b/.gitignore index 1babb552..4fb65167 100644 --- a/.gitignore +++ b/.gitignore @@ -138,3 +138,8 @@ portal/code_of_conduct.md portal/resource-gallery.md portal/resource-gallery/*.md resource-gallery-submission-input.json + +# Analytics +.github/workflows/analytics-api/ +.github/workflows/*.json +.github/workflows/*.png