Skip to content

Metrics api #125

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Mar 18, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 0 additions & 34 deletions .github/workflows/automate-metrics.yaml

This file was deleted.

201 changes: 177 additions & 24 deletions .github/workflows/get-metrics.py
Original file line number Diff line number Diff line change
@@ -1,36 +1,44 @@
import json
import math
import os
import base64
import hashlib

import cartopy
import matplotlib.cm as cm
import matplotlib.colors as colors
import matplotlib.pyplot as plt
import numpy as np
from google.analytics.data_v1beta import BetaAnalyticsDataClient
from google.analytics.data_v1beta.types import DateRange, Metric, RunReportRequest
from google.analytics.data_v1beta.types import DateRange, Dimension, Metric, RunReportRequest

PORTAL_ID = os.environ.get('PORTAL_ID')
FOUNDATIONS_ID = os.environ.get('FOUNDATIONS_ID')
COOKBOOKS_ID = os.environ.get('COOKBOOKS_ID')
PORTAL_ID = os.environ['PORTAL_ID']
FOUNDATIONS_ID = os.environ['FOUNDATIONS_ID']
COOKBOOKS_ID = os.environ['COOKBOOKS_ID']

PRIVATE_KEY_ID = os.environ.get('PRIVATE_KEY_ID')
PRIVATE_KEY = os.environ.get('PRIVATE_KEY').replace('$','\n')
PRIVATE_KEY = os.environ.get('PRIVATE_KEY').replace('$', '\n')

credentials_dict = {
"type": "service_account",
"project_id": "cisl-vast-pythia",
"private_key_id": PRIVATE_KEY_ID,
"private_key": PRIVATE_KEY,
"client_email": "[email protected]",
"client_id": "113402578114110723940",
"auth_uri": "https://accounts.google.com/o/oauth2/auth",
"token_uri": "https://oauth2.googleapis.com/token",
"auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
"client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/pythia-metrics-api%40cisl-vast-pythia.iam.gserviceaccount.com",
"universe_domain": "googleapis.com"
'type': 'service_account',
'project_id': 'cisl-vast-pythia',
'private_key_id': PRIVATE_KEY_ID,
'private_key': PRIVATE_KEY,
'client_email': '[email protected]',
'client_id': '113402578114110723940',
'auth_uri': 'https://accounts.google.com/o/oauth2/auth',
'token_uri': 'https://oauth2.googleapis.com/token',
'auth_provider_x509_cert_url': 'https://www.googleapis.com/oauth2/v1/certs',
'client_x509_cert_url': 'https://www.googleapis.com/robot/v1/metadata/x509/pythia-metrics-api%40cisl-vast-pythia.iam.gserviceaccount.com',
'universe_domain': 'googleapis.com',
}

client = BetaAnalyticsDataClient.from_service_account_info(credentials_dict)


def _run_total_users_report(property_id):
def _format_rounding(value):
return f"{round(value / 1000, 1):.1f}K"


def run_total_users_report(property_id):
request = RunReportRequest(
property=f'properties/{property_id}',
dimensions=[],
Expand All @@ -44,18 +52,163 @@ def _run_total_users_report(property_id):
for row in response.rows:
total_users += int(row.metric_values[0].value)

return total_users
return _format_rounding(total_users)


def _run_top_pages_report(property_id):
request = RunReportRequest(
property=f'properties/{property_id}',
dimensions=[Dimension(name='pageTitle')],
date_ranges=[DateRange(start_date='2020-03-31', end_date='today')],
metrics=[Metric(name='screenPageViews')],
)

response = client.run_report(request)

page_views = {}
for row in response.rows:
page = row.dimension_values[0].value
views = int(row.metric_values[0].value)
page_views[page] = views

top_10_pages = sorted(page_views.items(), key=lambda item: item[1], reverse=True)[:10]
top_10_pages_dict = {page: views for page, views in top_10_pages}

return top_10_pages_dict


def plot_top_pages(foundations_id, cookbooks_id):
foundations_page_views = _run_top_pages_report(foundations_id)
foundations_pages = []
foundations_sorted = {k: v for k, v in sorted(foundations_page_views.items(), key=lambda item: item[1])}
foundations_views = foundations_sorted.values()
for key in foundations_sorted:
newkey = key.split('—')[0]
foundations_pages.append(newkey)

cookbooks_page_views = _run_top_pages_report(cookbooks_id)
cookbooks_pages = []
cookbooks_sorted = {k: v for k, v in sorted(cookbooks_page_views.items(), key=lambda item: item[1])}
cookbooks_views = cookbooks_sorted.values()
for key in cookbooks_page_views:
newkey = key.split('—')[0]
cookbooks_pages.insert(0, newkey)

pages = cookbooks_pages + foundations_pages

fig, ax = plt.subplots()
plt.title('Most Popular Pages')

views_max = int(math.ceil(max(foundations_views) / 10000.0)) * 10000
ax.set_xlim([0, views_max])

y = np.arange(10)
y2 = np.arange(11, 21)
y3 = np.append(y, y2)

bar1 = ax.barh(y2, foundations_views, align='center', label='Foundations', color='royalblue')
bar2 = ax.barh(y, cookbooks_views, align='center', label='Cookbooks', color='indianred')

ax.set_yticks(y3, labels=pages)

ax.bar_label(bar1, fmt=_format_rounding)
ax.bar_label(bar2, fmt=_format_rounding)

plt.legend()
plt.savefig('bypage.png', bbox_inches='tight')


def _run_usersXcountry_report(foundations_id):
request = RunReportRequest(
property=f'properties/{foundations_id}',
dimensions=[Dimension(name='country')],
metrics=[Metric(name='activeUsers')],
date_ranges=[DateRange(start_date='2020-03-31', end_date='today')],
)

response = client.run_report(request)

user_by_country = {}
for row in response.rows:
country = row.dimension_values[0].value
users = int(row.metric_values[0].value)
user_by_country[country] = user_by_country.get(country, 0) + users

return user_by_country


def plot_usersXcountry(foundations_id):
users_by_country = _run_usersXcountry_report(foundations_id)

dict_api2cartopy = {
'Tanzania': 'United Republic of Tanzania',
'United States': 'United States of America',
'Congo - Kinshasa': 'Democratic Republic of the Congo',
'Bahamas': 'The Bahamas',
'Timor-Leste': 'East Timor',
'C\u00f4te d\u2019Ivoire': 'Ivory Coast',
'Bosnia & Herzegovina': 'Bosnia and Herzegovina',
'Serbia': 'Republic of Serbia',
'Trinidad & Tobago': 'Trinidad and Tobago',
}

for key in dict_api2cartopy:
users_by_country[dict_api2cartopy[key]] = users_by_country.pop(key)

top_10_countries = sorted(users_by_country.items(), key=lambda item: item[1], reverse=True)[:10]
top_10_text = '\n'.join(
f"{country}: {_format_rounding(value)}" for i, (country, value) in enumerate(top_10_countries)
)

fig = plt.figure(figsize=(10, 4))
ax = plt.axes(projection=cartopy.crs.PlateCarree(), frameon=False)
ax.set_title('Pythia Foundations Unique Users by Country')

shapefile = cartopy.io.shapereader.natural_earth(category='cultural', resolution='110m', name='admin_0_countries')
reader = cartopy.io.shapereader.Reader(shapefile)
countries = reader.records()

colormap = plt.get_cmap('Blues')
colormap.set_extremes(under='grey')
vmax = int(math.ceil(max(users_by_country.values()) / 100.0)) * 100
norm = colors.LogNorm(vmin=1, vmax=vmax)
mappable = cm.ScalarMappable(norm=norm, cmap=colormap)

for country in countries:
country_name = country.attributes['SOVEREIGNT']
if country_name in users_by_country.keys():
facecolor = colormap((users_by_country[country_name] / 105))

ax.add_geometries(
[country.geometry], cartopy.crs.PlateCarree(), facecolor=facecolor, edgecolor='white', linewidth=0.7
)
else:
ax.add_geometries(
[country.geometry], cartopy.crs.PlateCarree(), facecolor='grey', edgecolor='white', linewidth=0.7
)

cax = fig.add_axes([0.1, -0.015, 0.67, 0.03])
fig.colorbar(mappable=mappable, cax=cax, spacing='uniform', orientation='horizontal', extend='min')

props = dict(boxstyle='round', facecolor='white', edgecolor='white')
ax.text(1.01, 0.5, top_10_text, transform=ax.transAxes, fontsize=9, verticalalignment='center', bbox=props)

plt.tight_layout()
plt.savefig('bycountry.png', bbox_inches='tight')


def get_metrics():
metrics_dict = {}
metrics_dict['Portal'] = _run_total_users_report(str(PORTAL_ID))
metrics_dict['Foundations'] = _run_total_users_report(str(FOUNDATIONS_ID))
metrics_dict['Cookbooks'] = _run_total_users_report(str(COOKBOOKS_ID))

metrics_dict['Portal'] = run_total_users_report(str(PORTAL_ID))
metrics_dict['Foundations'] = run_total_users_report(str(FOUNDATIONS_ID))
metrics_dict['Cookbooks'] = run_total_users_report(str(COOKBOOKS_ID))
with open('user_metrics.json', 'w') as outfile:
json.dump(metrics_dict, outfile)

plot_top_pages(str(FOUNDATIONS_ID), str(COOKBOOKS_ID))

plot_usersXcountry(str(FOUNDATIONS_ID))


if __name__ == '__main__':
get_metrics()
25 changes: 23 additions & 2 deletions .github/workflows/nightly-build.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,30 @@ on:
schedule:
- cron: '0 0 * * *' # Daily “At 00:00”

env:
PORTAL_ID: ${{ secrets.PORTAL_ID }}
FOUNDATIONS_ID: ${{ secrets.FOUNDATIONS_ID }}
COOKBOOKS_ID: ${{ secrets.COOKBOOKS_ID }}
PRIVATE_KEY: ${{ secrets.PRIVATE_KEY }}
PRIVATE_KEY_ID: ${{ secrets.PRIVATE_KEY_ID }}

jobs:
update-metrics:
uses: ./.github/workflows/automate-metrics.yaml
automate-metrics:
runs-on: macos-latest
steps:
- uses: actions/checkout@v3
- name: Automate Metrics
run: |
python -m venv analytics-api
source analytics-api/bin/activate
pip install google-analytics-data
conda install cartopy matplotlib

curl -O https://raw.githubusercontent.com/jukent/projectpythia.github.io/main/.github/workflows/get-metrics.py
curl -O https://raw.githubusercontent.com/jukent/projectpythia.github.io/main/.github/workflows/write-metrics-md.py

python get-metrics.py
python write-metrics-md.py

build:
if: ${{ github.repository_owner == 'ProjectPythia' }}
Expand Down
25 changes: 23 additions & 2 deletions .github/workflows/publish-site.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,30 @@ on:
- main
workflow_dispatch:

env:
PORTAL_ID: ${{ secrets.PORTAL_ID }}
FOUNDATIONS_ID: ${{ secrets.FOUNDATIONS_ID }}
COOKBOOKS_ID: ${{ secrets.COOKBOOKS_ID }}
PRIVATE_KEY: ${{ secrets.PRIVATE_KEY }}
PRIVATE_KEY_ID: ${{ secrets.PRIVATE_KEY_ID }}

jobs:
update-metrics:
uses: ./.github/workflows/automate-metrics.yaml
automate-metrics:
runs-on: macos-latest
steps:
- uses: actions/checkout@v3
- name: Automate Metrics
run: |
python -m venv analytics-api
source analytics-api/bin/activate
pip install google-analytics-data
conda install cartopy matplotlib

curl -O https://raw.githubusercontent.com/jukent/projectpythia.github.io/main/.github/workflows/get-metrics.py
curl -O https://raw.githubusercontent.com/jukent/projectpythia.github.io/main/.github/workflows/write-metrics-md.py

python get-metrics.py
python write-metrics-md.py

build:
uses: ProjectPythia/cookbook-actions/.github/workflows/build-book.yaml@main
Expand Down
25 changes: 23 additions & 2 deletions .github/workflows/trigger-preview.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,30 @@ on:
- requested
- completed

env:
PORTAL_ID: ${{ secrets.PORTAL_ID }}
FOUNDATIONS_ID: ${{ secrets.FOUNDATIONS_ID }}
COOKBOOKS_ID: ${{ secrets.COOKBOOKS_ID }}
PRIVATE_KEY: ${{ secrets.PRIVATE_KEY }}
PRIVATE_KEY_ID: ${{ secrets.PRIVATE_KEY_ID }}

jobs:
update-metrics:
uses: ./.github/workflows/automate-metrics.yaml
automate-metrics:
runs-on: macos-latest
steps:
- uses: actions/checkout@v3
- name: Automate Metrics
run: |
python -m venv analytics-api
source analytics-api/bin/activate
pip install google-analytics-data
conda install cartopy matplotlib

curl -O https://raw.githubusercontent.com/jukent/projectpythia.github.io/main/.github/workflows/get-metrics.py
curl -O https://raw.githubusercontent.com/jukent/projectpythia.github.io/main/.github/workflows/write-metrics-md.py

python get-metrics.py
python write-metrics-md.py

find-pull-request:
uses: ProjectPythia/cookbook-actions/.github/workflows/find-pull-request.yaml@main
Expand Down
25 changes: 23 additions & 2 deletions .github/workflows/trigger-site-build.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,30 @@ name: trigger-site-build
on:
pull_request:

env:
PORTAL_ID: ${{ secrets.PORTAL_ID }}
FOUNDATIONS_ID: ${{ secrets.FOUNDATIONS_ID }}
COOKBOOKS_ID: ${{ secrets.COOKBOOKS_ID }}
PRIVATE_KEY: ${{ secrets.PRIVATE_KEY }}
PRIVATE_KEY_ID: ${{ secrets.PRIVATE_KEY_ID }}

jobs:
update-metrics:
uses: ./.github/workflows/automate-metrics.yaml
automate-metrics:
runs-on: macos-latest
steps:
- uses: actions/checkout@v3
- name: Automate Metrics
run: |
python -m venv analytics-api
source analytics-api/bin/activate
pip install google-analytics-data
conda install cartopy matplotlib

curl -O https://raw.githubusercontent.com/jukent/projectpythia.github.io/main/.github/workflows/get-metrics.py
curl -O https://raw.githubusercontent.com/jukent/projectpythia.github.io/main/.github/workflows/write-metrics-md.py

python get-metrics.py
python write-metrics-md.py

build:
uses: ProjectPythia/cookbook-actions/.github/workflows/build-book.yaml@main
Expand Down
Loading