Skip to content

Commit 54030ec

Browse files
Zhenshan-JinZhenshan Jin
authored and
atqy
committed
Add example notebook for model governance model card (aws#217)
Co-authored-by: Zhenshan Jin <[email protected]>
1 parent 3a41ad1 commit 54030ec

File tree

5 files changed

+897
-0
lines changed

5 files changed

+897
-0
lines changed

‎sagemaker_model_governance/example_metrics/clarify_bias.json

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
{
2+
"post_training_bias_metrics": {
3+
"facets": {
4+
"Gender": [
5+
{
6+
"metrics": [
7+
{
8+
"description": "Accuracy Difference (AD)",
9+
"name": "AD",
10+
"value": -0.11415724421435
11+
}
12+
],
13+
"value_or_threshold": "0"
14+
},
15+
{
16+
"metrics": [
17+
{
18+
"name": "AD",
19+
"description": "Accuracy Difference (AD)",
20+
"value": -0.1141572442143538
21+
}
22+
],
23+
"value_or_threshold": "1"
24+
}
25+
]
26+
},
27+
"label": "Target",
28+
"label_value_or_threshold": "1"
29+
},
30+
"pre_training_bias_metrics": {
31+
"facets": {
32+
"Gender": [
33+
{
34+
"metrics": [
35+
{
36+
"description": "Conditional Demographic Disparity in Labels (CDDL)",
37+
"name": "CDDL",
38+
"value": 0.21491590864936
39+
}
40+
],
41+
"value_or_threshold": "0"
42+
}
43+
]
44+
},
45+
"label": "Target",
46+
"label_value_or_threshold": "1"
47+
},
48+
"version": "1.0"
49+
}

‎sagemaker_model_governance/example_metrics/clarify_explanability.json

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
{
2+
"version": "1.0",
3+
"explanations": {
4+
"kernel_shap": {
5+
"label0": {
6+
"global_shap_values": {
7+
"Marital Status": 0.028630413807559794,
8+
"Occupation": 0.02663187314914846,
9+
"Relationship": 0.036107264188470226,
10+
"Ethnic group": 0.01974161417121363,
11+
"Capital Gain": 0.033577342651290996,
12+
"Capital Loss": 0.019363293576206416,
13+
"Hours per week": 0.021494553921674837,
14+
"Country": 0.04632862084382646
15+
},
16+
"expected_value": 0.0006380207487381995,
17+
"global_top_shap_text": {
18+
"[feature_name_1]": {
19+
"charming": 0.04089221769072609,
20+
"magnificent": 0.03013490694249078
21+
},
22+
"[feature_name_2]": {
23+
"charming": 0.04089221769072609,
24+
"magnificent": 0.03013490694249078
25+
}
26+
}
27+
},
28+
"label1": {
29+
"global_shap_values": {
30+
"Age": 0.036556198315954205,
31+
"Workclass": 0.01819397618827644,
32+
"fnlwgt": 0.02137033320407158,
33+
"Education": 0.018644355523796074,
34+
"Education-Num": 0.037243930827630646
35+
},
36+
"expected_value": 0.0006380207487381995,
37+
"global_top_shap_text": {
38+
"[feature_name_1]": {
39+
"charming": 0.04089221769072609,
40+
"magnificent": 0.03013490694249078
41+
},
42+
"[feature_name_2]": {
43+
"charming": 0.04089221769072609,
44+
"magnificent": 0.03013490694249078
45+
}
46+
}
47+
}
48+
},
49+
"pdp": [
50+
{
51+
"feature_name": "Age",
52+
"data_type": "numerical",
53+
"data_distribution": [
54+
0.05,
55+
0.165,
56+
0.15,
57+
0.095 ],
58+
"feature_values": [
59+
20.866666666666667,
60+
24.6,
61+
28.333366666666667,
62+
32.06666666666666 ],
63+
"model_predictions": [
64+
[
65+
0.7029578701406717,
66+
0.6787781745567918,
67+
0.7008252082392573,
68+
0.733054383918643
69+
]
70+
],
71+
"label_headers": [
72+
"label0"
73+
]
74+
},
75+
{
76+
"feature_name": "Animal",
77+
"data_type": "categorical",
78+
"data_distribution": [
79+
0.1,
80+
0.11,
81+
0.245,
82+
0.06 ],
83+
"feature_values": [
84+
"Cat",
85+
"Dog",
86+
"Horse",
87+
"Bird" ],
88+
"model_predictions": [
89+
[
90+
0.8154899527132511,
91+
0.787026209384203,
92+
0.7409455917775631,
93+
0.7098744075372815
94+
]
95+
],
96+
"label_headers": [
97+
"label0"
98+
]
99+
}
100+
]
101+
}
102+
}

‎sagemaker_model_governance/example_metrics/generic_modelcard_evaluation.json

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
{
2+
"metric_groups": [
3+
{
4+
"metric_data": [
5+
{
6+
"name": "AD",
7+
"type": "number",
8+
"value": -0.11415724421435
9+
}
10+
],
11+
"name": "post_training_bias_metrics - label Target = 1 and facet Gender=0"
12+
},
13+
{
14+
"metric_data": [
15+
{
16+
"name": "AD",
17+
"type": "number",
18+
"value": -0.1141572442143538
19+
}
20+
],
21+
"name": "post_training_bias_metrics - label Target = 1 and facet Gender=1"
22+
},
23+
{
24+
"metric_data": [
25+
{
26+
"name": "CDDL",
27+
"type": "number",
28+
"value": 0.21491590864936
29+
}
30+
],
31+
"name": "pre_training_bias_metrics - label Target = 1 and facet Gender=0"
32+
}
33+
]
34+
}

‎sagemaker_model_governance/example_metrics/model_monitor_model_quality_binary_classification.json

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
{
2+
"binary_classification_metrics": {
3+
"accuracy": {
4+
"standard_deviation": "NaN",
5+
"value": 0.5
6+
},
7+
"accuracy_best_constant_classifier": {
8+
"standard_deviation": 0.2,
9+
"value": 0.25
10+
},
11+
"auc": {
12+
"standard_deviation": 0.1,
13+
"value": 1
14+
},
15+
"confusion_matrix": {
16+
"0": {
17+
"0": 1,
18+
"1": 2
19+
},
20+
"1": {
21+
"0": 0,
22+
"1": 1
23+
}
24+
},
25+
"f0_5": {
26+
"standard_deviation": "NaN",
27+
"value": 0.38461538461538
28+
},
29+
"f0_5_best_constant_classifier": {
30+
"standard_deviation": "NaN",
31+
"value": 0.29411764705882
32+
},
33+
"f1": {
34+
"standard_deviation": "NaN",
35+
"value": 0.5
36+
},
37+
"f1_best_constant_classifier": {
38+
"standard_deviation": "NaN",
39+
"value": 0.4
40+
},
41+
"f2": {
42+
"standard_deviation": "NaN",
43+
"value": 0.71428571428571
44+
},
45+
"f2_best_constant_classifier": {
46+
"standard_deviation": "NaN",
47+
"value": 0.625
48+
},
49+
"false_negative_rate": {
50+
"standard_deviation": "NaN",
51+
"value": 0
52+
},
53+
"false_positive_rate": {
54+
"standard_deviation": "NaN",
55+
"value": 0.66666666666667
56+
},
57+
"precision": {
58+
"standard_deviation": "NaN",
59+
"value": 0.33333333333333
60+
},
61+
"precision_best_constant_classifier": {
62+
"standard_deviation": "NaN",
63+
"value": 0.25
64+
},
65+
"precision_recall_curve": {
66+
"precisions": [
67+
1,
68+
1,
69+
1,
70+
1,
71+
1
72+
],
73+
"recalls": [
74+
0,
75+
0.25,
76+
0.5,
77+
0.75,
78+
1
79+
]
80+
},
81+
"recall": {
82+
"standard_deviation": "NaN",
83+
"value": 1
84+
},
85+
"recall_best_constant_classifier": {
86+
"standard_deviation": "NaN",
87+
"value": 1
88+
},
89+
"receiver_operating_characteristic_curve": {
90+
"false_positive_rates": [
91+
0,
92+
0,
93+
0,
94+
0,
95+
0,
96+
1
97+
],
98+
"true_positive_rates": [
99+
0,
100+
0.25,
101+
0.5,
102+
0.75,
103+
1,
104+
1
105+
]
106+
},
107+
"true_negative_rate": {
108+
"standard_deviation": "NaN",
109+
"value": 0.33333333333333
110+
},
111+
"true_positive_rate": {
112+
"standard_deviation": "NaN",
113+
"value": 1
114+
}
115+
}
116+
}

0 commit comments

Comments
 (0)