Cloud Service Integration
Machine Learning
Streaming Data
Operationalization
Natural Language Processing
Bioinformatics and Genomics
Model Evaluation
Evaluate model performance by probability cutoff
Note: Extract probability values using method found here - https://www.sparkitecture.io/machine-learning/model-saving-and-loading#remove-unnecessary-columns-from-the-scored-data.
1
performance_df = spark.createDataFrame([(0,0,0)], ['cutoff', 'AUPR', 'AUC'])
2
3
for cutoff in range(5, 95, 5):
4
cutoff = (cutoff * 0.01)
5
6
print('Testing cutoff = ', str(format(cutoff, '.2f')))
7
8
lrpredictions_prob_temp = lrpredictions.withColumn('prediction_test', when(col('probability') >= cutoff, 1).otherwise(0).cast(DoubleType()))
9
aupr_temp = BinaryClassificationMetrics(lrpredictions_prob_temp['label', 'prediction_test'].rdd).areaUnderPR
10
auc_temp = BinaryClassificationMetrics(lrpredictions_prob_temp['label', 'prediction_test'].rdd).areaUnderROC
11
print('\tAUPR:', aupr_temp,'\tAUC:', auc_temp)
12
performance_df_row = spark.createDataFrame([(cutoff,aupr_temp,auc_temp)], ['cutoff', 'AUPR', 'AUC'])
13
performance_df = performance_df.union(performance_df_row)
14
15
display(performance_df)
Copied!
Evaluate multiclass classification models
1
from pyspark.ml.evaluation import MulticlassClassificationEvaluator
2
from pyspark.mllib.evaluation import MulticlassMetrics
3
4
# Evaluate best model
5
print('Accuracy:', lrevaluator.evaluate(lrpredictions))
6
lrmetrics = MulticlassMetrics(lrpredictions['label','prediction'].rdd)
7
print('Confusion Matrix:\n', lrmetrics.confusionMatrix())
8
print('F1 Score:', lrmetrics.fMeasure(1.0,1.0))
Copied!
Evaluate binary classification models
1
for model in ["lrpredictions", "dtpredictions", "rfpredictions", "nbpredictions", "gbpredictions"]:
2
df = globals()[model]
3
4
tp = df[(df.label == 1) & (df.prediction == 1)].count()
5
tn = df[(df.label == 0) & (df.prediction == 0)].count()
6
fp = df[(df.label == 0) & (df.prediction == 1)].count()
7
fn = df[(df.label == 1) & (df.prediction == 0)].count()
8
a = ((tp + tn)/df.count())
9
10
if(tp + fn == 0.0):
11
r = 0.0
12
p = float(tp) / (tp + fp)
13
elif(tp + fp == 0.0):
14
r = float(tp) / (tp + fn)
15
p = 0.0
16
else:
17
r = float(tp) / (tp + fn)
18
p = float(tp) / (tp + fp)
19
20
if(p + r == 0):
21
f1 = 0
22
else:
23
f1 = 2 * ((p * r)/(p + r))
24
25
print("Model:", model)
26
print("True Positives:", tp)
27
print("True Negatives:", tn)
28
print("False Positives:", fp)
29
print("False Negatives:", fn)
30
print("Total:", df.count())
31
print("Accuracy:", a)
32
print("Recall:", r)
33
print("Precision: ", p)
34
print("F1 score:", f1)
35
print('AUC:', BinaryClassificationMetrics(df['label','prediction'].rdd).areaUnderROC)
36
print("\n")
Copied!