random_forest_with_bagging.py

__author__ = 'hanhanw'

import sys
from pyspark import SparkConf, SparkContext
import numpy as np
from sklearn import ensemble
from sklearn import metrics
import matplotlib.pyplot as plt

conf = SparkConf().setAppName("Random Forest using Bagging")
sc = SparkContext(conf=conf)
assert sc.version >= '1.5.1'

inputs = sys.argv[1]


def main():
    indata = np.load(inputs)
    training_data = indata['data_training']
    training_labels = indata['label_training']
    validation_data = indata['data_val']
    validation_labels = indata['label_val']
    ts = range(1,11)
    sampling_rates = [round(0.1*t, 1) for t in ts]
    forest_sizes = [10, 20, 50, 100]


    for sampling_rate in sampling_rates:
        legend_label = 'sampling rate='+str(sampling_rate)
        accuracy_results = []
        for forest_size in forest_sizes:
            rf_clf = ensemble.BaggingClassifier(n_estimators=forest_size, max_samples=sampling_rate)
            rf_clf.fit(training_data, training_labels)
            predictions = rf_clf.predict(validation_data)
            accuracy = metrics.accuracy_score(validation_labels, predictions)
            accuracy_results.append(accuracy)
        plt.plot(range(len(forest_sizes)), accuracy_results, label=legend_label)

    plt.xticks(range(len(forest_sizes)), forest_sizes, size='small')
    plt.legend()
    plt.show()

if __name__ == '__main__':
    main()