#!/usr/bin/pythonimport sysfrom time import timeimport logging# Display progress logs on stdoutlogging.basicConfig(level = logging.DEBUG, format = '%(asctime)s %(message)s')sys.path.append("../DatasetProcessing/")from vectorize_split_dataset import preprocess### features_train and features_test are the featuresfor the training and testing datasets, respectively### labels_train and labels_test are the corresponding item labelsfeatures_train, features_test, labels_train, labels_test = preprocess()#########################################################from sklearn.naive_bayes import GaussianNBclf = GaussianNB()t0 = time()clf.fit(features_train, labels_train)pred = clf.predict(features_test)print("training time:", round(time() - t0, 3), "s")print(clf.score(features_test, labels_test))## Printing Metricsfor Training and Testingprint("No. of Testing Features:" + str(len(features_test)))print("No. of Testing Features Label:" + str(len(labels_test)))print("No. of Training Features:" + str(len(features_train)))print("No. of Training Features Label:" + str(len(labels_train)))print("No. of Predicted Features:" + str(len(pred)))## Calculating Classifier Performancefrom sklearn.metrics import classification_reporty_true = labels_testy_pred = predlabels = ['0', '1']target_names = ['class 0', 'class 1']print(classification_report(y_true, y_pred, target_names = target_names, labels = labels))# How to predict label of a new textnew_text = "You won a lottery at UK lottery commission. Reply to claim it"

#!/usr/bin/pythonimport osimport pickleimport numpynumpy.random.seed(42)path = os.path.dirname(os.path.abspath(__file__))### The words(features) and label_data(labels), already largely processed.###These files should have been created beforehandfeature_data_file = path + "./createdDataset/dataSet.pkl"label_data_file = path + "./createdDataset/dataLabel.pkl"feature_data = pickle.load(open(feature_data_file, "rb"))label_data = pickle.load(open(label_data_file, "rb"))### test_size is the percentage of events assigned to the test set(the### remainder go into training)### feature matrices changed to dense representationsfor compatibility with### classifier functions in versions 0.15.2 and earlierfrom sklearn import cross_validationfeatures_train, features_test, labels_train, labels_test = cross_validation.train_test_split(feature_data, label_data, test_size = 0.1, random_state = 42)from sklearn.feature_extraction.text import TfidfVectorizervectorizer = TfidfVectorizer(sublinear_tf = True, max_df = 0.5, stop_words = 'english')features_train = vectorizer.fit_transform(features_train)features_test = vectorizer.transform(features_test)#.toarray()## feature selection to reduce dimensionalityfrom sklearn.feature_selection import SelectPercentile, f_classifselector = SelectPercentile(f_classif, percentile = 5)selector.fit(features_train, labels_train)features_train_transformed_reduced = selector.transform(features_train).toarray()features_test_transformed_reduced = selector.transform(features_test).toarray()features_train = features_train_transformed_reducedfeatures_test = features_test_transformed_reduceddef preprocess():  return features_train, features_test, labels_train, labels_test

#!/usr/bin/pythonimport osimport pickleimport reimport sys# sys.path.append("../tools/")"""    Starter code to process the texts of accuate and inaccurate category to extract    the features and get the documents ready for classification.    The list of all the texts from accurate category are in the accurate_files list    likewise for texts of inaccurate category are in (inaccurate_files)    The data is stored in lists and packed away in pickle files at the end."""accurate_files = open("./rawDatasetLocation/accurateFiles.txt", "r")inaccurate_files = open("./rawDatasetLocation/inaccurateFiles.txt", "r")label_data = []feature_data = []### temp_counter is a way to speed up the development--there are### thousands of lines of accurate and inaccurate text, so running over all of them### can take a long time### temp_counter helps you only look at the first 200 lines in the list so you### can iterate your modifications quickertemp_counter = 0for name, from_text in [("accurate", accurate_files), ("inaccurate", inaccurate_files)]:  for path in from_text: ###only look at first 200 texts when developing### once everything is working, remove this line to run over full datasettemp_counter = 1if temp_counter < 200:  path = os.path.join('..', path[: -1])print(path)text = open(path, "r")line = text.readline()while line: ###use afunction parseOutText to extract the text from the opened text# stem_text = parseOutText(text)stem_text = text.readline().strip()print(stem_text)### use str.replace() to remove any instances of the words# stem_text = stem_text.replace("germani", "")### append the text to feature_datafeature_data.append(stem_text)### append a 0 to label_dataif text is from Sara, and 1if text is from Chrisif (name == "accurate"):  label_data.append("0")elif(name == "inaccurate"):  label_data.append("1")line = text.readline()text.close()print("texts processed")accurate_files.close()inaccurate_files.close()pickle.dump(feature_data, open("./createdDataset/dataSet.pkl", "wb"))pickle.dump(label_data, open("./createdDataset/dataLabel.pkl", "wb"))