res = requests.get(URL)

res = requests.get(URL, headers={'User-agent': 'YOUR NAME Bot 0.1'})

# Imports - used or otherwise.
import pandas as pd
import requests
import json
import time
import regex as re
import praw
import matplotlib.pyplot as plt
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer
from bs4 import BeautifulSoup
from sklearn.linear_model import LogisticRegression
from sklearn.neighbors import KNeighborsClassifier
from sklearn.pipeline import Pipeline
from sklearn.model_selection import train_test_split, cross_val_score, cross_val_predict, GridSearchCV, StratifiedKFold
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer, ENGLISH_STOP_WORDS
from sklearn.ensemble import RandomForestClassifier, VotingClassifier
from sklearn.naive_bayes import MultinomialNB, GaussianNB
from sklearn import metrics
from sklearn.metrics import roc_auc_score, roc_curve

# Create the URL variables
URL_ds = "http://www.reddit.com/r/datascience.json"
URL_stats = "https://www.reddit.com/r/statistics.json"

# Authenticating via OAuth for praw
reddit = praw.Reddit(client_id='AOJTLQLavhOXPg',
                     client_secret='eS08QOpy2lWh37qkVBGlN7yMjRI',
                     username='TCRAY_DSI',
                     password='dsi123',
                     user_agent='TK Bot 0.1')

# Check
print(reddit.user.me())

# Create subs for praw:
sub_ds = reddit.subreddit('datascience')
sub_stats = reddit.subreddit('statistics')

# Create top pulls
top_ds = sub_ds.top(time_filter='year')
top_stats = sub_stats.top(time_filter='year')

# These were used and attempted (to success) before creating the loop
# Request the JSON files
# I did them in seperate cells to space out the scrapping, so reddit wouldn't throw a 429 error
# res_ds = res.get(URL_ds, headers={'User-agent': 'TK Bot 0.1'})

# res_stats = requests.get(URL_stats, headers={'User-agent': 'TK Bot 0.1'})

# res_stats.status_code

data = res.json()

# These were used and attempted (to success) before creating the loop
# Convert the JSON responses
# data_ds = res_ds.json()
# data_stats = res_stats.json()

# Check out data
# data_ds
# data_stats

# Testing adding nested dictionaries to each other
# doubling_up = data_ds['data']['children'] + data_ds['data']['children']
# doubling_up

print(len(data['data']['children']))

time.sleep(3) # sleeps 3 seconds before continuing```

This will throttle your loop and keep you within Reddit's guidelines. You'll need to import the `time` library for this to work!


```python
# Check out length
# print(len(data_ds['data']['children']))
# print(len(data_stats['data']['children']))

# Test the last post pull
# data_ds['data']['after']

# For DS set - previously ran to generate CSV
url_ds = "https://www.reddit.com/r/datascience.json"
data_ds = []
total = []
next_get = ''

# I went with 40 b/c 40 * 25 = 1000 posts total
for i in range(40):

    # Request get
    res = requests.get(url_ds+next_get, headers={'User-agent': 'TK Bot 0.1'})

    # Convert the JSON
    new_dict = res.json()

    # Add to already collected data set
    data_ds.extend(new_dict['data']['children'])

    # Collect 'after' from new dict to generate next URL
    new_url_end = str(new_dict['data']['after'])

    # Generate the next URL
    next_get = '?after='+new_url_end

    # CSV add/update along with DF creation
    # Chose greater than 0 so the else executes on the first iteration
    if i > 0:
        # Read in previous csv for comparision/add
        # Establish current DF - left over from previous way of running
        # past_posts = pd.read_csv('data_ds.csv')
        # current_df = pd.DataFrame(data_ds)

        # Append new and old
        total = pd.DataFrame(data_ds)

        # Convert to DF and save to new csv file
        pd.DataFrame(total).to_csv('data_ds.csv', index = False)

    else:
        pd.DataFrame(data_ds).to_csv('data_ds.csv', index = False)

    # Sleep to fit within Reddit's pull limit
    time.sleep(3)

# For stats set - previously ran to generate CSV
url_stats = "https://www.reddit.com/r/statistics.json"
data_stats = []
total = []
next_get = ''

# I went with 40 b/c 40 * 25 = 1000 posts total
for i in range(40):

    # Request get
    res = requests.get(url_stats+next_get, headers={'User-agent': 'TK Bot 0.1'})

    # Convert the JSON
    new_dict = res.json()

    # Add to already collected data set
    data_stats.extend(new_dict['data']['children'])

    # Collect 'after' from new dict to generate next URL
    new_url_end = str(new_dict['data']['after'])

    # Generate the next URL
    next_get = '?after='+new_url_end

    # CSV add/update along with DF creation
    # Chose greater than 0 so the else executes on the first iteration
    if i > 0:
        # Read in previous csv for comparision/add
        # Establish current DF - left over from previous way of running
        # past_posts = pd.read_csv('data_stats.csv')
        # current_df = pd.DataFrame(data_stats)

        # Append new and old
        total = pd.DataFrame(data_stats)

        # Convert to DF and save to new csv file
        pd.DataFrame(total).to_csv('data_stats.csv', index = False)

    else:
        pd.DataFrame(data_stats).to_csv('data_stats.csv', index = False)

    # Sleep to fit within Reddit's pull limit
    time.sleep(3)

# This was an older attempt at writing the function, that I scrapped and decided to start fresh on:
# url_ds = "https://www.reddit.com/r/datascience.json?after=" + last_post_ds

# for i in range(25):
#     # Get the name of the last post
#     last_post_ds = data_ds['data']['after']

#     # Set the url from the last post
#     new_url_ds = "https://www.reddit.com/r/datascience.json?after=" + last_post_ds

#     # Perform request get
#     new_res_ds = res.get(new_url_ds, headers={'User-agent': 'TK Bot 0.1'})

#     # Convert the JSON to a dict
#     new_data_ds = new_res_ds.json()

#     # Add the new dict to the already existing one
#     data_ds.update(new_data_ds)
#     data_ds['data']['children'] = data_ds['data']['children'] + new_data_ds['data']['children']
#     data_ds['data']['after'] = new_data_ds['data']['after']

#     # Sleep
#     # time.sleep(3)

# Next few cells devoted to understanding how to generate a combined dict
# new_data_ds.items()

# OG_ds_data = data_ds.copy()

# new_data_ds = new_res_ds.json()
# data_ds.update(new_data_ds)

# Testing adding nested dictionaries to each other
# doubling_up = data_ds['data']['children'] + data_ds['data']['children']
# doubling_up

# My loop in the previous cell completes this step.

%pwd

df_ds = pd.read_csv('./data_ds.csv')
df_stats = pd.read_csv('./data_stats.csv')

# 983 DS posts vs 978 stats posts
# df_ds.shape[0]
df_stats.shape[0]

# for i in df_ds.shape[0]
# Testing what I want to loop
df_ds['body'] = pd.Series(re.findall('(?<=selftext).{4}(.*).{4}(?=author_fullname)', df_ds['data'][0]))

df_ds.head()

type(df_ds['body'].iloc[0,])

# To pull out the body of the post and make it a new column
for i in range(0, df_ds.shape[0]):
    try: #Since regex makes it a list, this helps deal with nulls
        df_ds['body'][i] = re.findall('(?<=selftext).{4}(.*).{4}(?=author_fullname)', df_ds['data'][i])[0]
    except:
        df_ds['body'][i] = ''

df_ds.head()

# For some reason, wrapping it in pd.Series makes this work before I loop it
try:
    df_ds['title'] = pd.Series(re.findall('(?<= .title).{4}(.*).{4}(?=link_flair_richtext)', df_ds['data'][0]))[0]
except:
    df_ds['title'] = ''

# To pull out the title of the post and make it a new column
for i in range(0, df_ds.shape[0]):
    try:
        df_ds['title'][i] = re.findall('(?<= .title).{4}(.*).{4}(?=link_flair_richtext)', df_ds['data'][i])[0]
    except:
        df_ds['title'][i] = ''

df_stats.shape[0]

df_ds.head()

# Looks like the body/title got pulled in as a list, turning it into a str
# This is leftover from an older method
# for w in range(0,df_ds['body'].shape[0]):
#     df_ds['body'][w] = str(df_ds['body'][w])

# Additional Clean-up - DS
df_ds['body'] = df_ds['body'].map(lambda x: x.replace('\\n',''))
df_ds['body'] = df_ds['body'].map(lambda x: x.replace('\n',''))
df_ds['body'] = df_ds['body'].map(lambda x: x.replace('\\',''))
df_ds['body'] = df_ds['body'].map(lambda x: x.replace("\\'","'"))
# df_ds['body'] = [w.replace('/n', '') for w in df_ds['body']]

df_ds['body']

# Add target column for later combination
df_ds['subreddit_target'] = 1

# Check out the nulls
df_ds.isnull().sum().sort_values()

# Same process of pulling out body/post for df_stats
try:
    df_stats['body'] = pd.Series(re.findall('(?<=selftext).{4}(.*).{4}(?=author_fullname)', df_stats['data'][0]))[0]
except:
    df_stats['body'] = ''

# To pull out the body of the post and make it a new column
for i in range(0, df_stats.shape[0]):
    try:
        df_stats['body'][i] = re.findall('(?<=selftext).{4}(.*).{4}(?=author_fullname)', df_stats['data'][i])[0]
    except:
        df_stats['body'] = ''

# For some reason, wrapping it in pd.Series makes this work before I loop it
try:
    df_stats['title'] = pd.Series(re.findall('(?<= .title).{4}(.*).{4}(?=link_flair_richtext)', df_stats['data'][0]))[0]
except:
    df_stats['title'] = ''

# To pull out the title of the post and make it a new column
for i in range(0, df_stats.shape[0]):
    try:
        df_stats['title'][i] = re.findall('(?<= .title).{4}(.*).{4}(?=link_flair_richtext)', df_stats['data'][i])[0]
    except:
        df_stats['title'] = ''

# Looks like the body got pulled in as a list
# restricting how I clean it up, turning it into a str
# Leftover
# for w in range(0,df_stats['body'].shape[0]):
#     df_stats['body'][w] = str(df_stats['body'][w])

# Additional Clean-up - DS
df_stats['body'] = df_stats['body'].map(lambda x: x.replace('\\n',''))
df_stats['body'] = df_stats['body'].map(lambda x: x.replace('\n',''))
df_stats['body'] = df_stats['body'].map(lambda x: x.replace('\\',''))
df_stats['body'] = df_stats['body'].map(lambda x: x.replace("\\'","'"))
# df_stats['body'] = [w.replace('/n', '') for w in df_stats['body']]

df_stats['subreddit_target'] = 0

# Check out the nulls
df_stats.isnull().sum().sort_values()

# Renaming the columns so they're easier to discern
# Left over from previous way of solving
# df_ds.columns = ['data','kind','body_ds','title_ds']
# df_stats.columns = ['data','kind','body_stats','title_stats']

df_ds.head(1)

# Create combined list for later usage
dflist = [df_ds, df_stats]
dfCombined = pd.concat(dflist, axis=0, sort=True)

dfCombined.head()
# .fillna(value=" ")

# Check length is what I expected
dfCombined['body'].shape[0]

dfCombined['title_body'] = dfCombined['body'] + dfCombined['title']

dfCombined

# Save the cleaned-up product on the side
dfCombined.to_csv('Combined.csv', index = False)

# Going back after the fact to add some obvious stop words
# This was form a 'normal' run of CountVectorizer, e.g. n-grams = 1
# amp seems to be some bad html code that got pulled in mistakenly
new_stop_words = {'science', 'like', 'https', 'com', 've', '10', '12', 'amp'}
stop_words = ENGLISH_STOP_WORDS.union(new_stop_words)

# Instantiate
cvec = CountVectorizer(stop_words=stop_words) # First run through of n-grams = 1

# Set variables and train_test_split
# Sticking with the normal 75/25 split
X = dfCombined['title_body'].values
y = dfCombined['subreddit_target']
# .map({'statistics':0, 'datascience':1})

X_train, X_test, y_train, y_test = train_test_split(X,
                                                    y,
                                                    random_state=42)

# Fit and transform
cvec.fit(X_train)
X_train_transform = cvec.transform(X_train)
X_test_transform = cvec.transform(X_test)

df_view_stats = pd.DataFrame(X_test_transform.todense(),
                             columns=cvec.get_feature_names(),
                             index=y_test.index)
df_view_stats.head()
# .T.sort_values('statistics', ascending=False).head(10).T

# Most commonly used words on data science
# This was run multiple times for different sets of n-grams
word_count_test = pd.concat([df_view_stats, y_test], axis=1)
word_count_test['subreddit_target'] = word_count_test['subreddit_target'].map({0:'statistics', 1:'datascience'})
word_count_test.groupby(by='subreddit_target').sum().sort_values(by='datascience', axis=1, ascending=False).T

# Most commonly used words on statistics
# This was run multiple times for different sets of n-grams
word_count_test = pd.concat([df_view_stats, y_test], axis=1)
word_count_test['subreddit_target'] = word_count_test['subreddit_target'].map({0:'statistics', 1:'datascience'})
word_count_test.groupby(by='subreddit_target').sum().sort_values(by='statistics', axis=1, ascending=False).T

# This was the second run of CountVectorizer, e.g. n-grams = 2
# I removed science, because I wanted to make a differentation b/w 'science' and 'data science', and also 've', b/c it was only getting picked up b/c 'I've'
# Going to leave stop words as is for n-grams = 2, aside from html crap that got pulled in
new_stop_words = {'amp', 'x200b', 'amp x200b'}
stop_words = ENGLISH_STOP_WORDS.union(new_stop_words)

# Instantiate
cvec2 = CountVectorizer(stop_words=stop_words, ngram_range=(2,2)) #Second run through of n-grams = 2

# Set variables and train_test_split
# Sticking with the normal 75/25 split
X = dfCombined['title_body'].values
y = dfCombined['subreddit_target']
# .map({'statistics':0, 'datascience':1})

X_train2, X_test2, y_train2, y_test2 = train_test_split(X,
                                                    y,
                                                    random_state=42)

# Fit and transform
cvec2.fit(X_train2)
X_train_transform2 = cvec2.transform(X_train2)
X_test_transform2 = cvec2.transform(X_test2)

df_view_stats2 = pd.DataFrame(X_test_transform2.todense(),
                             columns=cvec2.get_feature_names(),
                             index=y_test2.index)
df_view_stats2.head()
# .T.sort_values('statistics', ascending=False).head(10).T

# Most commonly used words on data science
word_count_test = pd.concat([df_view_stats2, y_test2], axis=1)
word_count_test['subreddit_target'] = word_count_test['subreddit_target'].map({0:'statistics', 1:'datascience'})
word_count_test.groupby(by='subreddit_target').sum().sort_values(by='datascience', axis=1, ascending=False).T

# Most commonly used words on statistics
word_count_test = pd.concat([df_view_stats2, y_test2], axis=1)
word_count_test['subreddit_target'] = word_count_test['subreddit_target'].map({0:'statistics', 1:'datascience'})
word_count_test.groupby(by='subreddit_target').sum().sort_values(by='statistics', axis=1, ascending=False).T

# Instantiate and fit
lr2 = LogisticRegression()
lr2.fit(X_train_transform2, y_train2)
lr2.score(X_train_transform2, y_train2)

lr2.score(X_test_transform2, y_test2)
# Looks like a pretty decent overfit

# Instantiate and fit
# From here on out, it's n-grams = 2
lr = LogisticRegression()
lr.fit(X_train_transform, y_train)
lr.score(X_train_transform, y_train)

lr.score(X_test_transform, y_test)
# Looks like a pretty decent overfit

preds = lr.predict(X_test_transform)
pred_proba = lr.predict_proba(X_test_transform)[:,1]

roc_auc = roc_auc_score(y_test, preds)
roc_auc

roc_auc = roc_auc_score(y_test, preds)
FPR, TPR, thresholds = roc_curve(y_test, pred_proba)
plt.figure(figsize=(8, 6))
plt.plot(FPR, TPR, label='Logistic Regression (area = %0.2f)' % roc_auc)
plt.title('ROC-AUC (n-grams=1)')
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.05])
plt.plot([0, 1], [0, 1],'r--')
plt.legend(loc="lower right")
plt.show()

## I'm going to take an educated guess that the baseline accuracy is 50%, as in, randomly guessing

# Instantiate
rf = RandomForestClassifier()
cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)

cvs_train = cross_val_score(rf, X_train_transform, y_train, cv=cv, n_jobs=-1)

print(cvs_train)
print(cvs_train.mean())

cvs_test = cross_val_score(rf, X_test_transform, y_test, cv=cv, n_jobs=-1)

print(cvs_test)
print(cvs_test.mean())
# Still slight overfit

mnb = MultinomialNB()

cvs_train = cross_val_score(mnb, X_train_transform, y_train, cv=cv, n_jobs=-1)

print(cvs_train)
print(cvs_train.mean())

cvs_test = cross_val_score(mnb, X_test_transform, y_test, cv=cv, n_jobs=-1)

print(cvs_test)
print(cvs_test.mean())
# Not as bad of an overfit

gnb = GaussianNB()

cvs_train = cross_val_score(gnb, X_train_transform.toarray(), y_train, cv=cv, n_jobs=-1)

print(cvs_train)
print(cvs_train.mean())

cvs_test = cross_val_score(gnb, X_test_transform.toarray(), y_test, cv=cv, n_jobs=-1)

print(cvs_test)
print(cvs_test.mean())
# Overfit isn't as much of a problem on this model
# However, the overall score isn't as strong as the other models