2-reddit-chatgpt-consolidate-posts.Rmd

---
title: "Generative AI Generating Buzz: Volume, Engagement, and Content of Initial Reactions to ChatGPT in Discussions Across Education-Related Subreddits"
author: "Bret Staudt Willet"
date: "2023-04-04"
output: html_document
---

## Set Up

```{r setup}
knitr::opts_chunk$set(echo = TRUE)
library(tidyverse)
library(reticulate)
library(RedditExtractoR)
library(anytime)
library(lubridate)
```

```{r condaenv-setup}
use_condaenv("~/r-reticulate")
```

```{python setup}
import pandas as pd
pd.__file__

import datetime as dt
dt.__file__
```

```{r, include=FALSE}
my_client_id <- Sys.getenv('praw_client_id') 
my_client_secret <- Sys.getenv('praw_client_token') 
my_user_agent <- Sys.getenv('praw_user_agent') 
```

```{python, include=FALSE}
import praw

reddit = praw.Reddit(
  client_id = r.my_client_id, 
  client_secret = r.my_client_secret, 
  user_agent = r.my_user_agent
  )
```

```{python initialize_PushShift}
from pmaw import PushshiftAPI
api_praw = PushshiftAPI(praw = reddit)
```



## Consolidate posts from different subreddits

```{r, eval=FALSE, message=FALSE}
posts_raw <-
    list.files(path = "./data-gpt/",
               pattern = "*.csv", 
               full.names = TRUE) %>% 
  map_df(~read_csv(.)) %>%
  mutate(post_date_time = with_tz(post_date_time, tzone = "US/Eastern")) %>%
  distinct(post_id, .keep_all = TRUE) %>%
  filter()
```

```{r, eval=FALSE, message=FALSE}
write_csv(posts_raw, "./data-gpt/!all-posts-raw.csv")
```

```{r, eval=FALSE, message=FALSE}
all_posts <- read_csv("./data-gpt/!all-posts-raw.csv")
```

```{r}
first_post <-
  all_posts %>% 
  group_by(subreddit) %>%
  summarize(earliest_gpt_date = min(date)) %>%
  arrange(earliest_gpt_date)
first_post
```

```{r}
chatgpt_count <-
  all_posts %>% 
  group_by(subreddit) %>%
  count() %>%
  rename(chatgpt_post_n = n) %>%
  arrange(desc(chatgpt_post_n))
chatgpt_count
```

```{r}
ren <- c(
  "r/education", "r/Teachers", "r/teachingresources", "r/edtech",
  "r/AdultEducation", "r/ArtEd", "r/CSEducation", "r/ECEProfessionals",
  "r/ELATeachers", "r/highereducation", "r/historyteachers", "r/itinerantteachers",
  "r/matheducation", "r/MusicEd", "r/ScienceTeachers", "r/slp",
  "r/specialed", "r/TeachersPromote", "r/TeachersInTransition", "r/teaching",
  "r/Professors", "r/academia", "r/instructionaldesign", "r/TeacherTales", 
  "r/OnlineEducation"
)
```

```{python}
feb_start = int(dt.datetime(2023, 2, 1).timestamp())
feb_end = int(dt.datetime(2023, 2, 28).timestamp())
mar_start = int(dt.datetime(2023, 3, 1).timestamp())
mar_end = int(dt.datetime(2023, 3, 31).timestamp())
```

```{python}
posts = api_praw.search_submissions(
  subreddit = "TeacherTales",
  after = feb_start,
  before = mar_end,
  limit = 1000
)
post_list = [post for post in posts]
posts_df = pd.DataFrame(post_list)
posts_df.shape
```

```{r education}
sum(472, 698)
```

```{r slp}
sum(647, 821)
```

```{r teaching}
sum(449, 490)
```

```{r Professors}
sum(381, 409) + sum(420, 515)
```

```{r Teachers}
sum(870, 814, 837, 767) + sum(860, 796, 767, 840, 361)
```

```{r}
ren <- c(
  "r/education", "r/Teachers", "r/teachingresources", "r/edtech",
  "r/AdultEducation", "r/ArtEd", "r/CSEducation", "r/ECEProfessionals",
  "r/ELATeachers", "r/highereducation", "r/historyteachers", "r/itinerantteachers",
  "r/matheducation", "r/MusicEd", "r/ScienceTeachers", "r/slp",
  "r/specialed", "r/TeachersPromote", "r/TeachersInTransition", "r/teaching",
  "r/Professors", "r/academia", "r/instructionaldesign", "r/TeacherTales", 
  "r/OnlineEducation"
)
total_post_n <-
  c(1170, 6912, 322, 142,
    13, 103, 40, 724,
    156, 394, 146, 0,
    142, 149, 279, 1468,
    330, 27, 648, 939,
    1725, 747, 356, 71,
    251
  )

total_posts <- 
  tibble(ren, total_post_n) %>%
  rename(subreddit = ren)

total_posts
```

```{r}
chatgpt_table <-
  tibble(first_post) %>%
  left_join(chatgpt_count, by="subreddit") %>%
  left_join(total_posts, by="subreddit") %>%
  replace(is.na(.), 0) %>%
  mutate(p_posts = 100*chatgpt_post_n/total_post_n) %>%
  arrange(subreddit)
chatgpt_table
```

```{r}
posts <- 
  all_posts %>%
  left_join(chatgpt_table, by="subreddit") %>%
  mutate(status = ifelse(post_text == "[deleted]",
                         "deleted",
                         ifelse(post_text == "[removed]",
                                "removed",
                                "remaining")),
         status = ifelse(is.na(status),
                         "remaining",
                         status),
         post_url = paste0("https://www.reddit.com", permalink)
  )
posts %>% count(status)
```

```{r}
filtered_posts <-
  posts %>%
  filter(status == "remaining")
nrow(filtered_posts); paste("Expected number of comments:", sum(filtered_posts$num_comments))
```

```{r, eval=FALSE}
write_csv(posts, "./data-gpt/!all-posts.csv")
write_csv(filtered_posts, "./data-gpt/!filtered-posts.csv")
```