library <- c("tidyverse", "lubridate", "tidytext", "stringi", "ggraph", "igraph",
             "textclean", "widyr", "scales", "mgsub", "wordcloud", "RColorBrewer", "knitr")
sapply(library, require, character.only= TRUE)

# it is from kaggle web page 
amazon <- read.csv("data/1429_1.csv")

table(amazon$id)

# AVphgVaX1cnluZ0-DR74 one seems to be the most reviewed product. Let's pick this one

product <- filter(amazon, id =="AVphgVaX1cnluZ0-DR74")

# what we have
glimpse(product)

product %>%
        mutate(Date= date(ymd_hms(reviews.date))) %>%
        ggplot(aes(x=Date)) + geom_freqpoly(bins = 100)+
        theme_bw()+
        labs(y= "Number of the reviews",
             x= "",
             title = " Timeline of the reviews")

reviews.text <- product %>%
  select(reviews.text)

# Encoding
# Check Encoding and Make it consistent
stri_enc_mark(reviews.text$reviews.text)
reviews.text$reviews.text <- sapply(reviews.text$reviews.text,
                                                  function(row) iconv(row,
                                                                      "latin1",
                                                                      "ASCII",
                                                                      sub = " "))

# Lowecase all text
reviews.text$reviewText <- tolower(reviews.text$reviews.text)

# make wasn't=was not, can't=can not, etc..
reviews.text$reviews.text <- gsub("wasn[\u2019']t", "was not", reviews.text$reviews.text)
reviews.text$reviews.text <- gsub("won[\u2019']t", "will not", reviews.text$reviews.text)
reviews.text$reviews.text <- gsub("can[\u2019']t", "can not", reviews.text$reviews.text)
reviews.text$reviews.text <- gsub("didn[\u2019']t", "did not", reviews.text$reviews.text)
reviews.text$reviews.text <- gsub("don[\u2019']t", "do not", reviews.text$reviews.text)
reviews.text$reviews.text <- gsub("I[\u2019']m", "I am", reviews.text$reviews.text)
reviews.text$reviews.text <- gsub("[\u2019']ve", " have", reviews.text$reviews.text) 
reviews.text$reviews.text <- gsub("[\u2019|']s", "", reviews.text$reviews.text)
reviews.text$reviews.text <- gsub("[\u2019']re", " are", reviews.text$reviews.text)
reviews.text$reviews.text <- gsub("[\u2019']ll", " will", reviews.text$reviews.text)

# If you view common typos during your analysis, fix them here.
reviews.text$reviews.text<- gsub("canceling", "cancelling", reviews.text$reviews.text)
reviews.text$reviews.text <- gsub("cancellation", "cancelling", reviews.text$reviews.text)

# omit the following two lines if you have not loaded the tm package
# Remove numbers in the text
reviews.text$reviews.text <- tm::removeNumbers(reviews.text$reviews.text)
# Remove punctuations in the text
reviews.text$reviews.textt <- tm::removePunctuation(reviews.text$reviews.text)


# Fix Negations
# Create a list to identify the sentiment shifters in the text
negation.words <- c("not",
                    "no",
                    "without",
                    "never",
                    "bad",
                    "none",
                    "never",
                    "nobody",
                    "nowhere",
                    "neither",
                    "nothing"
)

# Run the following to view Shifted sentiments sorted by polarity point
shifted.words <- reviews.text %>%
  unnest_tokens(bigram, reviews.text, token = "ngrams", n = 2)%>%
  count(bigram, sort = TRUE) %>%
  separate(bigram, c("word1", "word2"), sep = " ")%>%
  filter(word1 %in% negation.words & !word2 %in% stop_words$word)%>%
  inner_join(get_sentiments("bing"), by = c(word2 = "word"))%>%
  mutate(sentiment = ifelse(sentiment == "positive", 1, -1)) %>%
  mutate(score = sentiment * n) %>%
  mutate(word2 = reorder(word2, score))

head(shifted.words)

# Pick the most effective sentiment shifters
negated.phrases <- c("not worth", 
                     "not noise",
                     "no issues",
                     "no complaints",
                     "not disappoint",
                     "not disappointed",
                     "not cheap",
                     "no regrets"
                     
)
# Find synonyms for the phrases above to replace
synonyms <- c("expensive",
              "functional",
              "cool",
              "satisfied",
              "satisfied",
              "satisfied",
              "expensive",
              "satisfied"
)



# Replace the negations with their synonyms.
reviews.text <- mgsub(reviews.text$reviews.text, negated.phrases, synonyms) %>%
  dplyr::as_tibble() %>%
  rename(reviews.text = value)

# if you want to ignore words that are frequent but doesn't help, add them to this list.
ignore.words <- tibble(word = c("amazon"))

# create the words freq table
word.freq.table<- reviews.text %>% 
  unnest_tokens(word, reviews.text) %>%
  anti_join(stop_words) %>%
  anti_join(ignore.words) %>%
  count(word, sort = TRUE)

word.freq.table

# Plotting a Wordcloud
word.freq.table %>% 
  filter(n>100) %>%
  with(wordcloud(word, n,
                 scale = c(5,0.3),
                 colors = brewer.pal(8, "Dark2")))

# Most Common Bigrams
reviews.text %>%
  unnest_tokens(bigram, reviews.text, token = "ngrams", n = 2) %>%
  count(bigram, sort = TRUE) %>%
  separate(bigram, c("word1", "word2"), sep = " ") %>%
  filter(!word1 %in% stop_words$word & !word2 %in% stop_words$word) %>%
  filter(n>7) %>%
  unite(word, word1:word2, sep = " ") %>%
  with(wordcloud(word, n,
                 scale = c(3,0.5),
                 colors = brewer.pal(8, "Dark2")))

# Most common Positive and Negative words using Bing
reviews.text %>% 
  unnest_tokens(word, reviews.text) %>%
  anti_join(stop_words) %>%
  anti_join(ignore.words) %>%
  inner_join(get_sentiments("bing")) %>%
  count(word, sentiment, sort = TRUE) %>%
  filter(n > 70) %>%
  mutate(word = reorder(word, n)) %>%
  mutate(percent = round(n/sum(n), 3)) %>%
  ggplot(aes(x = word, y = percent, fill = sentiment, label = percent)) +
  geom_col(show.legend = FALSE) +
  facet_wrap(~sentiment, scales = "free_y") +
  geom_text(aes(y = 0.7*percent)) +
  labs(title = "Amazon Kindle (bing)") +
  coord_flip() + 
  theme_bw() +
  theme(plot.title = element_text(hjust = 0.5))

product %>% 
        unnest_tokens(word, reviews.text) %>%
        filter(!word %in% stop_words$word) %>%
        select(id, reviews.date, word) %>%
        inner_join(get_sentiments("bing"), by = c("word" = "word")) %>%
        select(id, reviews.date, sentiment ) %>%
        count(id,reviews.date, sentiment) %>%
        mutate(n = ifelse(sentiment == 'negative', -n, n)) %>%
        group_by(Date = as.Date(ymd_hms(reviews.date)), sentiment) %>%
        summarize(total = sum(n)) %>%
        ggplot(aes(x=Date, y=total, fill = sentiment)) + geom_bar(stat='identity') +
        theme_light()+
        labs(title = " Sentiment scores since 2015",
             y= "Total ssentiment scores",
             x= NULL)

# Bing
bing.mean.score <- word.freq.table %>% 
  inner_join(get_sentiments("bing")) %>%
  mutate(sentiment = ifelse(sentiment == "positive", 1, -1)) %>%
  summarise(mean = mean(sentiment))

# rescale the range to 5 star range.
bing.mean.score<-rescale(bing.mean.score$mean, to = c(1,5), from = c(-1,1))
  

# Afinn scores are from -5 to 5.
afinn.mean.score <- word.freq.table %>% 
  inner_join(get_sentiments("afinn"))%>%
  summarise(mean = mean(value))

# rescale the range to 5 star range.
afinn.mean.score<-rescale(afinn.mean.score$mean, to = c(1,5), from = c(-5,5))

# Correlation Terms
# The correlation of appearing together in a review
correlation.terms <-reviews.text %>%
  mutate(review = row_number()) %>%
  unnest_tokens(word, reviews.text) %>%
  filter(!word %in% stop_words$word) %>%
  group_by(word) %>%
  filter(n() >= 5)%>%
  pairwise_cor(word, review, sort = TRUE)

correlation.terms

correlation.terms %>%
  filter(correlation >= 0.50) %>%
  graph_from_data_frame() %>%
  ggraph(layout = "igraph", algorithm = "kk") +
  geom_edge_link(aes(alpha = correlation), 
                 show.legend = FALSE)+
  geom_node_point(color = "lightblue", size = 2) +
  geom_node_text(aes(label = name), repel = TRUE) +
  theme_void()

bigrams.network.df<-reviews.text %>%
  unnest_tokens(bigram, reviews.text, token = "ngrams", n = 2) %>%
  separate(bigram, c("word1", "word2"), sep = " ") %>%
  filter(!word1 %in% stop_words$word & !word2 %in% stop_words$word) %>%
  count(word1, word2, sort = TRUE) %>%
  filter(n > 30)

bigrams.network <- graph_from_data_frame(bigrams.network.df)

# now we find the centrality measures of the network
# degree:the number of its adjacent edges (measure of direct influence)
deg <- degree(bigrams.network, mode = "all")

#K-core decomposition allows us to identify the core and the periphery of the network. A k-core is a maximal subnet of a network such that all nodes have at least degree K.
core <- coreness(bigrams.network, mode = "all")

# betweenness measures brokerage or gatekeeping potential. It is (approximately) the number of shortest paths between nodes that pass through a particular node.
betw <- betweenness(bigrams.network)

#Eigenvector centrality is a measure of being well-connected connected to the well-connected. First eigenvector of the graph adjacency matrix. Only works with undirected networks.
eigen <- eigen_centrality(bigrams.network, directed = TRUE)

members <- cluster_walktrap(bigrams.network)

bigrams.network <- simplify(bigrams.network, 
                            remove.multiple = FALSE,
                            remove.loops = TRUE)
V(bigrams.network)$color <- members$membership+1

# Using "Degree" as size
# degree=mode (number of edges of the node, in-degree:prestige

plot(bigrams.network,
     layout = layout_with_fr,
     vertex.label.color = "black",
     vertex.label.cex = 0.9,
     vertex.label.dist = 0,
     vertex.frame.color = 0,
     vertex.size = deg, 
     edge.arrow.size = 0.01,
     edge.curved = 0.7,
     edge.color = "gray",
     main = "Bigram Communities (Amazon Kindle)"
)
mtext("Degree")

# Using "Eigenvector Centrality" as size
# centrality (the most connected words)
plot(bigrams.network,
     layout = layout_with_fr,
     vertex.label.color = "black",
     vertex.label.cex = 0.8,
     vertex.label.dist = 0,
     vertex.size = eigen$vector*20, 
     edge.arrow.size = 0.01,
     edge.curved = 0.7,
     edge.color = "black",
     main = "Bigram Communities (Amazon Kindle)"
)
mtext("Eigenvector Centrality")

# Using "Betweenness" as size
# Betweenness -> median (weighted # of paths going through the node)
plot(bigrams.network,
     layout = layout_with_fr,
     vertex.label.color = "black",
     vertex.label.cex = 0.8,
     vertex.label.dist = 0,
     vertex.size = betw, 
     edge.arrow.size = 0.01,
     edge.curved = 0.7,
     edge.color = "lightgrey",
     main = "Bigram Communities (Amazon Kindle)"
)
mtext("Betweenness")