---
title: "EDA with R Markdown Code"
author: "Jonathan W. Duggins" 
date: "8/10/2022"
output:
  html_document:
    toc: true
    toc_float: true
    code_folding: hide
---

```{r setup, include=FALSE}
r = getOption("repos")
r["CRAN"] = "http://cran.us.r-project.org"
options(repos = r)
pax <- c("tidyverse","huxtable","scales","clean","Hmisc","pander","leaflet","DT")
install.packages(setdiff(pax,rownames(installed.packages())))
knitr::opts_chunk$set(echo = FALSE, cache = TRUE)
library(tidyverse)
library(huxtable)
library(scales)
library(clean)
library(Hmisc)
```

## Tables with `huxtable`
```{r hux, echo = TRUE, eval = TRUE}
summary(cars)
```

## Tables with `huxtable`
```{r hux2, echo = TRUE, eval = TRUE}
as_hux(summary(cars),add_rownames = FALSE) %>% set_bold(row=1,col=everywhere) %>%
  set_font_size(row=1,,24) %>% set_all_padding(1) %>% theme_striped()
```

## Interactivity with Leaflet
```{r leafletP, echo = TRUE, eval = FALSE}
library(leaflet)
leaflet() %>%
  setView(-87.6553,41.9485 , zoom = 16) %>% 
  addTiles() %>%
  addMarkers(-87.6553,41.9485 , popup = "Wrigley!") 
```

## Interactivity with Leaflet
```{r leafletE, echo = FALSE, eval = TRUE}
library(leaflet)
leaflet() %>%
  setView(-87.6553,41.9485 , zoom = 16) %>% 
  addTiles() %>%
  addMarkers(-87.6553,41.9485 , popup = "Wrigley!") 
```

## Interactivity with DataTables
```{r DTP, echo = TRUE, eval = FALSE}
library(DT)
datatable(ToothGrowth)
```

## Interactivity with DataTables
```{r DTE, echo = FALSE, eval = TRUE}
library(DT)
datatable(ToothGrowth)
```

## Interactivity with JavaScript
```{r scatter3dP, eval = FALSE}
if(!require("devtools")) install.packages("devtools")
devtools::install_github("bwlewis/rthreejs")

library(threejs)

scatterplot3js(x = iris$Sepal.Width, y = iris$Sepal.Length, 
							 z = iris$Petal.Width, color = 
							 	c(rep("blue", 50), rep("red", 50), 
							 		rep("green", 50)), 
							 size = 0.5)
```

## Interactivity with JavaScript
```{r scatter3dE, echo = FALSE, message = FALSE, warning = FALSE}
library(threejs)

scatterplot3js(x = iris$Sepal.Width, y = iris$Sepal.Length, 
							 z = iris$Petal.Width, color = 
							 	c(rep("blue", 50), rep("red", 50), 
							 		rep("green", 50)), 
							 size = 0.5)
```

## Know Your Data!
```{r, echo=TRUE, eval = FALSE}
#install.packages(c("dslabs", "pander"))
library(dslabs)
library(pander)
glimpse(gapminder)
```

## Choosing Your Summary Statistics
```{r, echo = TRUE}
gapminder %>% 
  select(infant_mortality) %>% 
    summarise(var = "Infant Mortality",
              Avg = mean(infant_mortality, na.rm = TRUE),
              SD = sd(infant_mortality, na.rm = TRUE),
              Med = median(infant_mortality, na.rm = TRUE),
              IQR = IQR(infant_mortality, na.rm = TRUE),
              mid90 = quantile(infant_mortality,0.95, na.rm = TRUE) -
                        quantile(infant_mortality, 0.05, na.rm = TRUE)
             ) %>%
      pander()
```

## Displaying Multiple Univariate Analyses
```{r, echo = FALSE}
gapminder %>% 
  select(infant_mortality) %>% 
    summarise(var = "Infant Mortality",
              Avg = mean(infant_mortality, na.rm = TRUE),
              SD = sd(infant_mortality, na.rm = TRUE),
              Med = median(infant_mortality, na.rm = TRUE),
              IQR = IQR(infant_mortality, na.rm = TRUE),
              mid90 = quantile(infant_mortality,0.95, na.rm = TRUE) -
                        quantile(infant_mortality, 0.05, na.rm = TRUE)
             ) %>%
      pander()
gapminder %>% 
  select(life_expectancy) %>% 
    summarise(var = "Life Expectancy",
              Avg = mean(life_expectancy, na.rm = TRUE),
              SD = sd(life_expectancy, na.rm = TRUE),
              Med = median(life_expectancy, na.rm = TRUE),
              IQR = IQR(life_expectancy, na.rm = TRUE),
              mid90 = quantile(life_expectancy,0.95, na.rm = TRUE) -
                        quantile(life_expectancy, 0.05, na.rm = TRUE)
             ) %>%
      pander()
```

## Combining Multiple Univariate Analyses
```{r, echo = FALSE}
infMort <- gapminder %>% 
  select(infant_mortality) %>% 
    summarise(var = "Infant Mortality",
              Avg = mean(infant_mortality, na.rm = TRUE),
              SD = sd(infant_mortality, na.rm = TRUE),
              Med = median(infant_mortality, na.rm = TRUE),
              IQR = IQR(infant_mortality, na.rm = TRUE),
              mid90 = quantile(infant_mortality,0.95, na.rm = TRUE) -
                        quantile(infant_mortality, 0.05, na.rm = TRUE)
             )
lifeExp <- gapminder %>% 
  select(life_expectancy) %>% 
    summarise(var = "Life Expectancy",
              Avg = mean(life_expectancy, na.rm = TRUE),
              SD = sd(life_expectancy, na.rm = TRUE),
              Med = median(life_expectancy, na.rm = TRUE),
              IQR = IQR(life_expectancy, na.rm = TRUE),
              mid90 = quantile(life_expectancy,0.95, na.rm = TRUE) -
                        quantile(life_expectancy, 0.05, na.rm = TRUE)
             )
fert <- gapminder %>% 
  select(fertility) %>% 
    summarise(var = "Fertility",
              Avg = mean(fertility, na.rm = TRUE),
              SD = sd(fertility, na.rm = TRUE),
              Med = median(fertility, na.rm = TRUE),
              IQR = IQR(fertility, na.rm = TRUE),
              mid90 = quantile(fertility,0.95, na.rm = TRUE) -
                        quantile(fertility, 0.05, na.rm = TRUE)
             )
results <- bind_rows(infMort, lifeExp, fert)
pander(results)
```

## Grouped Analysis
```{r, echo = TRUE}
gapminder %>% 
  select(continent, infant_mortality) %>% 
    group_by(continent) %>% 
      summarise(Avg = mean(infant_mortality, na.rm=TRUE), 
                SD = sd(infant_mortality, na.rm=TRUE), 
                Med = median(infant_mortality, na.rm=TRUE), 
                IQR = IQR(infant_mortality, na.rm=TRUE), 
                Mid90 = quantile(infant_mortality,.95,na.rm=TRUE) - 
                          quantile(infant_mortality,.05,na.rm=TRUE))
```

## Step 5: Select and Order Columns
```{r, echo = TRUE}
#install.packages("scales")
#library(scales)
gapminder %>%
  group_by(continent) %>%
    summarise(Count = n()) %>%
      mutate(.rf = Count/sum(Count), .crf = cumsum(.rf), Rank = row_number(desc(.rf))) %>%
        mutate(Count = comma(Count), `Relative Frequency` = round(100*.rf,1), `Cumulative Relative Frequency` = round(100*.crf,1)) %>%
          select(continent, Rank, !(starts_with(".")))
```

## Frequency Table- Option \#2

```{r, echo = TRUE}
#install.packages("clean")
#library(clean)
gapminder %>% select(continent) %>% freq()
```

## Option \#1 is More Flexible!
```{r, echo = FALSE, eval = TRUE}
gapminder %>%
  group_by(continent) %>%
    summarise(Count = n()) %>%
      arrange(desc(Count)) %>% 
        mutate(Difference = Count - lag(Count,1),
               CumDiff = cumsum(ifelse(is.na(Difference),0,Difference)))
```

## Step 3: Cumulative Difference
```{r, echo = TRUE}
gapminder %>%
  group_by(continent) %>%
    summarise(Count = n()) %>%
      arrange(desc(Count)) %>% 
        mutate(Difference = Count - lag(Count,1),
               CumDiff = cumsum(Difference))
```

## Aside: `cumsum` function
```{r, echo = TRUE, error = TRUE, message = TRUE}
gapminder %>%
  group_by(continent) %>%
    summarise(Count = n()) %>%
      arrange(desc(Count)) %>% 
        mutate(Difference = Count - lag(Count,1),
               CumDiff = cumsum(Difference, na.rm = TRUE))
```

## Step 3 (again): Try Using a function!
```{r, echo = TRUE, error = TRUE, message = TRUE}
gapminder %>%
  group_by(continent) %>%
    summarise(Count = n()) %>%
      arrange(desc(Count)) %>% 
        mutate(Difference = Count - lag(Count,1),
               CumDiff = cumsum(na.omit(Difference)))
```

## Step 3 (yet again): Using Logic!
```{r, echo = TRUE}
gapminder %>%
  group_by(continent) %>%
    summarise(Count = n()) %>%
      arrange(desc(Count)) %>% 
        mutate(Difference = Count - lag(Count,1),
               CumDiff = cumsum(ifelse(is.na(Difference),0,Difference)))
```

## Frequency Table - Option \#3
```{r, echo = TRUE}
gapminder %>% select(continent) %>% table()
```

## Two-Way Tables
```{r, echo = TRUE}
g<- gapminder %>% mutate(TFLcutoff = ifelse(fertility<=2.1,"At or below", "Exceeds"))
table(g$continent, g$TFLcutoff)
```

## Covariance
```{r, echo = TRUE}
gapminder[complete.cases(gapminder),] %>% 
  select(infant_mortality:population) %>% 
    cov()
```

## Pearson Correlation
```{r, echo = TRUE}
gapminder[complete.cases(gapminder),] %>% 
  select(infant_mortality:population) %>% 
    cor() #Pearson is default method
```

## Other Correlation Methods
```{r, echo = TRUE}
gapminder[complete.cases(gapminder),] %>% 
  select(infant_mortality:population) %>% 
    cor(method = "spearman")
```

## P-values!
```{r, echo = TRUE, eval = FALSE}
install.packages("Hmisc")
library(Hmisc)
```

```{r, echo = FALSE, eval = TRUE, message = FALSE, error = FALSE}
#install.packages("Hmisc", repos = "http://cran.us.r-project.org")
#library(Hmisc)
```

## Example of `rcorr`
```{r, echo = TRUE}
gapminder[complete.cases(gapminder),] %>% 
  select(infant_mortality:population) %>% as.matrix() %>% rcorr()
```

## Goal: Build a Bar Chart
```{r, echo = FALSE, eval = TRUE}
ggplot(data = gapminder, aes(x = continent)) + 
  geom_bar() +
  labs(x = "Continent", y = "Frequency", title = "Absolute Frequency of Responses in Gapminder Data") + 
  scale_y_continuous(labels = comma)
```

## Building a Grouped Bar Chart
```{r, echo = FALSE, eval = TRUE}
g2 <- gapminder %>% subset(is.na(fertility) == FALSE)

ggplot(data = g2, aes(x = continent, fill = (fertility<2.1))) + 
  geom_bar(position = "dodge") +
  labs(x = "Continent", y = "Frequency", title = "Absolute Frequency of Responses in Gapminder Data", subtitle = "Grouped by Total Fertility Rate < 2.1") + 
  scale_y_continuous(labels = comma) + 
  scale_fill_discrete(name = "TFR<2.1", labels = c("No", "Yes"))
```

## `facet`ing Your Graph
```{r, echo=TRUE, fig.dim=c(6,2.5)}
g2 <- gapminder %>% subset(is.na(fertility) == FALSE & year > 2011)
ggplot(data = g2, aes(x = continent, fill = (fertility<2.1))) + 
  geom_bar(position = "dodge") +
  labs(x = "Continent", y = "Frequency", 
       title = "Absolute Frequency of Responses in Gapminder Data", 
       subtitle = "Grouped by Total Fertility Rate < 2.1") + 
  scale_y_continuous(labels = comma) + 
  scale_fill_discrete(name = "TFR<2.1", labels = c("No", "Yes")) + 
  facet_wrap(~year)
```

## Using `facet_grid`
```{r, echo=TRUE, fig.dim=c(7,3.5)}
g3 <- gapminder %>% subset(is.na(fertility) == FALSE & year > 2013)
ggplot(data = g3, aes(x = continent, fill = (fertility<2.1))) + 
  geom_bar(position = "fill") +
  facet_grid((population > 1000000)~year)
```

## Creating a Histogram
```{r, echo = TRUE, message = FALSE}
c <- ggplot(data = gapminder, aes(x = fertility))
c + geom_histogram(na.rm = TRUE)
```

## Modifying our Histogram
```{r, echo = TRUE, fig.dim=c(7,3.5)}
c + geom_histogram(color = "blue", linetype = 2, linewidth = 0.75,
                   fill = "#FF0000", alpha = 0.5, 
                   binwidth = 0.25, na.rm = TRUE)
```

## First, Just the Density
```{r, echo = TRUE}
ggplot(gapminder, aes(x=fertility)) + 
  geom_density(kernel = "gaussian", na.rm = TRUE)
```

## Testing out `adjust`
::: {style="float:left; width:50%;"}
```{r, echo = TRUE, fig.dim=c(5,2.5)}
ggplot(gapminder, aes(x=fertility)) + 
  geom_density(kernel = "gaussian", 
               adjust = .1, 
               na.rm = TRUE)
```
:::

::: {style="float:left; width:50%;"}
```{r, echo = TRUE, fig.dim=c(5,2.5)}
ggplot(gapminder, aes(x=fertility)) + 
  geom_density(kernel = "gaussian", 
               adjust = 10, 
               na.rm = TRUE)
```
:::

## Histogram + (New) Density
```{r, echo = TRUE, fig.dim=c(7,3.5)}
ggplot(gapminder, aes(x=fertility)) + 
  geom_histogram(aes(y = after_stat(density)), alpha = 0.5, 
                   binwidth = 0.25, na.rm = TRUE) + 
  geom_density(kernel = "triangular", linewidth = 1.25, color = "red", na.rm = TRUE)
```

## Grouping with Histograms
```{r, echo = TRUE, message = FALSE, fig.dim=c(7,3.5)}
g4 <- gapminder %>% subset(is.na(fertility)==FALSE & year > 2013)
ggplot(g4, aes(x=fertility, fill=factor(year))) + 
  geom_histogram(alpha=.5)
```

## Grouping with Densities
```{r, echo = TRUE, fig.dim=c(6.5,3.25)}
g4 <- gapminder %>% subset(is.na(fertility)==FALSE & year > 2013)
ggplot(g4, aes(x=fertility, fill=factor(year))) + 
  geom_density(alpha=.5)
```

## Grouping with Overlain Histograms and Densities
```{r, echo = TRUE, fig.dim=c(7,3.5)}
g4 <- gapminder %>% subset(is.na(fertility)==FALSE & year > 2013)
ggplot(g4, aes(x=fertility, fill=as.factor(year))) + 
  geom_histogram(aes(y = after_stat(density)), bins = 50) + 
  geom_density(alpha=.5)  
```

## "Proper" Grouping
```{r, echo = TRUE, fig.dim=c(7,3.5)}
g4 <- gapminder %>% subset(is.na(fertility)==FALSE & year > 2013)
ggplot(g4, aes(x=fertility, fill=factor(year))) + 
  geom_histogram(aes(y = after_stat(density)), bins = 50) + 
  geom_density(alpha=.5, position = "stack")  
```

## Box Plot
```{r, echo = TRUE, fig.dim=c(6,3)}
g5 <- gapminder %>% subset(is.na(fertility)==FALSE & year > 2009)
c <- ggplot(g5, aes(x=factor(year), y = fertility))
c + geom_boxplot(fill="grey") 
      labs(x="Year")
```

## Box Plot + Points
```{r, echo = TRUE, fig.dim=c(7,3.5)}
g5 <- gapminder %>% subset(is.na(fertility)==FALSE & year > 2009)
c <- ggplot(g5, aes(x=factor(year), y = fertility))
c + geom_boxplot(fill="grey") + 
      geom_jitter(width=.15, color = "blue") + labs(x="Year")
```

## Scatter
```{r, echo = TRUE, fig.dim=c(6,3)}
g6 <- gapminder %>% subset(is.na(fertility) == FALSE & 
                           is.na(infant_mortality) == FALSE &
                           year > 2012)
ggplot(g6, aes(x=fertility, y=infant_mortality)) + 
  geom_point(color="blue", shape = 3, size=2, stroke = .5)
```

## Grouping: Results
```{r, echo = TRUE, fig.dim=c(7,3.5)}
ggplot(g6, aes(x=fertility, y=infant_mortality)) + 
  geom_point(size=2, stroke = .5, aes(shape = factor(year), color = continent)) + 
  labs(x="Total Fertility Rate", y = "Infant Mortality Rate") + 
  scale_shape_discrete(name = "Year", labels = c("'13", "'14", "'15")) + 
  scale_color_discrete(name = "Continent")
```

## Scatter + Trend: Results
```{r, echo = TRUE, message = FALSE, fig.dim=c(7,3.5)}
ggplot(g6, aes(x = fertility, y = infant_mortality)) +
  geom_point() +  
  geom_smooth(aes(col = "loess")) +
  geom_smooth(method = lm, aes(col = "bob")) + 
  scale_colour_manual(name = 'Smoother', values =c('bob'='red', 'loess'='purple'), 
                      labels = c('Linear','GAM'), guide = 'legend')
```
