d4.3.Rmd

---
output: word_document
---

```{r, setup, echo=FALSE, message = FALSE}
knitr::opts_chunk$set(
  comment = "#>",
  collapse = TRUE,
  warning = FALSE,
  message = FALSE,
  echo = TRUE,
  fig.width = 6,
  fig.asp = 0.618,
  out.width = "70%",
  fig.align = "center",
  fig.path = "figure/",
  dpi = 300
)
options(scipen = 999, digits = 4)
knitr::knit_hooks$set(
  inline = function(x) {
    if (is.numeric(x)) {
      return(prettyNum(x, big.mark = ","))
    } else{
      return(x)
    }
  }
)
library(tidyverse)
library(bigrquery)
library(jsonlite)
library(DBI)
library(cowplot)
library(ggeconodist)
```

### NPL coverage

```{r}
con <- DBI::dbConnect(
  bigrquery::bigquery(),
  project = "api-project-764811344545"
)
```

```{r}
cpc_by_year <- readr::read_file("sql/cpc_by_year.sql")
cpc_by_year_df <- DBI::dbGetQuery(con, cpc_by_year)

patent_by_year <- cpc_by_year_df %>%
  group_by(pub_year) %>%
  summarise(all = sum(f0_))
  patent_by_year
```

```{r}
cpc_by_year_npl <- readr::read_file("sql/cpc_by_year_npl.sql")
cpc_by_year_npl_df <- DBI::dbGetQuery(con, cpc_by_year_npl)

npl_by_year <- cpc_by_year_npl_df %>%
  group_by(pub_year) %>%
  summarise(npl = sum(f0_))
npl_by_year
```

```{r, npl_plot}
npl_plot_df <- inner_join(npl_by_year, patent_by_year, by = "pub_year") %>%
  mutate(npl_lack = all - npl) %>%
  select(-all) %>%
  pivot_longer(!pub_year)
ggplot(npl_plot_df, aes(as.character(pub_year), value, fill = name, group = name)) +
  geom_area(position = position_stack(reverse = T)) +
   scale_fill_manual(
      values = c(npl_lack = "#cccccca0", npl = "#56b4e9"),
      name = "Cites non-patent-literature",
      labels = c("FALSE", "TRUE")
    ) +
  scale_y_continuous(expand = expansion(mult = c(0, 0.05)),
  limit = c(0, 3000000), labels = scales::label_number()) +
  cowplot::theme_minimal_hgrid() +
  labs(x = "Publication year", y = "Patent family") +
  theme(legend.position = "top",
          legend.justification = "right")
```

Total number of patents in sample:

`r sum(patent_by_year$all)`

Total number of patents citing at least one NPL

`r sum(npl_by_year$npl)`

Share 

`r sum(npl_by_year$npl) / sum(patent_by_year$all)`

## by subject

```{r}
cpc_patent_by_year <- cpc_by_year_df %>%
  group_by(cpc) %>%
  summarise(all = sum(f0_))

cpc_npl_by_year <- cpc_by_year_npl_df %>%
  group_by(cpc) %>%
  summarise(npl = sum(f0_))

cpc <- tibble::tribble(

~code, ~section,
"A", "Human necessities",
"B","Performing operations; transporting",
"C","Chemistry; metallurgy",
"D","Textiles; paper",
"E","Fixed constructions",
"F", "Mechanical engineering; lighting; heating; weapons; blasting engines or pumps",
"G","Physics",
"H","Electricity",
"Y","General"
)
cpc_df <- inner_join(cpc_patent_by_year, cpc_npl_by_year, by = "cpc") %>%
  inner_join(cpc, by = c("cpc" = "code")) %>%
  mutate(cpc = paste(cpc, section, sep = " - ")) %>%
  select(-section) %>%
  mutate(prop = round(npl / all * 100, 2))
knitr::kable(cpc_df, format.args = list(big.mark = ','))
```

## by country

```{r}
country_per_year <- readr::read_file("sql/country.sql")
country_per_year_df <- DBI::dbGetQuery(con, country_per_year)

country_npl_per_year <- readr::read_file("sql/country_npl.sql")
country_npl_per_year_df <- DBI::dbGetQuery(con, country_npl_per_year)

country_df <- inner_join(country_per_year_df, country_npl_per_year_df, by = "country_code") %>%
  mutate(country = countrycode::countrycode(country_code, origin = "iso2c", destination = 'country.name'))

top_10 <- country_df %>%
  top_n(10, patent_families) %>%
  pull(country_code)
country_df <- country_df %>%
  mutate(country_fac = ifelse(country_code %in% top_10, country, "Other")) %>%
  mutate(country_fac = ifelse(is.na(country_fac), "No Info", country_fac)) %>%
  mutate(country_fac = fct_inorder(country_fac)) %>%
  group_by(country_fac) %>%
  summarise(patent_families_with_npl = sum(patent_families_with_npl, na.rm = TRUE),
    patent_families = sum(patent_families)) %>%
    mutate(prop = round(patent_families_with_npl / patent_families * 100, 2))

country_df %>%
  select(Country = country_fac, `Patent families` = patent_families, `with NPL` = patent_families_with_npl, `in%` = prop) %>%
  knitr::kable()
```

### Number of NPLs per patent family

by inventor's country

```{r dimi_country}
npl_per_country <- readr::read_file("sql/npl_perc_per_country.sql")
npl_per_country_df <- DBI::dbGetQuery(con, npl_per_country)

npl_country_perc_df <- npl_per_country_df %>%
  mutate(country = countrycode::countrycode(country_code, origin = "iso2c", destination = 'country.name')) %>%
  mutate(country_fac = ifelse(country_code %in% top_10, country, "Other")) %>%
  mutate(country_fac = ifelse(is.na(country_fac), "No Info", country_fac)) %>%
  filter(country_code %in% top_10) %>%
  mutate(country_code = factor(country_code, levels = top_10)) %>%
  arrange(country_code = factor(country_code)) %>%
  mutate(country_fac = factor(country_fac, levels = country_fac)) %>%
  filter(!is.na(country))

a_plot <- npl_country_perc_df %>%
  ggplot() +
  geom_econodist(aes(x = reorder(country_fac, desc(country_fac)), ymin = p10, median = p50, ymax = p90),
         stat = "identity", show.legend = TRUE) +
  scale_y_continuous(position = "right", limits = c(0,max(npl_country_perc_df$p90)))  +  
  coord_flip() +
  labs(
    x = NULL, y = NULL
  ) +
  theme_minimal() +
  theme(panel.grid.minor = element_blank()) +
  theme(axis.ticks = element_blank()) +
  theme(panel.grid.major.y = element_blank()) +
  theme(panel.border = element_blank()) +
  theme(text = element_text(size = 14)) 


grid.newpage()
left_align(a_plot, c("subtitle", "title", "caption")) %>% 
  add_econodist_legend(econodist_legend_grob(family = "sans"), below = "subtitle") %>% 
  grid.draw()
```

by cpc

```{r npl_perc_per_cpc}
npl_per_cpc <- readr::read_file("sql/npl_perc_per_cpc.sql")
npl_per_cpc_df <- DBI::dbGetQuery(con, npl_per_cpc) %>%
  inner_join(cpc, by = c("cpc" = "code")) %>%
  mutate(cpc = paste(cpc, section, sep = " - ")) %>%
  select(-section)

a_plot <- npl_per_cpc_df %>%
  ggplot() +
  geom_econodist(aes(x = reorder(cpc, desc(cpc)), ymin = p10, median = p50, ymax = p90),
         stat = "identity", show.legend = TRUE) +
  scale_y_continuous(position = "right", limits = c(0,max(npl_per_cpc_df$p90)))  +  
  scale_x_discrete(labels = function(x) str_trunc(x, 35, side = "right")) +
  coord_flip() +
  labs(
    x = NULL, y = NULL) +
  theme_minimal() +
  theme(panel.grid.minor = element_blank()) +
  theme(axis.ticks = element_blank()) +
  theme(panel.grid.major.y = element_blank()) +
  theme(panel.border = element_blank()) +
  theme(text = element_text(size = 14)) 

grid.newpage()
left_align(a_plot, c("subtitle", "title", "caption")) %>% 
  add_econodist_legend(econodist_legend_grob(family = "sans"), below = "subtitle") %>% 
  grid.draw()
```

### OA

```{r}
oa_df <- readr::read_csv("data/oa_patent_matched_md.csv") %>%
    mutate(family_id = as.character(family_id)) %>%
    select(-publication_number_1, -family_id_1) %>%
    filter(year < 2021)
```

```{r}
# all
all_n <- length(unique(oa_df$pid))
all_n
# oa
oa_n <- length(unique(oa_df[oa_df$is_oa == TRUE,]$pid))
oa_n
oa_share <- oa_n / all_n
oa_share
```

```{r}
oa_all <- oa_df %>%
  group_by(is_oa, year) %>%
  summarise(n = n_distinct(pid)) 

pub_all <- oa_df %>%
  group_by(year) %>%
  summarise(all_n = n_distinct(pid))

oa_shares_by_year <- inner_join(pub_all, filter(oa_all, is_oa == TRUE), by = "year") %>%
  mutate(oa_share = n / all_n)
oa_shares_by_year

sum(oa_shares_by_year$all_n)

sum(oa_shares_by_year$n)

sum(oa_shares_by_year$n) / sum(oa_shares_by_year$all_n)


```

```{r oa_by_year}
oa_all %>%
  mutate(year = as.character(year)) %>%
  mutate(year = gsub("^.{2}", "'", year)) %>%
  ggplot(aes(year, n, fill = is_oa, group = is_oa)) +
  geom_area() +
   scale_fill_manual(
      values = c("#cccccca0", "#56b4e9"),
      name = NULL,
      labels = c("Closed", "Open Access")
    ) +
  scale_y_continuous(expand = expansion(mult = c(0, 0.05)),
    labels = scales::label_number()) +
  cowplot::theme_minimal_hgrid() +
  labs(x = "Publication Year", y = "SNPR") +
  theme(legend.position = "top",
          legend.justification = "right")
```

lag


```{r lap_patent_publication}
lag_df <- oa_df %>%
  filter(pub_year < 2021) %>%
  distinct(pub_year, family_id, year) %>%
  mutate(npl_year = substr(year, 1, 4)) %>%
  mutate(lag = as.integer(pub_year) - as.integer(npl_year))
ggplot(lag_df, aes(x = lag)) +
  geom_histogram(stat = "count", fill="#56B4E9") +
  geom_vline(aes(xintercept = median(lag, na.rm = T)),
             colour = "#E69F00", linetype ="dashed", size = .8) +
  cowplot::theme_minimal_hgrid() +
  scale_y_continuous(expand = expand_scale(mult = c(0, 0.05)),
  labels = scales::label_number()) +
  labs(y = "Patent families", x = "Yearly lag between SNPR publication and citation in patent")
```


by source

```{r src_lag}
oa_df %>%
  filter(pub_year < 2021) %>%
  distinct(pub_year, family_id, year, src) %>%
  mutate(npl_year = substr(year, 1, 4)) %>%
  mutate(lag = as.integer(pub_year) - as.integer(npl_year)) %>%
  ggplot(aes(x = lag, fill = src)) +
  geom_density(adjust = 5) +
  scale_fill_manual(
    name = "Data source",
    values = alpha(c("#f39100", "#b71a49", "#009bb4"), 0.9)) +
  labs(x = "Yearly lag between SNPR publication and citation in patent") +
  scale_y_continuous(expand = expand_scale(mult = c(0, 0.05))) +
  cowplot::theme_minimal_hgrid()
```

```{r src_oa_lag}
oa_df %>%
  filter(pub_year < 2021) %>%
  distinct(pub_year, family_id, year, is_oa) %>%
  mutate(npl_year = substr(year, 1, 4)) %>%
  mutate(lag = as.integer(pub_year) - as.integer(npl_year)) %>%
  ggplot(aes(x = lag, fill = is_oa)) +
  geom_density(adjust = 5) +
  scale_fill_manual(values = alpha(c("#f39100", "#b71a49", "#009bb4"), 0.9)) +
  cowplot::theme_minimal_hgrid()
```

by type

```{r}
type_all <- oa_df %>%
  distinct(pub_type, pid) %>%
  mutate(pub_type_fac = fct_lump_n(pub_type, 3)) %>%
  count(pub_type_fac, sort = TRUE) %>%
  mutate(prop = n / sum(n) * 100) 
type_oa <- oa_df %>%
  distinct(pub_type, pid, is_oa) %>%
  mutate(pub_type_fac = fct_lump_n(pub_type, 3)) %>%
  count(pub_type_fac, is_oa, sort = TRUE, name = "oa") %>%
  filter(is_oa == TRUE)
inner_join(type_all, type_oa, by = "pub_type_fac") %>%
  select(-is_oa) %>%
  mutate(oa_prop = oa / n *100) %>%
  knitr::kable()
```

#### preprint

```{r}
preprint_df <- readr::read_csv("data/oa_patent_matched_md.csv") %>%
    mutate(family_id = as.character(family_id)) %>%
    select(-publication_number_1, -family_id_1) %>%
    filter(year < 2021, pub_year < 2021) %>%
    filter(pub_type == "posted-content")
# number of preprints
length(unique(preprint_df$pid))
# number of preprint arxiv
preprint_df %>%
  group_by(src) %>%
  summarise(n = n_distinct(pid))
preprint_cr <- readr::read_csv("data/preprint_cr.csv") %>%
  # only patent families 
  filter(tolower(doi) %in% tolower(preprint_df$pid))
preprint_cr %>%
  count(preprint_server)

# sars covid
preprint_cr %>% filter(grepl("SARS", title, ignore.case = TRUE)) %>% count(preprint_server)
```

by cpc

```{r cpc_oa, fig.asp=0.8}
cpc_oa <- oa_df %>%
  group_by(cpc, is_oa) %>%
  summarise(n = n_distinct(pid)) %>%
  ungroup() %>%
  inner_join(cpc, by = c("cpc" = "code")) %>%
  mutate(cpc_label = paste(cpc, section, sep = " - "))

cpc_all <- oa_df %>%
  group_by(cpc) %>%
  summarise(all = n_distinct(pid))


cpc_oa %>%
  inner_join(cpc_all, by = "cpc") %>%
  mutate(cpc_labels =  str_trunc(cpc_label, 35, side = "right")) %>%
  mutate(cpc_labels = paste0(cpc_labels, "\n(n = ", prettyNum(all, ","), ")")) %>%
  ggplot(aes(reorder(cpc_labels, desc(cpc_labels)), n, fill = is_oa)) +
  geom_bar(position = "fill",stat = "identity") +
  geom_hline(yintercept = oa_share, colour = "#E69F00", linetype ="dashed", size = .8) +
  coord_flip() +
  scale_fill_manual(
      values = c("#cccccca0", "#56b4e9"),
      name = NULL,
      labels = c("Closed", "Open Access")
    ) +
  scale_y_continuous(expand = expansion(mult = c(0, 0.05)),
    labels = scales::percent) +
    cowplot::theme_minimal_vgrid() +
     labs(x = NULL, y = "SNPR OA Share") +
  theme(legend.position = "top",
          legend.justification = "right")

cpc_oa %>%
  inner_join(cpc_all, by = "cpc") %>%
  mutate(cpc_labels =  str_trunc(cpc_label, 35, side = "right")) %>%
  mutate(prop = n / all *100) %>%
  filter(is_oa == TRUE) %>%
  select(cpc_labels, prop)
```

by inventor country 

```{r country_oa, fig.asp=0.8}
cn_oa_df <- oa_df %>%
  inner_join(npl_country_perc_df, by = "country_code") %>%
  group_by(country_fac, is_oa) %>%
  summarise(n = n_distinct(pid))

cn_oa_all <- oa_df %>%
  inner_join(npl_country_perc_df, by = "country_code") %>%
  group_by(country_fac) %>%
  summarise(all = n_distinct(pid)) %>%
  mutate(cn_labels = paste0(country_fac, "\n(n = ", prettyNum(all, ","), ")"))

inner_join(cn_oa_df, cn_oa_all, by = "country_fac") %>%
  ggplot(aes(reorder(cn_labels, desc(country_fac)), n, fill = is_oa)) +
    geom_bar(position = "fill", stat = "identity") +
    coord_flip() +
    geom_hline(yintercept = oa_share, colour = "#E69F00", linetype ="dashed", size = .8) +
    scale_fill_manual(
        values = c("#cccccca0", "#56b4e9"),
        name = NULL,
        labels = c("Closed", "Open Access")
      ) +
      scale_y_continuous(expand = expansion(mult = c(0, 0.05)),
      labels = scales::percent) +
     # scale_x_discrete(labels = cn_labels) +
      cowplot::theme_minimal_vgrid() +
      labs(x = NULL, y = "SNPR OA Share") +
    theme(legend.position = "top",
            legend.justification = "right")

cn_oa_df <- oa_df %>%
  inner_join(npl_country_perc_df, by = "country_code") %>%
  group_by(country_fac, is_oa) %>%
  summarise(n = n_distinct(pid))

cn_oa_all <- oa_df %>%
  inner_join(npl_country_perc_df, by = "country_code") %>%
  group_by(country_fac) %>%
  summarise(all = n_distinct(pid)) %>%
  mutate(cn_labels = paste0(country_fac, "\n(n = ", prettyNum(all, ","), ")"))
inner_join(cn_oa_df, cn_oa_all, by = "country_fac") %>%
  filter(is_oa == TRUE) %>%
  mutate(prop = n / all * 100)
```

### oa by provider

```{r compare_cat, fig.width=7}
oa_cat <- readr::read_csv("data/oa_patent_matched_md_cat.csv") %>%
  filter(is_oa == TRUE, year < 2021) %>%
  mutate(provider_cat = gsub("Journal", "Publisher", provider_cat)) %>%
  mutate(provider_cat = factor(provider_cat, levels=c("Publisher only", "Publisher & Repository", "Repository only")))

oa_yrl_cat <- oa_cat %>%
  group_by(provider_cat, year) %>%
  summarise(n = n_distinct(pid)) 

cat_all <- oa_cat %>%
  group_by(year) %>%
  summarise(n = n_distinct(pid))


ggplot(oa_yrl_cat, aes(x = year, y = n)) +
  geom_bar(
    data = cat_all,
    aes(fill = "All OA Articles"),
    color = "transparent",
    stat = "identity"
  ) +
  geom_bar(aes(fill = "by Host"), color = "transparent", stat = "identity") +
  facet_wrap( ~ provider_cat, nrow = 1) +
  scale_fill_manual(values = c("#b3b3b3a0", "#56B4E9"), name = "") +
  scale_y_continuous(
      labels = scales::number_format(big.mark = ","),
      expand = expansion(mult = c(0, 0.05)),
      limits = c(0, 11500)) +
  #scale_x_continuous(labels = scales::label_number()) +
  labs(x = "Publication Year", y = "OA SNPR") +
  theme(legend.position = "top", legend.justification = "right") +
  cowplot::theme_minimal_hgrid() +
  theme(legend.position = "top",
           legend.justification = "right")

oa_cat %>%
  group_by(provider_cat) %>%
  summarise(n = n_distinct(pid)) %>%
  mutate(prop = n / sum(n))
```

```{r snpr_oa_type}
cat_all %>%
  rename(n_all = n) %>%
  inner_join(oa_yrl_cat, by = "year") %>%
  mutate(prop = n / n_all) %>%
  mutate(year = gsub("^.{2}", "'", year)) %>%
  ggplot(aes(year, prop, fill = provider_cat)) +
  geom_bar(stat = "identity") +
  scale_fill_manual(values = c("#E69F00", "#0072B2", "#009E73"), name = NULL,
    guide = guide_legend(reverse = TRUE)) +
   scale_y_continuous(expand = expansion(mult = c(0, 0.05)),
    labels = scales::percent) +
    cowplot::theme_minimal_hgrid() +
     labs(x = "Publication year", y = "SNPR OA") +
  theme(legend.position = "top",
          legend.justification = "right")
```

by cpc

```{r provider_cat_cpc, fig.asp=0.8}
cpc_oa <- oa_cat %>%
  group_by(cpc, provider_cat) %>%
  summarise(n = n_distinct(pid)) %>%
  ungroup() %>%
  inner_join(cpc, by = c("cpc" = "code")) %>%
  mutate(cpc_label = paste(cpc, section, sep = " - "))

cpc_all <- oa_cat %>%
  group_by(cpc) %>%
  summarise(all = n_distinct(pid))


cpc_oa %>%
  inner_join(cpc_all, by = "cpc") %>%
  mutate(cpc_labels =  str_trunc(cpc_label, 35, side = "right")) %>%
  mutate(cpc_labels = paste0(cpc_labels, "\n(n = ", prettyNum(all, ","), ")")) %>%
  ggplot(aes(reorder(cpc_labels, desc(cpc_labels)), n, fill = provider_cat)) +
  geom_bar(position = "fill",stat = "identity") +
  coord_flip() +
  scale_fill_manual(values = c("#E69F00", "#0072B2", "#009E73"),
  guide = guide_legend(reverse = TRUE),
  name = NULL) +
  scale_y_continuous(expand = expansion(mult = c(0, 0.05)),
    labels = scales::percent) +
    cowplot::theme_minimal_vgrid() +
     labs(x = NULL, y = "SNPR OA") +
  theme(legend.position = "top",
          legend.justification = "right")

cpc_oa %>%
  inner_join(cpc_all, by = "cpc") %>%
  mutate(cpc_labels =  str_trunc(cpc_label, 35, side = "right")) %>%
  mutate(prop = n / all * 100) %>%
  select(cpc_label, provider_cat, n, prop) %>%
  print(n=40)
```

by country

```{r provider_cat_country, fig.asp=0.8}

cat_oa_df <- oa_cat %>%
  inner_join(npl_country_perc_df, by = "country_code") %>%
  group_by(country_fac, provider_cat) %>%
  summarise(n = n_distinct(pid))

cat_oa_all <- oa_cat %>%
  inner_join(npl_country_perc_df, by = "country_code") %>%
  group_by(country_fac) %>%
  summarise(all = n_distinct(pid)) %>%
  mutate(cn_labels = paste0(country_fac, "\n(n = ", prettyNum(all, ","), ")"))

inner_join(cat_oa_df, cat_oa_all, by = "country_fac") %>%
  ggplot(aes(reorder(cn_labels, desc(country_fac)), n, fill = provider_cat)) +
    geom_bar(position = "fill", stat = "identity") +
    coord_flip() +
    scale_fill_manual(values = c("#E69F00", "#0072B2", "#009E73"),
  guide = guide_legend(reverse = TRUE),
  name = NULL)  +
      scale_y_continuous(expand = expansion(mult = c(0, 0.05)),
      labels = scales::percent) +
     # scale_x_discrete(labels = cn_labels) +
      cowplot::theme_minimal_vgrid() +
      labs(x = NULL, y = "SNPR OA") +
    theme(legend.position = "top",
            legend.justification = "right")
```

oa benchmark

```{r upw_comparision}
upw <- readr::read_file("sql/upw_by_year.sql")
upw_by_year <- DBI::dbGetQuery(con, upw)
upw_all <- upw_by_year %>%
  group_by(year) %>%
  summarise(all = sum(n))
upw_oa_by_year <- upw_by_year %>%
  filter(is_oa == TRUE) %>%
  inner_join(upw_all) %>%
  mutate(prop = n / all) %>%
  arrange(year) %>%
  mutate(year = as.character(year))
# snpr oa journals
oa_jn <- oa_df %>%
  filter(pub_type == "journal-article", is_oa == TRUE, src == "Unpaywall") %>%
  group_by(is_oa, year) %>%
  summarise(n_oa = n_distinct(pid))
jn_all <-  oa_df %>%
  filter(pub_type == "journal-article") %>%
  group_by(year) %>%
  summarise(n = n_distinct(pid))

oa_prop_df <- inner_join(oa_jn, jn_all, by = "year") %>%
  mutate(oa_share_npl = n_oa / n) %>%
  mutate(year = as.character(year)) %>%
  select(year, oa_share_npl) %>% 
  inner_join(upw_oa_by_year, by = "year") %>%
  select(year, `Unpaywall total` = prop, `SNPR subset` = oa_share_npl) %>%
  pivot_longer(c(`Unpaywall total`,  `SNPR subset`))
oa_prop_df %>%
  mutate(year = gsub("^.{2}", "'", year)) %>%
  ggplot(aes(as.character(year), value, group = name, color = name)) +
  geom_line() +
  geom_point(color = "grey20") +
  scale_y_continuous(labels = scales::label_percent(accuracy = 1L)) +
  scale_color_manual(values = c("#b71a49", "#009bb4"), name = NULL) +
  labs(x = "Publication year", y = "OA percentage") +
  guides(colour = guide_legend(reverse = TRUE)) +
  cowplot::theme_minimal_hgrid() +
  theme(legend.position = "top",
          legend.justification = "right")
oa_prop_df
```