1.0_read.Rmd

---
title: "Read"
date: "`r Sys.Date()`"
output:
  html_document:
    toc: true
    toc_depth: 3
    toc_float:
      collapsed: FALSE
    code_folding: show
    number_sections: TRUE
---

# SET-UP

```{r markdown, echo=F}
library(knitr)
opts_chunk$set(echo = T, collapse = T, fig.align = "center", fig.width = 9, fig.height = 6)
options(width = 140)
```    

```{r settings, message = F}
source('0.0_settings.R')
```

# CATCH
## ZIFF

```{r ziff, message = F}
ziffR <- paste0(dir.rdat, "ziff.Rdata")
sp_code <- find.species(my.species,'ziff');sp_code

if(!file.exists(ziffR)){
    ziff <- read.ziff(sp_code,path=dir.ziff,year=my.year,language='en')
    
    # Species included:
    with(ziff, table(species,year))
    ziff <- ziff[ziff$species==250,]   # code 960: few heads in 1996  code 951: wrong whelk

    save(ziff, file = ziffR)
}else{
    load(ziffR)
}

```
### CHECK {.tabset}
#### areas
```{r ziff areas}
# NAFO 5 = landings in Canadian EEZ (northern contingent)
kable(with(ziff, table(nafo,month)))     
```
#### NA weights
```{r ziff nas}
na <- cbind(all=with(ziff,table(year,useNA = 'always')),
                na=with(ziff[is.na(ziff$catch),],table(year,useNA = 'always')))
kable(na)
```
#### 0 weights
```{r ziff zeros}
with(ziff[!is.na(ziff$catch) & ziff$catch==0,],table(year))
```
#### Jan-1
Check that there is no abnormal quantity of landings on January first (possibly NA)
```{r ZIFF_jan1}
ggplot(ziff,aes(x=lubridate::yday(capt.date)))+geom_histogram(binwidth = 1)+
    scale_x_continuous(expand=c(0,0))+
    scale_y_continuous(expand=c(0,0))
```

## NAFO

```{r nafo, message = F}
nafoAR <- paste0(dir.rdat, "nafoA.Rdata")
nafoBR <- paste0(dir.rdat, "nafoB.Rdata")

if(!all(file.exists(c(nafoAR,nafoBR)))){
    ### NAFO database A (annual) ---------------------------------
    sp_code <- find.species(my.species,'nafoA');sp_code
    nafoA <- read.nafoA(paste0(dir.nafo,'statlandA.csv'),species=sp_code[1], year=my.year) 

    # Canadian caught mackerel outside of my.nafo
    with(nafoA[!grepl(paste0(my.nafo, collapse = "|"),nafoA$nafo)&
                 grepl('canada',nafoA$country,ignore.case = TRUE),] ,table(year,nafo))

    # only keep my.nafo; other areas are NOT from nothern contingent (winter months in US or Europe)
    nafoA <- nafoA[grepl(paste0(my.nafo, collapse = "|"),nafoA$nafo),] 
    table(nafoA$nafo)
    
    save(nafoA, file = nafoAR)
    
    ### NAFO data base B (monthly) ---------------------------------
    sp_code <- find.species(my.species,'nafoB');sp_code
    nafoB <- read.nafoB(path=dir.nafo,species=sp_code[1],year=my.year,overwrite=FALSE)     

    # Canadian caught mackerel outside of my.nafo
    with(nafoB[!grepl(paste0(my.nafo, collapse = "|"),nafoB$nafo)&
                 grepl('canada',nafoB$country,ignore.case = TRUE),] ,table(year,nafo))
    with(nafoB[!grepl(paste0(my.nafo, collapse = "|"),nafoB$nafo)&
                 grepl('canada',nafoB$country,ignore.case = TRUE),] ,table(month,nafo))
    
    # only keep my.nafo  (though 5Ze in summer; wouldn't they be northern contingent fish??)
    nafoB <- nafoB[grepl(paste0(my.nafo, collapse = "|"),nafoB$nafo),] 
    table(nafoB$nafo)
    
    # remove useless columns 
    nafoB <- nafoB[,c('year','month','country','nafo','gear','gear.cat','tonnage.class','prov','catch')]

    save(nafoB, file = nafoBR)
}else{
    load(nafoAR)
    load(nafoBR)
}
```
### CHECK {.tabset}
#### compare
```{r NAFO_comparison}
ggplot()+
    geom_bar(data=nafoA,aes(x=year,y=catch),stat='identity',alpha=0.2,fill='blue')+
    geom_bar(data=nafoB,aes(x=year,y=catch),stat='identity',alpha=0.2,fill='red')+
    geom_vline(xintercept=1985)+
    scale_x_continuous(expand=c(0,0))+ scale_y_continuous(expand=c(0,0))
```

## CATCH = NAFO + ZIFF

```{r CATCH, message = F}
## bind both
ziff$source <-'ziff'
ziff$catch <- ziff$catch/1000            # kg to tonnes
ziff$country <- 'Canada'                 # consistency with nafo
ziff$prov <- ifelse(ziff$prov.land=='Unknown',as.character(ziff$prov.home),as.character(ziff$prov.land))

nafoB$source <- 'nafo'                   # work with detailed B base

sel <- c('year','month','nafo','gear.cat','source','catch','country','prov')
catch <- rbind(ziff[ziff$year>=1995,sel],        # ziff from 1995 onwards
               nafoB[nafoB$year<1995,sel],       # nafoB before
               nafoB[nafoB$year>=1995 & nafoB$country!='Canada',sel]) # foreign post 1994

save(catch, file = paste0(dir.rdat, "catch.Rdata"))

plotCatch(catch,x='year',y='catch')
```

# LENGTH-FREQUENCIES

```{r lf, message = F}
lfR <- paste0(dir.rdat, "lf.Rdata")
if(!file.exists(lfR)){
    lf <- read.lf(file=paste0(dir.bio,'lf.dat'), year=my.year)

    # correct weight for state (conversion factors in catchR/data_raw/lf_fishshape.csv)
    lf$weight.land <- correct.weight(lf$weight.land,lf$state.id,species=my.species)
    lf$weight.sample <- correct.weight(lf$weight.sample,lf$state.id,species=my.species)
    
    save(lf, file = lfR)
}else{
    load(lfR)
}
```

# BIO
```{r bio, message = F}
bioR <- paste0(dir.rdat, "bio.Rdata")
check.distribution <- FALSE

if(!file.exists(bioR)){
    bio <- read.bio(file=paste0(dir.bio,'carbio.dat'),year=my.year,species=my.species)
    
    # add static ids if missing (here by month,day,nafo,gear), for mackerel always when no length is available
    v <- bio[is.na(bio$sample.id),]
    idchar <- do.call(paste0,list(month(v$date),day(v$date),v$nafo,v$gear))

    # add info on whether samples were taken randomly or not (see readme.txt)
    random <- read.csv2(file=paste0(dir.bio,'carbio_random.csv'),sep=',')   # update after new bio
    random$date <- as.Date(random$date,format = '%d-%m-%y')
    bio <- merge(bio,random,all.x = T)
    
    # correct mistakes
    bio[bio$nafo=='4WE','nafo'] <- '4W'
    
    # save
    save(bio, file = bioR)
    
    # check distributions (random samples will be appended to lf database)
    if(check.distribution){
      source('1.0_read_bio.R')
    }
}else{
    load(bioR)
}
```