---
title: "2_linking_repair_annotations"
author: "Marlou"
date: "24-09-2021"
output: html_document
---

```{r}
library(dplyr)
```

#import data
```{r}
#set colclasses for pairnr otherwise it drops leading 0 (which we need for sorting)
df_repair <- read.csv2("processing_output/1_repair.csv", header=T, colClasses = c(pairnr = "character")) 
df_speech <- read.csv2("processing_output/1_speech.csv", header=T, colClasses = c(pairnr = "character"))
df_trials <- read.csv2("processing_output/1_trials.csv", header=T, colClasses = c(pairnr = "character"))


```

#change col types
```{r}
#df_repair

#most cols are factors, change all character cols to factors
df_repair <- df_repair %>% 
  mutate(across(where(is.character), as.factor))

#but set speech and onset/offset cols to character
df_repair <- df_repair %>%
  mutate(A_speech = as.character(A_speech),
          B_speech = as.character(B_speech),
          onset = as.character(onset),
          offset = as.character(offset))


#df_speech
df_speech <- df_speech %>% 
  mutate(across(where(is.character), as.factor))
df_speech <- df_speech %>%
  mutate(A_speech = as.character(A_speech),
          B_speech = as.character(B_speech),
          onset = as.character(onset),
          offset = as.character(offset))

#df_trials
df_trials <- df_trials %>% 
  mutate(across(where(is.character), as.factor))
df_trials <- df_trials %>%
  mutate(onset = as.character(onset),
          offset = as.character(offset),
          target = as.factor(target))

```



#REPAIR + SPEECH
In df_repair only the speech for single TCU annotations are included; for Tmin1/T0/Tplus1 annotations spanning more than one TCU, the speech cols are NA --> we have to 'find' the speech in df_speech

```{r}
#set up repair+speech df
df_repair_speech <- df_repair
#sort it by pairnr, then onset
df_repair_speech <- df_repair_speech %>%
 arrange(pairnr, onset_msec)

```

For this we will use a loop, which needs to be executed for every pair separately
Here we create 'large list' formats using the split function, which contains multiple dataframes, one for each pair
```{r}
#create 'large lists'
df_repair_speech_pairs <- split(df_repair_speech, df_repair_speech$pairnr)
df_speech_pairs <- split(df_speech, df_speech$pairnr)

#create a list of pairnrs to loop over them later
list_pairnrs <- levels(df_speech$pairnr)
```

*takes about 1 or 2 min*
```{r}
for (p in 1:length(list_pairnrs)) { #for every pair:
  for (x in 1:nrow(df_repair_speech_pairs[[p]])){ #loop over repair_speech df (i.e., loop over repair annotations)
    #if no speech there yet for BOTH participants, i.e., if there are multiple TCUs included in the repair annotation
    if (is.na(df_repair_speech_pairs[[p]][x,]$A_speech)&is.na(df_repair_speech_pairs[[p]][x,]$B_speech)) {  
    #then store the onset and offset of the repair annotation
      onset_msec <- df_repair_speech_pairs[[p]][x,]$onset_msec
      offset_msec <- df_repair_speech_pairs[[p]][x,]$offset_msec
    #and start looping over the speech_df to 'find' the speech (from either ppA or ppB)
    #set up empty character strings for 'intermittent' speech (can be multiple TCUs, will be pasted together)
      intermittent_TCUs_A <- ""
      intermittent_TCUs_B <- ""
      for (y in 1:nrow(df_speech_pairs[[p]])){
        if (onset_msec == df_speech_pairs[[p]][y,]$onset_msec) { #if onset of repair annotation matches onset of a TCU, it is the first
          first_TCU_A <- df_speech_pairs[[p]][y,]$A_speech
          first_TCU_B <- df_speech_pairs[[p]][y,]$B_speech
      }
        if (offset_msec == df_speech_pairs[[p]][y,]$offset_msec) { #if offset of repair annotation matches offset of a TCU, it is the last
          last_TCU_A <- df_speech_pairs[[p]][y,]$A_speech
          last_TCU_B <- df_speech_pairs[[p]][y,]$B_speech
      }
        if (onset_msec < df_speech_pairs[[p]][y,]$onset_msec & offset_msec > df_speech_pairs[[p]][y,]$offset_msec) { #if TCU is between repair annotation boundaries, it is intermittent
          intermittent_TCUs_A <- paste0(intermittent_TCUs_A, df_speech_pairs[[p]][y,]$A_speech, " ")
          intermittent_TCUs_B <- paste0(intermittent_TCUs_B, df_speech_pairs[[p]][y,]$B_speech, " ")
      }
    }
    #then merge all TCUs into one string
    #by default they are empty; only speech added when there are first and last TCUs (that match onset+offset repair annotation)
    merged_string_A <- NA
    merged_string_B <- NA
    if (first_TCU_A!=""&last_TCU_A!="") {
      merged_string_A <- paste0(first_TCU_A, " ", intermittent_TCUs_A, last_TCU_A)}
    if (first_TCU_B!=""&last_TCU_B!="") {
      merged_string_B <- paste0(first_TCU_B, " ", intermittent_TCUs_B, last_TCU_B)}
    #add that to df_repair_speech_pairs
    df_repair_speech_pairs[[p]][x,]$A_speech <- merged_string_A
    df_repair_speech_pairs[[p]][x,]$B_speech <- merged_string_B
  }
  }
}
```

##unlist and check
```{r}
#unlist
df_repair_speech <- do.call("rbind", df_repair_speech_pairs)

#check if there are no annotations without speech left
nrow(subset(df_repair_speech, is.na(A_speech)&is.na(B_speech)))==0
```

##check df
```{r}
length(levels(df_repair_speech$T0)) == 1
length(levels(df_repair_speech$Tmin1)) == 1
length(levels(df_repair_speech$Tplus1)) == 1

length(levels(df_repair_speech$T0_type)) == 3
length(levels(df_repair_speech$T0_position)) == 4

length(levels(df_repair_speech$pairnr)) == 20

length(levels(df_trials$trial)) == 96
length(levels(df_trials$target)) == 16


table(df_repair_speech$T0, df_repair_speech$pairnr)
table(df_repair_speech$T0_type)

```



# REPAIR + SPEECH + TRIALS

Assign trial/target/director to each repair annotation

For this we will use a loop, which needs to be executed for every pair separately
Here we create 'large list' formats using the split function, which contains multiple dataframes, one for each pair
```{r}
#set up df
df_repair_speech_trials <- df_repair_speech
df_repair_speech_trials$round <- NA
df_repair_speech_trials$trial <- NA
df_repair_speech_trials$target <- NA
df_repair_speech_trials$director <- NA
df_repair_speech_trials$correct_answer <- NA
df_repair_speech_trials$given_answer <- NA
df_repair_speech_trials$accuracy <- NA

#create 'large lists'
df_repair_speech_trials_pairs <- split(df_repair_speech_trials, df_repair_speech_trials$pairnr)
df_trials_pairs <- split(df_trials, df_trials$pairnr)
```

*takes about 1 min*
```{r}
for (p in 1:length(list_pairnrs)) { #for every pair:
  for (x in 1:nrow(df_repair_speech_trials_pairs[[p]])){ #loop over repair annotation
    for (y in 1:nrow(df_trials_pairs[[p]])) { #loop over df with trial information
      #match trial
      if (df_repair_speech_trials_pairs[[p]][x,]$onset_msec > df_trials_pairs[[p]][y,]$onset_msec&&df_repair_speech_trials_pairs[[p]][x,]$onset_msec < df_trials_pairs[[p]][y,]$offset_msec) { 
      #then copy trial information and add it to repair annotations
          df_repair_speech_trials_pairs[[p]][x,]$round <- as.character(gsub("\\..*", "", df_trials_pairs[[p]][y,]$trial))
          df_repair_speech_trials_pairs[[p]][x,]$trial <- as.character(df_trials_pairs[[p]][y,]$trial)
          df_repair_speech_trials_pairs[[p]][x,]$target <- as.character(df_trials_pairs[[p]][y,]$target)
          df_repair_speech_trials_pairs[[p]][x,]$director <- as.character(df_trials_pairs[[p]][y,]$director)
          df_repair_speech_trials_pairs[[p]][x,]$correct_answer <- as.character(df_trials_pairs[[p]][y,]$correct_answer)
          df_repair_speech_trials_pairs[[p]][x,]$given_answer <- as.character(df_trials_pairs[[p]][y,]$given_answer)
          df_repair_speech_trials_pairs[[p]][x,]$accuracy <- as.character(df_trials_pairs[[p]][y,]$accuracy)}
      }
  }
}

```


##unlist and clean up; sort cols
```{r}
#unlist
df_repair_speech_trials <- do.call("rbind", df_repair_speech_trials_pairs)

#order
col_order <- c("pairnr", "onset", "onset_msec", "offset", "offset_msec", "round", "trial", "target", "director", "correct_answer", "given_answer", "accuracy", "A_speech", "B_speech", "Tmin1", "T0", "Tplus1", "T0_type", "T0_position")

df_repair_speech_trials <- df_repair_speech_trials[,col_order]


```


# LINKING REPAIR ANNOTATIONS
We will have to 'link' repair annotations, first of all to the correct repair annotations (make sure that each T0 corresponds to correct T-1 and T+1), and later also to the corresponding gestures. 
To this end we will use two columns:
- 'repair_id' : an id nr for every individual repair annotation
- 'repair_seq_id' : an id nr for every repair sequence (which includes T-1 & T0 & T+1 or T0 and T+1)

```{r}
df_repair_ids <- df_repair_speech_trials

df_repair_ids <- df_repair_ids %>%
  arrange(pairnr, onset_msec)

df_repair_ids$repair_id <- c(1:nrow(df_repair_ids)) #simply count from start till end
df_repair_ids$repair_seq_id <- NA
```

As for linking the annotations within a sequence: usually this is simply the closest one, but sometimes we get complex sequences:
T-1_a > T-1_b > T0_b > T+1_b > T0_a > T+1_a

We'll start with computing the 'default' ordering, and then manually correct the order for some deviant cases

```{r}
#give a nr to every T0
count_seq_ids <- 1

for (x in 1:nrow(df_repair_ids)){
  if (!is.na(df_repair_ids[x,]$T0)){
    df_repair_ids[x,]$repair_seq_id <- count_seq_ids
    count_seq_ids = count_seq_ids + 1}}

#then link T+1s for every T0 (the next row)
for (x in 1:nrow(df_repair_ids)){
  if (!is.na(df_repair_ids[x,]$Tplus1)){
  df_repair_ids[x,]$repair_seq_id <- df_repair_ids[x-1,]$repair_seq_id}}

#then link T-1s for every T0 which has position==single or first (the previous row)
#but only if that previous row contains a T-1 annotation
for (x in 1:nrow(df_repair_ids)){
  if (!is.na(df_repair_ids[x,]$T0_position)&(df_repair_ids[x,]$T0_position=="single"|df_repair_ids[x,]$T0_position=="first")){
    if (!is.na(df_repair_ids[x-1,]$Tmin1)){
      df_repair_ids[x-1,]$repair_seq_id <- df_repair_ids[x,]$repair_seq_id}}}

```

As based on prior coding notes, there is a shifting of orders:
- p04: trials 2.11 and 5.13
- p08: trials 1.1 and 6.3*
- p15: trial 1.3
- p16: trial 1.13

* p08: trial 6.3 is quite complex because there is non-minimal sequence (repair ids 168-170) which occurs within a non-minimal sequence (repair ids 165-167, followed by 171-176). This is not a problem for the repair_ids and repair_seq_ids; those are correct

Let's see if indeed for these 5 cases we do not yet have repair_seq_ids
```{r}
subset(df_repair_ids, is.na(repair_seq_id)) %>% select(pairnr, trial)
```
This matches, so let's add the repair_seq_ids for those NA cases

```{r}
for (x in 1:nrow(df_repair_ids)){
  if (is.na(df_repair_ids[x,]$repair_seq_id)){df_repair_ids[x,]$repair_seq_id <- df_repair_ids[x+4,]$repair_seq_id}}

#cases have been manually inspected too; these added seq ids are correct, apart for p08 trial 1.1 (repair_id 132)

df_repair_ids[df_repair_ids$repair_id==132,]$repair_seq_id <- 50

```

Now we need to check if we always have all repair_parts for each sequence
```{r}
check_order <- df_repair_ids %>% mutate(repair_part = coalesce(Tmin1, T0, Tplus1)) %>%
         select(repair_seq_id, repair_part, T0_position)

check_order <- check_order %>%
  group_by(repair_seq_id, .drop=F) %>%
  count(repair_part)

check_order <- merge(check_order, df_repair_ids[,c("repair_seq_id", "T0_position")], by="repair_seq_id", all.x=T)
check_order <- unique(check_order)
check_order <- subset(check_order, !is.na(T0_position))


nrow(subset(check_order, repair_part=="T-1"&(T0_position=="first"|T0_position=="single")&n==0)) #ok
nrow(subset(check_order, repair_part=="T0"&n==0)) #ok


nrow(subset(check_order, repair_part=="T+1"&n==0)) #2
#T+1 missing for #141 = p13, trial 1.9 --> T+1 starts after onset of next T0 (they overlap), fix as follows
df_repair_ids[df_repair_ids$repair_id==396,]$repair_seq_id <- 141
#T+1 missing for #359 = p24, trial 1.8 --> T+1 starts after onset of next T0 (they overlap), fix as follows
df_repair_ids[df_repair_ids$repair_id==1016,]$repair_seq_id <- 359



```




#final check
```{r}
unique(table(df_repair_ids$repair_seq_id)) #should always be 2 or 3: check

#if position == single or first, there should be a corresponding T-1

nrow(subset(df_repair_ids, T0_position=="single"|T0_position=="first"))==nrow(subset(df_repair_ids, Tmin1=="T-1"))

```


#export
```{r}
#export
write.csv2(df_repair_ids, "processing_output/2_repair_speech.csv", row.names=F)
```





