---
title: "1_import_ELAN_data"
author: "Marlou"
date: "24-09-2021"
output: html_document
---

#packages
```{r}
library(dplyr)
```

#load in data
We read in the raw output from ELAN. It is generated with "export multiple files as" > tab-delimited text > tick only two output options: 
- separate column for each tier
- include file name column

and the 21 following tiers:
repair tiers: repair_T0, repairT-1, repairT+1, T0_type, T0_position
speech tiers: A_po, B_po
gesture tiers: only the type and referent tiers
other: trial, target, director, correct_answer, given_answer, accuracy

In total there are 27 columns in the file because there are 4 onset/offset timestamp cols and 2 file name cols

```{r}
df_original <- read.csv2("R:/analysis/3_DivLabour/1_data/ELAN_output.csv", header=T, sep=",", encoding="UTF-8", na.strings=c("","NA"))

ncol(df_original)==27
```

#initial cleaning
```{r}
#rename cols, delete File.path col, fix pairnr col
df <-
  df_original %>%
  rename("onset" = Begin.Time...hh.mm.ss.ms,
         "onset_msec" = Begin.Time...msec,
         "offset" = End.Time...hh.mm.ss.ms,
         "offset_msec" = End.Time...msec,
         "T0" = repair_T0, 
         "Tplus1" = repair_T.1.1,
         "Tmin1" = repair_T.1,
         "pairnr" = File, 
         "A_speech" = A_po,
         "B_speech" = B_po)
df$pairnr <- gsub("pair", "", df$pairnr)
df$pairnr <- gsub(".eaf", "", df$pairnr)
df$File.Path <- NULL

#names(df)

#make sure columns are in the correct order
col_order <- c("pairnr", "onset", "onset_msec", "offset", "offset_msec", "trial", "target", "director", "correct_answer", "given_answer", "accuracy", "A_speech", "B_speech", "Tmin1", "T0", "Tplus1", "T0_type", "T0_position", "A_LH_gesture_type","A_RH_gesture_type","B_LH_gesture_type","B_RH_gesture_type",
"A_LH_gesture_referent","A_RH_gesture_referent","B_LH_gesture_referent","B_RH_gesture_referent")

df <- df[,col_order]

#take out accidental spaces from repair annotations
df$T0 <- gsub(" ", "", df$T0)
df$Tmin1 <- gsub(" ", "", df$Tmin1)
df$Tplus1 <- gsub(" ", "", df$Tplus1)
df$T0_type <- gsub(" ", "", df$T0_type)
df$T0_position <- gsub(" ", "", df$T0_position)

#most cols are factors, change all character cols to factors
df <- df %>% 
  mutate(across(where(is.character), as.factor))

#but set speech and onset/offset cols to character, and target to factor
df <- df %>%
  mutate(A_speech = as.character(A_speech),
          B_speech = as.character(B_speech),
          onset = as.character(onset),
          offset = as.character(offset),
          target = as.factor(target))

#sort by pairnr, then onset
df <- df %>%
 arrange(pairnr, onset_msec)

```

#quick check
```{r}
#check
length(levels(df$T0)) == 1
length(levels(df$Tmin1)) == 1
length(levels(df$Tplus1)) == 1

length(levels(df$T0_type)) == 3
length(levels(df$T0_position)) == 4

length(levels(df$pairnr)) == 20

length(levels(df$trial)) == 96
length(levels(df$target)) == 16


```
#create separate dataframes


##SPEECH
I.e., a df with ONLY the speech annotations (no gestures,  no trial info etc.)
We need this to later on locate the speech for all repair annotations
```{r}
df_speech <- df %>% select("pairnr", "onset", "onset_msec", "offset", "offset_msec", "A_speech", "B_speech")

#delete rows without speech
df_speech <- subset(df_speech, !is.na(A_speech)|!is.na(B_speech))

#convert NAs to empty strings (necessary for loop later on; strings need to be concatenated)
df_speech$A_speech[is.na(df_speech$A_speech)] <- ""
df_speech$B_speech[is.na(df_speech$B_speech)] <- ""

#sort it by pairnr, then onset
df_speech <- df_speech %>%
 arrange(pairnr, onset_msec)

#check: there should be no rows with speech from both pp
nrow(subset(df_speech, A_speech!=""&B_speech!=""))==0

```


##TRIAL INFO
```{r}
#seperate trial df
df_trials <-
  df %>%
  subset(!is.na(trial))

#keep cols on time, trial, target, director
df_trials <- df_trials %>% select("pairnr", "onset", "onset_msec", "offset", "offset_msec", "trial", "target", "director", "correct_answer", "given_answer", "accuracy")

#check
nrow(df_trials) == 20*96+1 #1 extra because twice trial 1.1 for pair 8

#sort by pairnr, then onset
df_trials <- df_trials %>%
 arrange(pairnr, onset_msec)

#add round col
df_trials$round <- gsub("\\..*", "", df_trials$trial) #add round column

#compute number of turns per trial
#first add trial info to all speech turns
df_speech$trial <- NA

for (x in 1:nrow(df_speech)) {
for (y in 1:nrow(df_trials)) {
      if ((df_speech[x,]$pairnr==df_trials[y,]$pairnr)&
         (df_speech[x,]$onset_msec > df_trials[y,]$onset_msec&&df_speech[x,]$onset_msec < df_trials[y,]$offset_msec))
          {
      row <- y
      df_speech[x,]$trial <- as.character(df_trials[row,]$trial)
      }}}

#count speech turns per trial, per dyad
turn_count <- df_speech %>% group_by(pairnr, trial) %>% count(name="turn_count")
#merge into df_trials
df_trials <- merge(df_trials, turn_count, by=c("pairnr","trial"))
```

##REPAIR
I.e., a df with only T-1 / T0 / T+1 annotations (with speech tiers still present)
```{r}
df_repair <- df %>%
             select(pairnr, onset, onset_msec, offset, offset_msec, A_speech, B_speech, Tmin1, T0, Tplus1, 
                    T0_type, T0_position) %>%
              subset(!is.na(Tmin1)|!is.na(T0)|!is.na(Tplus1))

#check: there should be the same amount of T0s as Tplus1
nrow(subset(df_repair, !is.na(T0))) == nrow(subset(df_repair, !is.na(Tplus1)))

#check: there should be a Tmin1 for every T0 with position=="single" or "first"
nrow(subset(df_repair, !is.na(Tmin1))) == (nrow(subset(df_repair, T0_position=="single"|T0_position=="first")))

#check: there should be same amount of T0 with position== 'first' and 'last'
nrow(subset(df_repair, T0_position=="first")) == nrow(subset(df_repair, T0_position=="last"))

#sort it by pairnr, then onset
df_repair <- df_repair %>%
 arrange(pairnr, onset_msec)


```


##GESTURE
I.e., a df with only gesture annotations (when 'type' and/or 'referent' are annotated; empty annotations and 'non-gesture' annotations were not exported from ELAN).
```{r}
#check if there are no manual mistakes where by accident 2-handed gestures were made on tiers belonging to two different participants
nrow(df %>% subset(!is.na(A_LH_gesture_type)&!is.na(B_LH_gesture_type)))==0   #A_LH+B_LH
nrow(df %>% subset(!is.na(A_LH_gesture_type)&!is.na(B_LH_gesture_type)))==0   #A_LH+B_RH
nrow(df %>% subset(!is.na(A_RH_gesture_type)&!is.na(B_LH_gesture_type)))==0   #A_RH+B_LH
nrow(df %>% subset(!is.na(A_RH_gesture_type)&!is.na(B_RH_gesture_type)))==0   #A_RH+B_RH

df_gestA <- df %>% subset(!is.na(A_LH_gesture_type)|!is.na(A_RH_gesture_type))
df_gestA <- df_gestA %>% select("pairnr", "onset", "onset_msec", "offset", "offset_msec", "A_LH_gesture_type", "A_RH_gesture_type", "A_LH_gesture_referent", "A_RH_gesture_referent")

df_gestB <- df %>% subset(!is.na(B_LH_gesture_type)|!is.na(B_RH_gesture_type))
df_gestB <- df_gestB %>% select("pairnr", "onset", "onset_msec", "offset", "offset_msec", "B_LH_gesture_type", "B_RH_gesture_type", "B_LH_gesture_referent", "B_RH_gesture_referent")

#sort by pairnr then onset
df_gestA <- df_gestA %>%
 arrange(pairnr, onset_msec)
df_gestB <- df_gestB %>%
 arrange(pairnr, onset_msec)

```


#export
```{r}
write.csv2(df_trials, "processing_output/1_trials.csv", row.names=F)
write.csv2(df_speech, "processing_output/1_speech.csv", row.names=F)
write.csv2(df_repair, "processing_output/1_repair.csv", row.names=F)
write.csv2(df_gestA, "processing_output/1_gestures_A.csv", row.names=F)
write.csv2(df_gestB, "processing_output/1_gestures_B.csv", row.names=F)

```





