---
title: "4_speech_cleaning"
author: "Marlou"
date: "2-9-2021"
output: html_document
---

```{r}
library(dplyr)
library(stringr)
```


#DATA

```{r}
df_repair <- read.csv2("processing_output/2_repair_speech.csv", header=T, colClasses = c(pairnr = "character"))
```

Initial processing
```{r}
#convert A_speech and B_speech cols into 1 col for speech, and 1 col for speaker
df_repair$speaker <- NA
df_repair$speech_original <- NA
for (x in 1:nrow(df_repair)){
  if (!is.na(df_repair[x,]$A_speech)) {
    df_repair[x,]$speaker <- "A"
    df_repair[x,]$speech_original <- df_repair[x,]$A_speech}
  else if (!is.na(df_repair[x,]$B_speech)) {
    df_repair[x,]$speaker <- "B"
    df_repair[x,]$speech_original <- df_repair[x,]$B_speech
  }
}

df_repair$A_speech <- NULL
df_repair$B_speech <- NULL

#most cols are factors, change all character cols to factors
df_repair <- df_repair %>% 
  mutate(across(where(is.character), as.factor))

#but set speech and onset/offset cols to character, and target to factor
df_repair <- df_repair %>%
  mutate(speech_original = as.character(speech_original),
          onset = as.character(onset),
          offset = as.character(offset),
          target = as.factor(target))




```



#SPEECH CLEANING
Note that with the prior scripts (1 t0 3) it is just the dataframe that has been altered (repair/gesture/trial annotations are linked); the speech has not yet been cleaned in any way!

### Speech segmentation
Every speech annotation corresponds to *one Turn Constructional Unit (TCU)*, defined as follows:
"Each TCU is a coherent and self-contained utterance, recognizable in context as 'possibly complete'. Each TCU's completion establishes a transition-relevance place (TRP) where a change of speakership becomes a salient possibility that may or may not be realized at any particular TRP." (Clayman 2013, p. 151).

### Speech transcription
Speech has been transcribed orthographically using the standard spelling conventions of Dutch.

**What is transcribed?**

* Repetitions: e.g., 'deze heeft twee twee stokjes')
* Self-corrections: e.g., aan de zijkant twee ku- kubussen' / 'deze heeft een cirkel heeft een bolletje bovenop'
* Filled pauses: um and uh
* Reactive tokens: e.g., hmm, oh, aha
* Question marks: e.g. 'en nog een vierkant?' (only used where clearly a question with rising intonation, not for boundary cases such question-syntax but no rising intonation)
* Non-verbal vocal sounds: e.g., #laughs#, #click#, #sigh#
* Task answers: e.g., 'dan is het 8' (marked with asterisks)
* Uncertain/missing speech: e.g., 'deze heeft een soort van (bank) aan de zijkant' / 'deze heeft een soort van (?) aan de zijkant'

**What is not transcribed?**

* unfilled pauses
* prosodic / phonetic information such as speech rate, volume, pitch, lengthening, etc.
* task-irrelevant sounds (e.g. scratching arm, breathing)

Cleaning speech
```{r}
df_repair$speech_clean <- df_repair$speech_original
```


# UNCERTAIN/MISSING SPEECH
When the transcriber was unsure, this was indicated as: e.g., 'deze heeft een soort van (bank) aan de zijkant' / 'deze heeft een soort van (?) aan de zijkant'.
Here all characters between parentheses are deleted.

```{r}
#check if regex works well

# original = df_repair$speech
# deleted = gsub("*\\([^\\)]+\\)"," ",df_repair$speech)
# 
# for (x in 1:length(original)){
#   if (original[x] != deleted[x]) {
#     print(original[x])
#     print('vs.')
#     print(deleted[x])
#     print(' ')}
# }

df_repair$speech_clean <- gsub("*\\([^\\)]+\\)"," ",df_repair$speech_clean)

#make sure there are no empty TCU annotations
nrow(df_repair[df_repair$speech_clean=="",])==0


```

# NON-LEXICAL VOCALIZATIONS
e.g. laughing, coughing, breathing, sniffing, clicking with tongue/other mouth noises, clapping, etc.
```{r}
#check non-linguistic vocalizations
# del_nonwords <- str_extract_all(df_repair$speech_clean, "#\\s*(.*?)\\s*#")
# del_nonwords<- do.call("rbind", del_nonwords)
# list_nonwords <- unique(del_nonwords[,1])
# list_nonwords

#all sounds can be deleted, but there are two special cases
# #f#: non-lexical sound symbolic vocalization (occurs once, can be taken out)
# #m#: occurs rather often, simply 'hm' or 'mm'; should be kept in (is backchannel), convert to 'hm' (might have been m or mhm - such variations are ignored here)

df_repair$speech_clean <- gsub("#m#", 'hm', df_repair$speech_clean)

#check if regex works well

# original = df_repair$speech_clean
# deleted = gsub("\\s*#[^#]+#","",df_repair$speech_clean)
# 
# for (x in 1:length(original)){
#   if (original[x] != deleted[x]) {
#     print(original[x])
#     print('vs.')
#     print(deleted[x])
#     print(' ')}
# }


df_repair$speech_clean = gsub("\\s*#[^#]+#","",df_repair$speech_clean)

#make sure there are no empty TCU annotations
nrow(df_repair[df_repair$speech_clean=="",])==0



```

# TASK ANSWERS? 
```{r}
#check if there are task answers (e.g. 'dan is het nummer *5*') in any of the repair annotations

original = df_repair$speech_clean
deleted = gsub("\\*","",df_repair$speech_clean)

count = 0
for (x in 1:length(original)){
  if (original[x] != deleted[x]) {
    count = count +1}
}

count == 0

```

# INCOMPLETE WORDS: keep partial word, just remove hyphen
```{r}
#check if regex works well
# original = df_repair$speech
# deleted = gsub("- "," ",df_repair$speech)
# 
# for (x in 1:length(original)){
#   if (original[x] != deleted[x]) {
#     print(original[x])
#     print('vs.')
#     print(deleted[x])
#     print(' ')}
# }

#remove hyphen, but keep incomplete word
df_repair$speech_clean = gsub("- "," ",df_repair$speech_clean)
df_repair$speech_clean = gsub("-$"," ",df_repair$speech_clean) #to delete hyphens at end of string

```



# PUNCTUATION
```{r}
#remove some punctuation (dots, commas, colon, semicolon, question marks)
df_repair$speech_clean = gsub("\\.", "",df_repair$speech_clean)
df_repair$speech_clean = gsub(",", "",df_repair$speech_clean)
df_repair$speech_clean = gsub(":" ,"", df_repair$speech_clean)
df_repair$speech_clean = gsub(";" ,"", df_repair$speech_clean)
df_repair$speech_clean = gsub("\\?" ,"", df_repair$speech_clean)


#remove double spaces
df_repair$speech_clean = gsub("  " ," ", df_repair$speech_clean)

#remove leading+closing spaces
df_repair$speech_clean = trimws(df_repair$speech_clean)

#replace hyphen by space
df_repair$speech_clean = gsub("-" ," ", df_repair$speech_clean)

#make sure there are no empty TCU annotations
nrow(df_repair[df_repair$speech_clean=="",])==0

```

# CHECK FOR REMAINING WEIRD THINGS
```{r}

#rows <- grep("[^a-zA-Zéè' ]", df_repair$speech_clean)
#df_repair[rows,]


df_repair$speech_clean = gsub("=" ,"is", df_repair$speech_clean)
df_repair$speech_clean = gsub("45" ,"vijfenveertig", df_repair$speech_clean)


#check for apostrophes
#rows <- grep("'", df_repair$speech_clean)
#df_repair[rows,]

#z'n, zo'n
df_repair$speech_clean = gsub("z'n" ,"zijn", df_repair$speech_clean)
df_repair$speech_clean = gsub("zo'n" ,"zon", df_repair$speech_clean)

```


# export
```{r}
write.csv2(df_repair, "processing_output/4_repair_speech_clean.csv", row.names=F)
```

