
# 1. PACKAGES ET DOSSIERS ------------------------------------------------------

packages <- c(
  "readr", "dplyr", "tidyr", "stringr", "jsonlite",
  "sandwich", "lmtest", "openxlsx2"
)
missing <- packages[!packages %in% rownames(installed.packages())]
if (length(missing)) install.packages(missing, repos = "https://cloud.r-project.org")
suppressPackageStartupMessages(invisible(lapply(packages, function(package) {
  suppressWarnings(library(package, character.only = TRUE))
})))

# Le script fonctionne depuis la racine du projet ou depuis le dossier scripts.
file_arg <- sub("^--file=", "", grep("^--file=", commandArgs(FALSE), value = TRUE))
roots <- unique(c(
  normalizePath(getwd()),
  if (length(file_arg)) dirname(dirname(normalizePath(file_arg))) else character()
))
PROJECT_ROOT <- roots[file.exists(file.path(roots, "data", "input", "events_master.csv"))][1]
if (is.na(PROJECT_ROOT)) stop("Lancez le script depuis thesis_ecb_fed_comparison.")

DIR <- list(
  input = file.path(PROJECT_ROOT, "data", "input"),
  text = file.path(PROJECT_ROOT, "data", "raw", "statements"),
  market = file.path(PROJECT_ROOT, "data", "raw", "markets"),
  final = file.path(PROJECT_ROOT, "data", "final"),
  output = file.path(PROJECT_ROOT, "outputs")
)
invisible(lapply(DIR[c("final", "output")], dir.create, recursive = TRUE, showWarnings = FALSE))

FILE <- c(
  events = file.path(DIR$input, "events_master.csv"),
  stoxx50 = file.path(DIR$market, "STOXX50_daily.json"),
  sp500 = file.path(DIR$market, "SP500_daily.json"),
  ecb2y = file.path(DIR$market, "ECB_YC_2Y.csv"),
  ecb10y = file.path(DIR$market, "ECB_YC_10Y.csv"),
  us2y = file.path(DIR$market, "US_DGS2_daily.csv"),
  us10y = file.path(DIR$market, "US_DGS10_daily.csv"),
  vstoxx = file.path(DIR$market, "VSTOXX_daily.csv"),
  vix = file.path(DIR$market, "VIX_daily.json")
)
missing_files <- FILE[!file.exists(FILE)]
if (length(missing_files)) stop("Fichiers manquants : ", paste(missing_files, collapse = ", "))
if (!length(list.files(DIR$text, "\\.txt$"))) stop("Aucun texte dans data/raw/statements.")


# 2. FONCTIONS COURTES ---------------------------------------------------------

z_score <- function(x) {
  s <- sd(x, na.rm = TRUE)
  if (!is.finite(s) || s == 0) return(rep(NA_real_, length(x)))
  (x - mean(x, na.rm = TRUE)) / s
}

# Un shock textuel est le residu d'un AR(1), estime separement par banque.
ar1_innovation <- function(x) {
  d <- tibble(value = as.numeric(x), lag1 = lag(as.numeric(x)))
  ok <- complete.cases(d)
  out <- rep(NA_real_, length(x))
  if (sum(ok) >= 10) out[ok] <- residuals(lm(value ~ lag1, d[ok, ]))
  out
}

tokenize <- function(text) str_extract_all(tolower(text), "[a-z]+(?:'[a-z]+)?")[[1]]

count_syllables <- function(word) {
  word <- tolower(word)
  if (nchar(word) <= 3) return(1L)
  max(1L, length(str_extract_all(sub("e$", "", word), "[aeiouy]+")[[1]]))
}

make_bigrams <- function(tokens) {
  if (length(tokens) < 2) return(character())
  unique(paste(head(tokens, -1), tail(tokens, -1), sep = "__"))
}

jaccard <- function(current, previous) {
  if (is.null(previous)) return(NA_real_)
  all_terms <- union(current, previous)
  if (!length(all_terms)) return(NA_real_)
  length(intersect(current, previous)) / length(all_terms)
}

signed_score <- function(positive, negative, n_words) {
  if (!n_words) return(NA_real_)
  if (positive + negative == 0) return(0)
  (positive - negative) / (positive + negative)
}

# Les trois fichiers Excel ont le meme format lisible.
write_workbook <- function(sheets, path) {
  wb <- wb_workbook()
  for (sheet in names(sheets)) {
    x <- as.data.frame(sheets[[sheet]])
    wb <- wb_add_worksheet(wb, sheet)
    wb <- wb_add_data(wb, sheet, x, with_filter = TRUE, na = "")
    wb <- wb_freeze_pane(wb, sheet, first_row = TRUE)
    wb <- wb_set_row_heights(wb, sheet, 1, 30)
    widths <- vapply(seq_along(x), function(j) {
      min(35, max(10, max(nchar(c(names(x)[j], as.character(x[[j]]))), na.rm = TRUE) + 2))
    }, numeric(1))
    wb <- wb_set_col_widths(wb, sheet, seq_along(x), widths)
    header <- paste0("A1:", int2col(ncol(x)), "1")
    wb <- wb_add_font(wb, sheet, header, color = wb_color(hex = "FFFFFFFF"), bold = TRUE)
    wb <- wb_add_fill(wb, sheet, header, color = wb_color(hex = "FF1F4E78"))
    wb <- wb_add_border(
      wb, sheet, header, bottom_border = "thin",
      bottom_color = wb_color(hex = "FF17365D"),
      left_border = "", right_border = "", top_border = ""
    )
  }
  wb_save(wb, path, overwrite = TRUE)
}


# 3. CALENDRIER DES REUNIONS ---------------------------------------------------

events_raw <- read_csv(FILE[["events"]], show_col_types = FALSE)
needed <- c(
  "event_id", "central_bank", "meeting_date", "policy_rate",
  "delta_policy_rate", "emergency_event", "scheduled_meeting",
  "included_primary", "exclusion_reason"
)
missing_columns <- setdiff(needed, names(events_raw))
if (length(missing_columns)) stop("Colonnes manquantes : ", paste(missing_columns, collapse = ", "))

events <- events_raw %>%
  transmute(
    event_id,
    central_bank = if_else(central_bank %in% c("FED", "Fed"), "Fed", "ECB"),
    meeting_date = as.Date(meeting_date),
    policy_rate = as.numeric(policy_rate),
    emergency_event = if_else(is.na(emergency_event), 0L, as.integer(emergency_event)),
    scheduled_meeting = as.integer(scheduled_meeting),
    included_primary = as.integer(included_primary),
    exclusion_reason = if_else(is.na(exclusion_reason), "", as.character(exclusion_reason)),
    rate_change_bp = round(100 * as.numeric(delta_policy_rate), 8)
  ) %>%
  distinct(central_bank, meeting_date, .keep_all = TRUE) %>%
  arrange(central_bank, meeting_date)

if (anyDuplicated(paste(events$central_bank, events$meeting_date))) stop("Date dupliquee.")
if (any(events$included_primary == 1L & events$emergency_event == 1L)) {
  stop("Une reunion d'urgence se trouve dans l'echantillon principal.")
}


# 4. VARIABLES TEXTUELLES ------------------------------------------------------

POSITIVE_WORDS <- c(
  "improve", "improved", "improvement", "growth", "recovery", "support",
  "favourable", "favorable", "strong", "strengthen", "resilient", "stable",
  "confidence", "progress", "robust", "positive", "accommodative"
)
NEGATIVE_WORDS <- c(
  "risk", "risks", "weak", "weaker", "uncertainty", "volatile", "downside",
  "slow", "sluggish", "negative", "decline", "dampened", "subdued"
)

DICTIONARY_VERSION <- "policy_dictionary_v2_2026-08-14"
HAWKISH_TERMS <- c(
  "rate cuts are not warranted", "not sufficiently restrictive",
  "no longer accommodative", "upside risks to inflation", "upside risk to inflation",
  "inflation remains too high", "inflation is too high", "persistent inflation pressures",
  "persistent price pressures", "maintain a restrictive stance",
  "restrictive for some time", "keep rates restrictive", "higher for longer",
  "additional tightening", "further tightening", "balance sheet reduction",
  "balance sheet runoff", "quantitative tightening", "reduce reinvestments",
  "taper asset purchases", "tapering asset purchases", "ending net purchases",
  "end net purchases", "inflation above target", "above target",
  "inflationary pressure", "inflationary pressures", "price pressure", "price pressures",
  "wage pressure", "wage pressures", "strong demand", "tight labor", "tight labour",
  "labor market tightness", "labour market tightness", "rate increase", "policy firming",
  "restrictive stance", "inflationary", "persistent", "persistence", "elevated",
  "upside", "tightening", "tighten", "tighter", "restrictive", "restriction",
  "normalisation", "normalization", "increase", "increases", "increased", "higher",
  "hike", "hikes", "firming", "overheating", "tightness", "vigilance", "raise",
  "raised", "rising"
)
DOVISH_TERMS <- c(
  "further tightening is not warranted", "no further tightening",
  "policy remains accommodative", "supportive monetary policy",
  "downside risks to growth", "downside risk to growth",
  "targeted longer term refinancing operations", "pandemic emergency purchase programme",
  "pandemic emergency purchase program", "emergency purchase programme",
  "emergency purchase program", "increase asset purchases", "expanded asset purchases",
  "expand asset purchases", "slower pace of tightening", "pausing rate increases",
  "pause rate increases", "rate cuts are warranted", "inflation below target",
  "maintain accommodation", "additional accommodation", "ample accommodation",
  "lower for longer", "full reinvestment", "full reinvestments", "asset purchase programme",
  "asset purchase program", "asset purchases", "quantitative easing", "lower rates",
  "low rates", "below target", "downside risk", "downside risks",
  "policy accommodation", "accommodative stance", "accommodative", "accommodation",
  "easing", "ease", "eased", "lower", "cut", "cuts", "decrease", "decreased",
  "decline", "weak", "weaker", "weakness", "slack", "downside", "subdued",
  "support", "stimulus", "stimulative", "purchase", "purchases", "recession",
  "slowdown", "fragile", "uncertainty"
)
AMBIGUOUS_TERMS <- c(
  "inflationary", "persistent", "persistence", "elevated", "upside", "increase",
  "increases", "increased", "higher", "firming", "tightness", "restriction",
  "raise", "raised", "rising", "lower", "cut", "cuts", "decrease", "decreased",
  "decline", "weak", "weaker", "weakness", "slack", "downside", "subdued",
  "support", "purchase", "purchases", "fragile", "uncertainty"
)
CONTEXT_WORDS <- c(
  "inflation", "inflationary", "price", "prices", "wage", "wages", "rate", "rates",
  "policy", "monetary", "stance", "target", "demand", "economy", "economic",
  "growth", "output", "employment", "unemployment", "labor", "labour", "market",
  "markets", "financial", "financing", "credit", "conditions", "pressure",
  "pressures", "balance", "sheet", "asset", "assets", "reinvestment",
  "reinvestments", "pepp", "app", "tltr", "tightening", "easing"
)
NEGATIONS <- c(
  "not", "no", "neither", "without", "unlikely", "cannot", "can't", "isn't",
  "aren't", "wasn't", "weren't", "doesn't", "don't", "didn't"
)
NEGATION_EXCEPTIONS <- c(
  "rate cuts are not warranted", "not sufficiently restrictive",
  "no longer accommodative", "further tightening is not warranted",
  "no further tightening"
)

# Les expressions longues sont testees avant les mots seuls pour eviter les doublons.
policy_dictionary <- bind_rows(
  tibble(category = "hawkish", term = HAWKISH_TERMS),
  tibble(category = "dovish", term = DOVISH_TERMS)
) %>%
  distinct(category, term) %>%
  mutate(
    tokens = lapply(term, tokenize), n_tokens = lengths(tokens),
    requires_context = n_tokens == 1L & term %in% AMBIGUOUS_TERMS,
    negation_exception = term %in% NEGATION_EXCEPTIONS,
    dictionary_version = DICTIONARY_VERSION
  ) %>%
  arrange(desc(n_tokens), term)
if (anyDuplicated(policy_dictionary$term)) stop("Terme classe hawkish et dovish.")

dictionary_definition <- policy_dictionary %>%
  transmute(dictionary_version, category, term, n_tokens, requires_context, negation_exception)

has_context <- function(tokens, first, last, window = 3L) {
  any(tokens[max(1L, first - window):min(length(tokens), last + window)] %in% CONTEXT_WORDS)
}

has_negation <- function(tokens, first, window = 3L) {
  if (first <= 1L) return(FALSE)
  positions <- seq.int(max(1L, first - window), first - 1L)
  for (position in positions) {
    if (!tokens[position] %in% NEGATIONS) next
    if (tokens[position] == "not" && position < length(tokens) && tokens[position + 1L] == "only") next
    return(TRUE)
  }
  FALSE
}

# Comptage longest-match-first avec contexte et negation.
count_policy_terms <- function(tokens) {
  found <- list(hawkish = character(), dovish = character(), negated = character(), rejected = character())
  position <- 1L
  while (position <= length(tokens)) {
    matched <- FALSE
    for (row in seq_len(nrow(policy_dictionary))) {
      expression <- policy_dictionary$tokens[[row]]
      last <- position + length(expression) - 1L
      if (last > length(tokens) || !identical(tokens[position:last], expression)) next
      term <- policy_dictionary$term[row]
      if (policy_dictionary$requires_context[row] && !has_context(tokens, position, last)) {
        found$rejected <- c(found$rejected, term)
      } else if (!policy_dictionary$negation_exception[row] && has_negation(tokens, position)) {
        found$negated <- c(found$negated, term)
      } else {
        category <- policy_dictionary$category[row]
        found[[category]] <- c(found[[category]], term)
      }
      position <- last + 1L
      matched <- TRUE
      break
    }
    if (!matched) position <- position + 1L
  }
  found
}

# Calcule toutes les mesures d'une communication preparee.
analyse_text <- function(event, previous_bigrams) {
  file_name <- paste0(gsub(" ", "_", event$event_id), ".txt")
  path <- file.path(DIR$text, file_name)
  if (!file.exists(path)) stop("Texte manquant : ", path)
  text <- paste(readLines(path, warn = FALSE, encoding = "UTF-8"), collapse = "\n")
  tokens <- tokenize(text)
  n_positive <- sum(tokens %in% POSITIVE_WORDS)
  n_negative <- sum(tokens %in% NEGATIVE_WORDS)
  matches <- count_policy_terms(tokens)
  n_hawkish <- length(matches$hawkish)
  n_dovish <- length(matches$dovish)

  sentences <- str_split(text, "[.!?]+")[[1]]
  n_sentences <- max(1L, sum(lengths(strsplit(trimws(sentences), "\\s+")) >= 3))
  average_sentence_length <- length(tokens) / n_sentences
  fk_grade <- 0.39 * average_sentence_length +
    11.8 * (sum(vapply(tokens, count_syllables, integer(1))) / length(tokens)) - 15.59
  bigrams <- make_bigrams(tokens)

  list(
    data = tibble(
      central_bank = event$central_bank, meeting_date = event$meeting_date,
      n_words = length(tokens), n_positive, n_negative,
      tone = signed_score(n_positive, n_negative, length(tokens)),
      n_hawkish, n_dovish,
      hawkish_score = signed_score(n_hawkish, n_dovish, length(tokens)),
      n_policy_negated = length(matches$negated),
      n_policy_context_rejected = length(matches$rejected),
      n_sentences, average_sentence_length, fk_grade, clarity = -fk_grade,
      similarity = jaccard(bigrams, previous_bigrams)
    ),
    bigrams = bigrams
  )
}

text_rows <- list()
row <- 1L
for (bank in c("ECB", "Fed")) {
  previous_bigrams <- NULL
  bank_events <- events %>% filter(central_bank == bank) %>% arrange(meeting_date)
  for (i in seq_len(nrow(bank_events))) {
    result <- analyse_text(bank_events[i, ], previous_bigrams)
    text_rows[[row]] <- result$data
    previous_bigrams <- result$bigrams
    row <- row + 1L
  }
}

text_metrics <- bind_rows(text_rows) %>%
  group_by(central_bank) %>%
  arrange(meeting_date, .by_group = TRUE) %>%
  mutate(
    hawkish_shock_ar1 = ar1_innovation(hawkish_score),
    tone_shock_ar1 = ar1_innovation(tone),
    clarity_shock_ar1 = ar1_innovation(clarity),
    similarity_shock_ar1 = ar1_innovation(similarity),
    hawkish_shock_z_cb = z_score(hawkish_shock_ar1),
    tone_shock_z_cb = z_score(tone_shock_ar1),
    clarity_shock_z_cb = z_score(clarity_shock_ar1),
    similarity_shock_z_cb = z_score(similarity_shock_ar1),
    word_count_z_cb = z_score(n_words)
  ) %>%
  ungroup()


# 5. REACTIONS FINANCIERES -----------------------------------------------------

# Lecture harmonisee des quatre formats de series locales.
read_series <- function(path, type, column = NULL) {
  series <- switch(
    type,
    yahoo = {
      result <- fromJSON(path, simplifyVector = FALSE)$chart$result[[1]]
      values <- vapply(
        result$indicators$adjclose[[1]]$adjclose,
        function(value) if (is.null(value)) NA_real_ else as.numeric(value), numeric(1)
      )
      tibble(
        date = as.Date(as.POSIXct(unlist(result$timestamp), origin = "1970-01-01", tz = "UTC")),
        value = values
      )
    },
    ecb = read_csv(path, show_col_types = FALSE) %>%
      transmute(date = as.Date(TIME_PERIOD), value = as.numeric(OBS_VALUE)),
    fred = read_csv(path, na = c("", "."), show_col_types = FALSE) %>%
      transmute(date = as.Date(observation_date), value = as.numeric(.data[[column]])),
    vstoxx = read_delim(path, delim = ";", show_col_types = FALSE) %>%
      transmute(date = as.Date(Date, "%d.%m.%Y"), value = as.numeric(Indexvalue))
  )
  series %>% filter(!is.na(date), is.finite(value)) %>% arrange(date)
}

meeting_reaction <- function(series, date, type) {
  row <- which(series$date == date)[1]
  if (is.na(row) || row < 2) {
    return(c(trade_date = NA_real_, reaction = NA_real_, pre_level = NA_real_))
  }
  current <- series$value[row]
  previous <- series$value[row - 1L]
  reaction <- if (type == "log") 100 * log(current / previous) else 100 * (current - previous)
  c(trade_date = as.numeric(series$date[row]), reaction = reaction, pre_level = previous)
}

market_series <- list(
  ECB = list(
    equity = read_series(FILE[["stoxx50"]], "yahoo"),
    yield2y = read_series(FILE[["ecb2y"]], "ecb"),
    yield10y = read_series(FILE[["ecb10y"]], "ecb"),
    volatility = read_series(FILE[["vstoxx"]], "vstoxx")
  ),
  Fed = list(
    equity = read_series(FILE[["sp500"]], "yahoo"),
    yield2y = read_series(FILE[["us2y"]], "fred", "DGS2"),
    yield10y = read_series(FILE[["us10y"]], "fred", "DGS10"),
    volatility = read_series(FILE[["vix"]], "yahoo")
  )
)

market_rows <- vector("list", nrow(events))
for (i in seq_len(nrow(events))) {
  event <- events[i, ]
  series <- market_series[[event$central_bank]]
  equity <- meeting_reaction(series$equity, event$meeting_date, "log")
  yield2y <- meeting_reaction(series$yield2y, event$meeting_date, "yield")
  yield10y <- meeting_reaction(series$yield10y, event$meeting_date, "yield")
  volatility <- meeting_reaction(series$volatility, event$meeting_date, "log")
  market_rows[[i]] <- tibble(
    central_bank = event$central_bank, meeting_date = event$meeting_date,
    market_trade_date = as.Date(equity["trade_date"], origin = "1970-01-01"),
    equity_y = equity["reaction"], yield2y_y = yield2y["reaction"],
    yield10y_y = yield10y["reaction"], volatility_y = volatility["reaction"],
    volatility_pre_level = volatility["pre_level"],
    yield2y_trade_date = as.Date(yield2y["trade_date"], origin = "1970-01-01"),
    yield10y_trade_date = as.Date(yield10y["trade_date"], origin = "1970-01-01"),
    volatility_trade_date = as.Date(volatility["trade_date"], origin = "1970-01-01")
  )
}
market_metrics <- bind_rows(market_rows)

date_error <- market_metrics %>% filter(
  (!is.na(market_trade_date) & market_trade_date != meeting_date) |
    (!is.na(yield2y_trade_date) & yield2y_trade_date != meeting_date) |
    (!is.na(yield10y_trade_date) & yield10y_trade_date != meeting_date) |
    (!is.na(volatility_trade_date) & volatility_trade_date != meeting_date)
)
if (nrow(date_error)) stop("Une date de marche ne correspond pas au meeting.")


# 6. BASE FINALE ---------------------------------------------------------------

model_variables <- c(
  "hawkish_shock_z_cb", "tone_shock_z_cb", "clarity_shock_z_cb",
  "similarity_shock_z_cb", "rate_change_bp", "pre_volatility_z_cb",
  "word_count_z_cb", "equity_y", "yield2y_y", "yield10y_y", "volatility_y"
)

analysis_full <- events %>%
  left_join(text_metrics, by = c("central_bank", "meeting_date")) %>%
  left_join(market_metrics, by = c("central_bank", "meeting_date")) %>%
  group_by(central_bank) %>%
  mutate(
    pre_volatility_z_cb = (
      volatility_pre_level - mean(volatility_pre_level[included_primary == 1L], na.rm = TRUE)
    ) / sd(volatility_pre_level[included_primary == 1L], na.rm = TRUE)
  ) %>%
  ungroup() %>%
  mutate(
    primary_sample = included_primary == 1L,
    main_estimation_sample = primary_sample & complete.cases(across(all_of(model_variables)))
  ) %>%
  arrange(meeting_date, central_bank)

final_data <- analysis_full %>% select(
  event_id, central_bank, meeting_date, market_trade_date, rate_change_bp,
  emergency_event, scheduled_meeting, primary_sample, main_estimation_sample,
  exclusion_reason, n_words, n_positive, n_negative, tone, n_hawkish, n_dovish,
  hawkish_score, n_sentences, average_sentence_length, clarity, fk_grade, similarity,
  hawkish_shock_ar1, tone_shock_ar1, clarity_shock_ar1, similarity_shock_ar1,
  hawkish_shock_z_cb, tone_shock_z_cb, clarity_shock_z_cb, similarity_shock_z_cb,
  word_count_z_cb, pre_volatility_z_cb, equity_y, yield2y_y, yield10y_y, volatility_y
)

# Guide des variables livre dans le meme classeur que la base.
definitions <- c(
  event_id = "Identifiant unique banque centrale-date",
  central_bank = "ECB ou Fed ; sert a separer les echantillons",
  meeting_date = "Date officielle de la decision",
  market_trade_date = "Date de marche utilisee pour la reaction",
  rate_change_bp = "Variation realisee du taux directeur",
  emergency_event = "Evenement d'urgence hors calendrier regulier",
  scheduled_meeting = "Reunion reguliere programmee",
  primary_sample = "Vrai si l'evenement entre dans l'analyse principale",
  main_estimation_sample = "Vrai si toutes les variables du modele sont disponibles",
  exclusion_reason = "Motif documente d'exclusion",
  n_words = "Nombre de mots apres tokenisation",
  n_positive = "Nombre de termes positifs classes",
  n_negative = "Nombre de termes negatifs classes",
  tone = "(Positive - Negative) / (Positive + Negative)",
  n_hawkish = "Nombre de termes hawkish sans chevauchement",
  n_dovish = "Nombre de termes dovish sans chevauchement",
  hawkish_score = "(Hawkish - Dovish) / (Hawkish + Dovish)",
  n_sentences = "Nombre de phrases utilise pour la lisibilite",
  average_sentence_length = "Nombre moyen de mots par phrase",
  clarity = "Oppose du Flesch-Kincaid Grade",
  fk_grade = "Niveau scolaire Flesch-Kincaid",
  similarity = "Jaccard des bigrammes uniques avec le texte precedent",
  hawkish_shock_ar1 = "Innovation AR(1) du HawkishScore",
  tone_shock_ar1 = "Innovation AR(1) du Tone",
  clarity_shock_ar1 = "Innovation AR(1) de Clarity",
  similarity_shock_ar1 = "Innovation AR(1) de Similarity",
  hawkish_shock_z_cb = "HawkishShock standardise dans la banque",
  tone_shock_z_cb = "ToneShock standardise dans la banque",
  clarity_shock_z_cb = "ClarityShock standardise dans la banque",
  similarity_shock_z_cb = "SimilarityShock standardise dans la banque",
  word_count_z_cb = "Longueur du texte standardisee dans la banque",
  pre_volatility_z_cb = "Volatilite pre-meeting standardisee dans la banque",
  equity_y = "100 x rendement logarithmique close-to-close",
  yield2y_y = "Variation close-to-close du taux 2 ans",
  yield10y_y = "Variation close-to-close du taux 10 ans",
  volatility_y = "100 x variation logarithmique VIX/VSTOXX"
)
event_vars <- names(final_data)[1:10]
text_level_vars <- names(final_data)[11:22]

variable_guide <- tibble(variable = names(final_data)) %>% mutate(
  group = case_when(
    variable %in% event_vars ~ "Event identification",
    variable %in% text_level_vars ~ "Text measures",
    str_detect(variable, "_shock_|_z_cb$") ~ "Shocks and controls",
    TRUE ~ "Primary market reactions"
  ),
  definition = unname(definitions[variable]),
  unit = case_when(
    str_detect(variable, "date$") ~ "Date",
    variable %in% c("event_id", "central_bank", "exclusion_reason") ~ "Texte",
    variable %in% c("emergency_event", "scheduled_meeting", "primary_sample", "main_estimation_sample") ~ "0/1",
    variable %in% c("n_words", "n_positive", "n_negative", "n_hawkish", "n_dovish", "n_sentences") ~ "Nombre",
    variable == "average_sentence_length" ~ "Mots par phrase",
    variable %in% c("clarity", "fk_grade", "clarity_shock_ar1") ~ "Point de niveau scolaire",
    variable %in% c("rate_change_bp", "yield2y_y", "yield10y_y") ~ "Point de base",
    variable %in% c("equity_y", "volatility_y") ~ "Log-point de pourcentage",
    str_detect(variable, "_z_cb$") ~ "Ecart-type",
    TRUE ~ "Score"
  ),
  source = case_when(
    variable %in% event_vars[!event_vars %in% c("market_trade_date", "rate_change_bp")] ~
      "Calendriers et communications officielles ECB/Fed",
    variable == "rate_change_bp" ~ "ECB Data Portal et Federal Reserve",
    variable == "equity_y" ~ "Yahoo Finance historical closes",
    variable %in% c("yield2y_y", "yield10y_y") ~ "ECB Data Portal et FRED",
    variable %in% c("volatility_y", "pre_volatility_z_cb") ~ "STOXX VSTOXX et Yahoo Finance VIX",
    variable %in% text_level_vars ~ "Calculs de l'auteur sur les communications officielles",
    TRUE ~ "Calculs de l'auteur a partir des variables precedentes"
  ),
  used_in = case_when(
    variable %in% model_variables ~ "Regressions principales",
    variable %in% c("n_words", "tone", "hawkish_score", "clarity", "similarity") ~
      "Statistiques descriptives",
    TRUE ~ "Construction et validation"
  )
)

write_csv(final_data, file.path(DIR$final, "analysis_all.csv"), na = "")
write_workbook(
  list(Analysis_Data = final_data, Variable_Guide = variable_guide),
  file.path(DIR$final, "analysis_all.xlsx")
)


# 7. STATISTIQUES ET REGRESSIONS -----------------------------------------------

regular_data <- analysis_full %>% filter(primary_sample)

summarize_banks <- function(data, variables) {
  data %>%
    select(central_bank, all_of(variables)) %>%
    pivot_longer(-central_bank, names_to = "variable", values_to = "value") %>%
    group_by(central_bank, variable) %>%
    summarise(
      n = sum(!is.na(value)), mean = mean(value, na.rm = TRUE),
      sd = sd(value, na.rm = TRUE), .groups = "drop"
    ) %>%
    pivot_wider(
      names_from = central_bank, values_from = c(n, mean, sd),
      names_glue = "{central_bank}_{.value}"
    )
}

communication_by_bank <- summarize_banks(
  regular_data, c("n_words", "hawkish_score", "tone", "clarity", "similarity")
)
market_by_bank <- summarize_banks(
  regular_data, c("equity_y", "yield2y_y", "yield10y_y", "volatility_y")
)
sample_overview <- regular_data %>%
  count(central_bank, name = "regular_meetings") %>%
  left_join(analysis_full %>% count(central_bank, name = "texts_in_history"), by = "central_bank") %>%
  mutate(emergency_texts_excluded_from_market_models = if_else(central_bank == "Fed", 2L, 0L))

descriptive_variables <- c(
  "hawkish_score", "tone", "clarity", "similarity", "n_words",
  "equity_y", "yield2y_y", "yield10y_y", "volatility_y", "rate_change_bp"
)
summary_statistics <- regular_data %>%
  select(all_of(descriptive_variables)) %>%
  pivot_longer(everything(), names_to = "variable", values_to = "value") %>%
  group_by(variable) %>%
  summarise(
    n = sum(!is.na(value)), mean = mean(value, na.rm = TRUE),
    median = median(value, na.rm = TRUE), sd = sd(value, na.rm = TRUE),
    min = min(value, na.rm = TRUE), q1 = quantile(value, .25, na.rm = TRUE),
    q3 = quantile(value, .75, na.rm = TRUE), max = max(value, na.rm = TRUE),
    .groups = "drop"
  )

measure_labels <- c(
  hawkish_score = "HawkishScore", tone = "Tone",
  clarity = "Clarity", similarity = "Similarity"
)
ar1_diagnostics <- bind_rows(lapply(c("ECB", "Fed"), function(bank) {
  bind_rows(lapply(names(measure_labels), function(measure) {
    d <- analysis_full %>%
      filter(central_bank == bank) %>% arrange(meeting_date) %>%
      transmute(value = .data[[measure]], lag1 = lag(.data[[measure]])) %>%
      filter(is.finite(value), is.finite(lag1))
    model <- lm(value ~ lag1, d)
    residual_values <- residuals(model)
    tibble(
      bank, measure = measure_labels[[measure]], n = nrow(d),
      rho = unname(coef(model)["lag1"]), r_squared = summary(model)$r.squared,
      residual_lag1_correlation = cor(head(residual_values, -1), tail(residual_values, -1))
    )
  }))
}))

text_variables <- c(
  "hawkish_shock_z_cb", "tone_shock_z_cb", "clarity_shock_z_cb", "similarity_shock_z_cb"
)
control_variables <- c("rate_change_bp", "pre_volatility_z_cb", "word_count_z_cb")
dependent_variables <- c("equity_y", "yield2y_y", "yield10y_y", "volatility_y")
all_model_variables <- c(text_variables, control_variables, dependent_variables)
model_samples <- list(
  ECB = regular_data %>% filter(central_bank == "ECB") %>%
    drop_na(all_of(all_model_variables)) %>% arrange(meeting_date),
  Fed = regular_data %>% filter(central_bank == "Fed") %>%
    drop_na(all_of(all_model_variables)) %>% arrange(meeting_date)
)
if (nrow(model_samples$ECB) != 86 || nrow(model_samples$Fed) != 85) {
  stop("Echantillon inattendu : ECB=", nrow(model_samples$ECB), ", Fed=", nrow(model_samples$Fed))
}

hc3_coefficients <- function(model) {
  variance <- vcovHC(model, type = "HC3")
  estimate <- coef(model)
  std_error <- sqrt(diag(variance))
  t_stat <- estimate / std_error
  p_value <- 2 * pt(abs(t_stat), df.residual(model), lower.tail = FALSE)
  critical <- qt(.975, df.residual(model))
  tibble(
    term = names(estimate), estimate = as.numeric(estimate),
    std_error = as.numeric(std_error), t_stat = as.numeric(t_stat),
    p_value = as.numeric(p_value), ci_lower = estimate - critical * std_error,
    ci_upper = estimate + critical * std_error,
    significance = case_when(
      p_value < .01 ~ "***", p_value < .05 ~ "**", p_value < .10 ~ "*", TRUE ~ ""
    ),
    standard_error_method = "HC3"
  )
}

rhs <- paste(c(text_variables, control_variables), collapse = " + ")
models <- full_coefficients <- model_summaries <- list()
row <- 1L
for (bank in c("ECB", "Fed")) {
  for (dependent in dependent_variables) {
    formula <- as.formula(paste(dependent, "~", rhs))
    model <- lm(formula, model_samples[[bank]])
    models[[paste(bank, dependent, sep = "__")]] <- model
    model_summary <- summary(model)
    full_coefficients[[row]] <- hc3_coefficients(model) %>% mutate(
      central_bank = bank, dependent_variable = dependent, n_obs = nobs(model),
      r_squared = model_summary$r.squared, adj_r_squared = model_summary$adj.r.squared,
      model = "Joint multidimensional event-study model"
    )
    model_summaries[[row]] <- tibble(
      central_bank = bank, dependent_variable = dependent,
      formula = paste(deparse(formula), collapse = " "), n_obs = nobs(model),
      residual_df = df.residual(model),
      r_squared = model_summary$r.squared, adj_r_squared = model_summary$adj.r.squared,
      standard_error_method = "HC3"
    )
    row <- row + 1L
  }
}
full_coefficients <- bind_rows(full_coefficients)
model_summaries <- bind_rows(model_summaries)

text_results <- full_coefficients %>%
  filter(term %in% text_variables) %>%
  mutate(
    text_variable = term,
    text_dimension = recode(
      term, hawkish_shock_z_cb = "HawkishShock", tone_shock_z_cb = "ToneShock",
      clarity_shock_z_cb = "ClarityShock", similarity_shock_z_cb = "SimilarityShock"
    ),
    inference_family = if_else(
      term == "hawkish_shock_z_cb", "Four HawkishShock outcomes",
      "Twelve complementary text coefficients"
    )
  ) %>%
  group_by(central_bank, inference_family) %>%
  mutate(
    family_test_count = n(), q_value_bh = p.adjust(p_value, "BH"),
    survives_fdr_5pct = q_value_bh < .05, survives_fdr_10pct = q_value_bh < .10,
    adjusted_evidence = case_when(
      q_value_bh < .05 ~ "Statistically precise at the 5% FDR threshold",
      q_value_bh < .10 ~ "Suggestive at the 10% FDR threshold",
      TRUE ~ "Not statistically precise after FDR adjustment"
    )
  ) %>%
  ungroup() %>%
  select(
    term, estimate, std_error, t_stat, p_value, ci_lower, ci_upper, significance,
    standard_error_method, central_bank, dependent_variable, text_variable,
    text_dimension, n_obs, r_squared, adj_r_squared, model, inference_family,
    family_test_count, q_value_bh, survives_fdr_5pct, survives_fdr_10pct,
    adjusted_evidence
  ) %>%
  arrange(central_bank, inference_family, p_value)

economic_significance <- text_results %>% rowwise() %>% mutate(
  outcome_sd = sd(model_samples[[central_bank]][[dependent_variable]]),
  coefficient_in_outcome_sd = estimate / outcome_sd
) %>% ungroup()

complete_model_estimates <- full_coefficients %>% transmute(
  central_bank, dependent_variable, term, estimate,
  std_error_hc3 = std_error, ci_lower_95 = ci_lower, ci_upper_95 = ci_upper,
  p_value, n_obs, r_squared, adj_r_squared
)


# 8. DIAGNOSTICS ET EXPORTS ----------------------------------------------------

calculate_vif <- function(model) {
  x <- model.matrix(model)
  x <- x[, colnames(x) != "(Intercept)", drop = FALSE]
  tibble(term = colnames(x), vif = vapply(seq_len(ncol(x)), function(column) {
    y <- x[, column]
    auxiliary <- lm.fit(cbind(1, x[, -column, drop = FALSE]), y)
    r_squared <- 1 - sum(auxiliary$residuals^2) / sum((y - mean(y))^2)
    1 / (1 - r_squared)
  }, numeric(1)))
}

diagnostic_rows <- vif_rows <- list()
row <- 1L
for (key in names(models)) {
  model <- models[[key]]
  parts <- strsplit(key, "__", fixed = TRUE)[[1]]
  vif <- calculate_vif(model)
  cooks_d <- cooks.distance(model)
  diagnostic_rows[[row]] <- tibble(
    central_bank = parts[1], dependent_variable = parts[2], n_obs = nobs(model),
    max_vif = max(vif$vif),
    max_vif_term = vif$term[which.max(vif$vif)], breusch_pagan_p = bptest(model)$p.value,
    cooks_over_4n = sum(cooks_d > 4 / nobs(model)),
    max_cooks_d = max(cooks_d)
  )
  vif_rows[[row]] <- vif %>% mutate(
    central_bank = parts[1], dependent_variable = parts[2], .before = 1
  )
  row <- row + 1L
}
diagnostics <- bind_rows(diagnostic_rows)
vif_details <- bind_rows(vif_rows)

write_workbook(
  list(
    Dictionary = dictionary_definition, Sample = sample_overview,
    Summary = summary_statistics, Communication = communication_by_bank,
    Market_Reactions = market_by_bank, AR1_Diagnostics = ar1_diagnostics
  ),
  file.path(DIR$output, "thesis_descriptive_results.xlsx")
)
write_workbook(
  list(
    Main_Results = text_results,
    Full_ECB = filter(complete_model_estimates, central_bank == "ECB"),
    Full_Fed = filter(complete_model_estimates, central_bank == "Fed"),
    Model_Summary = model_summaries, Economic_Significance = economic_significance,
    Diagnostics = diagnostics, VIF = vif_details
  ),
  file.path(DIR$output, "thesis_regression_results.xlsx")
)
