From 2e2b9600fc85656c538fe502fd24b6df284fd7a2 Mon Sep 17 00:00:00 2001 From: Sergey Bondarkov Date: Tue, 22 Apr 2025 18:44:26 +0300 Subject: [PATCH 1/4] fixes adjustment of line 2400 and related issues --- .../4_combine_rosstat_fns_panels.R | 99 ++++++---- .../1_financials/5_build_articulation_panel.R | 46 ++--- code/1_financials/6_adjust_values.R | 169 ++++++++++-------- .../helpers/check_articulation_functions.R | 14 +- code/1_financials/helpers/lines_tags_dict.R | 2 +- code/1_financials/helpers/parsing_functions.R | 12 +- 6 files changed, 192 insertions(+), 150 deletions(-) diff --git a/code/1_financials/4_combine_rosstat_fns_panels.R b/code/1_financials/4_combine_rosstat_fns_panels.R index 471033d..ea9380e 100644 --- a/code/1_financials/4_combine_rosstat_fns_panels.R +++ b/code/1_financials/4_combine_rosstat_fns_panels.R @@ -5,58 +5,81 @@ setDTthreads(0) # Load data ============================================================================== rosstat_financials <- read_fst("temp/rosstat_financials_impFrNY.fst", as.data.table = T) fns_financials <- read_fst("temp/fns_financials_impFrNY.fst", as.data.table = T) -fns_financials_20172018 <- fns_financials[year < 2019] - -rosstat_financials_20172018_new_obs <- fns_financials_20172018[!rosstat_financials, on = c("inn", "year")] - -rosstat_financials <- rbindlist(list( - rosstat_financials, - rosstat_financials_20172018_new_obs - ), - use.names = T, fill = T -) - # Combine Rosstat and GIR BO (BFO) ========================================================== combined_financials <- rbindlist(list( rosstat_financials, + fns_financials[year < 2019][!rosstat_financials, on = c("inn", "year")], fns_financials[year >= 2019] ), fill = T, use.names = T +) +rm(rosstat_financials) +rm(fns_financials) +gc() # Check -print(dcast(combined_financials[, .N, keyby = .(year, all_na)], year ~ paste0("all_na_", all_na))) -print(dcast(combined_financials[, .N, keyby = .(year, new_obs)], year ~ paste0("new_obs_", new_obs))) -print(dcast(combined_financials[, .N, keyby = .(new_obs, all_na)], new_obs ~ paste0("all_na_", all_na))) -print(dcast(combined_financials[, .N, keyby = .(simplified, all_na)], simplified ~ paste0("all_na_", all_na))) -print(combined_financials[, lapply(.SD, mean), .SDcols = patterns("imp_"), keyby = year]) +# print(dcast(combined_financials[, .N, keyby = .(year, all_na)], year ~ paste0("all_na_", all_na))) +# print(dcast(combined_financials[, .N, keyby = .(year, new_obs)], year ~ paste0("new_obs_", new_obs))) +# print(dcast(combined_financials[, .N, keyby = .(new_obs, all_na)], new_obs ~ paste0("all_na_", all_na))) +# print(dcast(combined_financials[, .N, keyby = .(simplified, all_na)], simplified ~ paste0("all_na_", all_na))) +# print(combined_financials[, lapply(.SD, mean), .SDcols = patterns("imp_"), keyby = year]) -# Negative lines values to positive ================================================= +# Bracketed negative lines values to positive ================================================= -neg_lines <- paste0("line_", c(1320:1323, - 2120:2123, - 2210:2213, - 2220:2223, - 2330:2333, - 2350:2353, - 2411, - 3220:3227, - 3320:3327, - 4120:4129, - 4220:4229, - 4320:4329, - 6310:6313, - 6320:6326, - 6330, - 6350:6359, - 6300 - )) +neg_lines_full <- paste0("line_", c( + 1320:1323, + 2120:2123, + 2210:2213, + 2220:2223, + 2330:2333, + 2350:2353, + 2411, + 3220:3227, + 3320:3327, + 4120:4129, + 4220:4229, + 4320:4329, + 6310:6313, + 6320:6326, + 6330, + 6350:6359, + 6300 + )) # Not all the lines are present in data so we repack them as regex -neg_lines_pattern <- paste(neg_lines, collapse="|") -neg_lines_present <- grep(neg_lines_pattern, names(combined_financials), value = T) +neg_lines_pattern_full <- paste(neg_lines_full, collapse="|") +neg_lines_present_full <- grep(neg_lines_pattern_full, names(combined_financials), value = T) +combined_financials[simplified == 0, (neg_lines_present_full) := lapply(.SD, function(l) fifelse(l > 0, -l, l)), .SDcols = neg_lines_present_full] + +combined_financials[simplified == 0 & year < 2019 & line_2410 > 0, line_2410 := -line_2410] +combined_financials[simplified == 0 & year >= 2019 + & line_2410 > 0 + & (abs(line_2411) > line_2412 + | (!is.na(line_2411) & line_2411 != 0 & is.na(line_2412))), + line_2410 := -line_2410] -combined_financials[, (neg_lines_present) := lapply(.SD, function(l) fifelse(l < 0, -l, l)), .SDcols = neg_lines_present] +neg_lines_simple <- paste0("line_", c( + 2120, + 2330, + 2350, + 2410, + 3220:3227, + 3320:3327, + 4120:4129, + 4220:4229, + 4320:4329, + 6310, + 6320, + 6330, + 6350, + 6300 + )) + +# Not all the lines are present in data so we repack them as regex +neg_lines_pattern_simple <- paste(neg_lines_simple, collapse="|") +neg_lines_present_simple <- grep(neg_lines_pattern_simple, names(combined_financials), value = T) +combined_financials[simplified == 1, (neg_lines_present_simple) := lapply(.SD, function(l) fifelse(l > 0, -l, l)), .SDcols = neg_lines_present_simple] # Save setorderv(combined_financials, c("inn", "year")) diff --git a/code/1_financials/5_build_articulation_panel.R b/code/1_financials/5_build_articulation_panel.R index 902c54e..1040289 100644 --- a/code/1_financials/5_build_articulation_panel.R +++ b/code/1_financials/5_build_articulation_panel.R @@ -2,27 +2,29 @@ library(fst) library(data.table) setDTthreads(0) library(ggplot2) -source("financials/helpers/check_articulation_functions.R") +source("code/data_processing/helpers/check_articulation_functions.R") + +articulation_path <- file.path("output", "articulation") +dir.create(articulation_path, recursive = T, showWarnings = F) # Load panel =================================================================== russian_financials <- read_fst("temp/combined_financials_impFrNY_negLinesCorr.fst", as.data.table = T) - russian_financials[, articulation_basic := as.numeric(line_1600 == line_1700)] # Separate checks ================================================================= -rosneft <- russian_financials[inn == "7706107510"] -gazprom <- russian_financials[inn == "7736050003"] - -check_balance_full(rosneft) -check_finres_full(rosneft) -check_cashflow_full(rosneft) -rosneft[, .SD, .SDcols = patterns("year|fine")] - -check_balance_full(gazprom) -check_finres_full(gazprom) -check_cashflow_full(gazprom) -gazprom[, .SD, .SDcols = patterns("year|fine")] +# rosneft <- russian_financials[inn == "7706107510"] +# gazprom <- russian_financials[inn == "7736050003"] +# +# check_balance_full(rosneft) +# check_finres_full(rosneft) +# check_cashflow_full(rosneft) +# rosneft[, .SD, .SDcols = patterns("year|fine")] +# +# check_balance_full(gazprom) +# check_finres_full(gazprom) +# check_cashflow_full(gazprom) +# gazprom[, .SD, .SDcols = patterns("year|fine")] # ================================================================================ russian_financials_full <- russian_financials[simplified == 0] @@ -64,31 +66,31 @@ plot_check_failed_share <- function(dt, check_name) { labs(y = "Failed checks share", y = "") print(pl) +} ## Full statements check_balance_full(russian_financials_full) plot_check_failed_share(russian_financials_full, "balance") -ggsave("output/balance_check_full_impFrNY.pdf", width = 12, height = 7) +ggsave(file.path(articulation_path, "balance_check_full_impFrNY.pdf"), width = 12, height = 7) check_finres_full(russian_financials_full) plot_check_failed_share(russian_financials_full, "finres") -ggsave("output/finres_check_full_impFrNY.pdf", width = 12, height = 7) +ggsave(file.path(articulation_path, "finres_check_full_impFrNY.pdf"), width = 12, height = 7) check_cashflow_full(russian_financials_full) plot_check_failed_share(russian_financials_full, "cashflow") -ggsave("output/cashflow_check_full_impFrNY.pdf", width = 12, height = 7) - +ggsave(file.path(articulation_path, "cashflow_check_full_impFrNY.pdf"), width = 12, height = 7) ## Simplified statements check_balance_simple(russian_financials_simple) plot_check_failed_share(russian_financials_simple, "balance") -ggsave("output/balance_check_simple_impFrNY.pdf", width = 12, height = 7) +ggsave(file.path(articulation_path, "balance_check_simple_impFrNY.pdf"), width = 12, height = 7) check_finres_simple(russian_financials_simple) plot_check_failed_share(russian_financials_simple, "finres") -ggsave("output/finres_check_simple_impFrNY.pdf", width = 12, height = 7) +ggsave(file.path(articulation_path, "finres_check_simple_impFrNY.pdf"), width = 12, height = 7) # Check articulation =============================================================== @@ -99,4 +101,6 @@ russian_financials <- rbindlist(list(russian_financials_full, russian_financials # Save ============================================================================= setorderv(russian_financials, c("inn", "year")) -write_fst(russian_financials[, .(inn, year, new_obs, simplified, articulation, articulation_basic, imp_any_from_future)], "output/articulation_impFrNY.fst") +write_fst(russian_financials[, .(inn, year, new_obs, simplified, + articulation, articulation_basic, imp_any_from_future)], + file.path(articulation_path, "articulation_panel.fst")) diff --git a/code/1_financials/6_adjust_values.R b/code/1_financials/6_adjust_values.R index cf174cb..5762dd4 100644 --- a/code/1_financials/6_adjust_values.R +++ b/code/1_financials/6_adjust_values.R @@ -5,14 +5,13 @@ setDTthreads(0) # Load data russian_financials <- read_fst("temp/combined_financials_impFrNY_negLinesCorr.fst", as.data.table = T) russian_financials[, adj_any := 0] -# russian_financials[inn == "0100000621", .SD, .SDcols = patterns("line_1")] # Declare imputation function impute <- function(dt, imp_target, lines_to_sum, flag_imputation = T) { regex <- paste(lines_to_sum, collapse = "|") env <- list(imp_target = imp_target, imp_value = paste0(imp_target, "_imp")) - dt[, useless := rowSums(is.na(.SD)) == length(lines_to_sum), .SDcols = patterns(regex) ] + dt[, useless := pmax(rowSums(is.na(.SD)), rowSums(.SD == 0)) == length(lines_to_sum), .SDcols = patterns(regex) ] dt[useless == F, imp_value := rowSums(.SD, na.rm = T), .SDcols = patterns(regex), env = env] dt[, orig_value := imp_target, env = env] @@ -26,7 +25,6 @@ impute <- function(dt, imp_target, lines_to_sum, flag_imputation = T) { dt[, imp_value := NULL, env = env] dt[, useless := NULL] dt[, orig_value := NULL] - # return(dt) } # Function demo: @@ -44,30 +42,33 @@ impute <- function(dt, imp_target, lines_to_sum, flag_imputation = T) { # Lower levels imputation: sum of XXX[1-9] lines --------------------------- XXX0_lines_for_simple_imp <- c("line_3210", "line_3220", "line_3310", "line_3320", - "line_3410", "line_3420", "line_4110", "line_4120", "line_4210", "line_4220", "line_4310", "line_4320", "line_6310", "line_6320") for(imp_target in XXX0_lines_for_simple_imp) { - regex <- paste0(stringi::stri_sub(imp_target, 1, -2), "[1-9x]") # e.g. "line_1230" > "line_123[1-9]" + regex <- paste0(stringi::stri_sub(imp_target, 1, -2), "[1-9x]") # e.g. "line_1230" > "line_123[1-9x]" lines_to_sum <- grep(regex, names(russian_financials), value = T) - # message(imp_target, " = ", paste(lines_to_sum, collapse = " + ")) impute(russian_financials, imp_target, lines_to_sum) +} # Higher level imputation: sum of XX[1-9]0 lines ----------------------------- -XX00_lines_for_simple_imp <- c("line_1100", "line_1200", - "line_1300", "line_1400", - "line_1500", "line_6200", +XX00_lines_for_simple_imp <- c("line_1100", + "line_1200", + "line_1400", + "line_1500", + "line_6200", "line_6300") for(imp_target in XX00_lines_for_simple_imp) { regex <- paste0(stringi::stri_sub(imp_target, 1, -3), "[1-9]0") # e.g. "line_1200" > "line_12[1-9]0" + lines_to_sum <- grep(regex, names(russian_financials), value = T) + impute(russian_financials, imp_target, lines_to_sum) +} # Imputation using fomulas common to simplified and non-simplifieds --------- -russian_financials[, line_6300_neg := -line_6300] -impute(russian_financials, "line_6400", c("line_6100", "line_6200", "line_6300_neg")) +impute(russian_financials, "line_6400", c("line_6100", "line_6200", "line_6300")) # Imputation using specific formulas for simplifieds and non-simplifieds ---- russian_financials_full <- russian_financials[simplified == 0] @@ -78,99 +79,113 @@ gc() ## Full statements ### Prepare some lines -for(l in c("line_2120", "line_2210", "line_2220", "line_2330", "line_2350", - "line_3220", "line_3320", "line_4120", "line_4220", "line_4320")) { - russian_financials_full[, l_neg := -l, env = list(l = l, l_neg = paste0(l, "_neg"))] +# for(l in c("line_1320", "line_2120", "line_2210", "line_2220", "line_2330", "line_2350", + # "line_3220", "line_3320", "line_4120", "line_4220", "line_4320")) { + # russian_financials_full[, l_neg := -l, env = list(l = l, l_neg = paste0(l, "_neg"))] +# } ### Impute +impute(russian_financials_full, "line_1300", c("line_1310", "line_1320", "line_1340", "line_1350", "line_1360", "line_1370")) +# TODO: in NGOs case imputation of line 1300 should be different: line 1320 included as positive, impute(russian_financials_full, "line_1600", c("line_1100", "line_1200")) impute(russian_financials_full, "line_1700", c("line_1300", "line_1400", "line_1500")) -impute(russian_financials_full, "line_2100", c("line_2110", "line_2120_neg")) -impute(russian_financials_full, "line_2200", c("line_2100", "line_2210_neg", "line_2220_neg")) -impute(russian_financials_full, "line_2300", c("line_2200", "line_2310", "line_2320", "line_2330_neg", "line_2340", "line_2350_neg")) - - +impute(russian_financials_full, "line_2100", c("line_2110", "line_2120")) +impute(russian_financials_full, "line_2200", c("line_2100", "line_2210", "line_2220")) +impute(russian_financials_full, "line_2300", c("line_2200", "line_2310", "line_2320", "line_2330", "line_2340", "line_2350")) # Impute 24XX -russian_financials_full[, line_2410_neg := -line_2410] -impute(russian_financials_full[year >= 2020], "line_2410", c("line_2411", "line_2412")) -impute(russian_financials_full, "line_2400", c("line_2300", "line_2410_neg", "line_2460")) - -## Construct 24XX lines with the same meaning across different periods -russian_financials_full[, line_2410_uniform_tax := NA_real_] -russian_financials_full[, line_2411_uniform_tax := NA_real_] -russian_financials_full[, line_2412_uniform_tax := NA_real_] -russian_financials_full[, line_2400_uniform_tax := NA_real_] -russian_financials_full[, line_2500_uniform_tax := NA_real_] - -### Before 2018 -russian_financials_full[year < 2018, line_2411_uniform_tax := line_2410] -impute(russian_financials_full[year < 2018], "line_2410_uniform_tax", c("line_2410_neg", "line_2430", "line_2450"), flag_imputation = F) -impute(russian_financials_full[year < 2020], "line_2412_uniform_tax", c("line_2430", "line_2450"), flag_imputation = F) -russian_financials_full[year < 2018, line_2460_neg := -line_2460] -impute(russian_financials_full[year < 2018], "line_2400_uniform_tax", c("line_2300", "line_2410_uniform_tax", "line_2460_neg"), flag_imputation = F) -# impute(russian_financials_full[year == 2017], "line_2400_uniform_tax", c("line_2300", "line_2410_uniform_tax", "line_2460"), flag_imputation = F) -russian_financials_full[year == 2017, line_2400_uniform_tax_neg_2460 := line_2400] -impute(russian_financials_full[year == 2017], "line_2400_uniform_tax_neg_2460", c("line_2300", "line_2410_uniform_tax", "line_2460_neg"), flag_imputation = F) - - - -### In 2018 -russian_financials_full[year == 2018 & (!is.na(line_2411) | !is.na(line_2412)), line_2411_uniform_tax := line_2411] -russian_financials_full[year == 2018 & (!is.na(line_2411) | !is.na(line_2412)), line_2412_uniform_tax := line_2412] -russian_financials_full[year == 2018 & (!is.na(line_2411) | !is.na(line_2412)), line_2411_neg := -line_2411] -impute(russian_financials_full[year == 2018 & (!is.na(line_2411) | !is.na(line_2412))], "line_2410_uniform_tax", c("line_2411_neg", "line_2412"), flag_imputation = F) - -russian_financials_full[year == 2018 & (is.na(line_2411) & is.na(line_2412)), line_2410_neg := -line_2410] -impute(russian_financials_full[year == 2018 & (is.na(line_2411) & is.na(line_2412))], "line_2410_uniform_tax", c("line_2410_neg", "line_2430", "line_2450")) -russian_financials_full[year == 2018 & (is.na(line_2411) & is.na(line_2412)), line_2411_uniform_tax := line_2410] -impute(russian_financials_full[year == 2018 & (is.na(line_2411) & is.na(line_2412))], "line_2412_uniform_tax", c("line_2430", "line_2450"), flag_imputation = F) -impute(russian_financials_full[year == 2018 & (is.na(line_2411) & is.na(line_2412))], "line_2400_uniform_tax", c("line_2300", "line_2410_uniform_tax", "line_2460"), flag_imputation = F) - -### After 2018 -impute(russian_financials_full[year > 2018], "line_2400_uniform_tax", c("line_2300", "line_2410", "line_2460"), flag_imputation = F) +## Imputing 2412 and 2410 is theoretically right but in practice it adds erroneous values: +## 2411, 2412 as well as 2430 and 2450 often contain mistakes which affect +## the otherwise correct 2410 and, in turn, 2400. +# impute(russian_financials_full[year >= 2019], "line_2412", c("line_2430", "line_2450")) +# russian_financials_full[, line_2411 := -line_2411] +# impute(russian_financials_full[year >= 2019], "line_2410", c("line_2411", "line_2412")) + +impute(russian_financials_full, "line_2400", c("line_2300", "line_2410", "line_2460")) + +# # Check +# russian_financials_full[inn == "7703443256", .(year, line_2400, line_2300, line_2410, line_2411, line_2460)] +# russian_financials_full[year >= 2019][inn %in% sample(inn, 5), .(inn, year, line_2400, line_2300, line_2410, line_2411, line_2460)] + +# ## Construct 24XX lines with the same meaning across different periods +# russian_financials_full[, line_2410_uniform_tax := NA_real_] +# russian_financials_full[, line_2411_uniform_tax := NA_real_] +# russian_financials_full[, line_2412_uniform_tax := NA_real_] +# russian_financials_full[, line_2400_uniform_tax := NA_real_] +# russian_financials_full[, line_2500_uniform_tax := NA_real_] +# +# ### Before 2018 +# russian_financials_full[year < 2018, line_2411_uniform_tax := line_2410] +# impute(russian_financials_full[year < 2018], "line_2410_uniform_tax", c("line_2410", "line_2430", "line_2450"), flag_imputation = F) +# impute(russian_financials_full[year < 2020], "line_2412_uniform_tax", c("line_2430", "line_2450"), flag_imputation = F) +# russian_financials_full[year < 2018, line_2460 := -line_2460] +# impute(russian_financials_full[year < 2018], "line_2400_uniform_tax", c("line_2300", "line_2410_uniform_tax", "line_2460"), flag_imputation = F) +# # impute(russian_financials_full[year == 2017], "line_2400_uniform_tax", c("line_2300", "line_2410_uniform_tax", "line_2460"), flag_imputation = F) +# russian_financials_full[year == 2017, line_2400_uniform_tax_neg_2460 := line_2400] +# impute(russian_financials_full[year == 2017], "line_2400_uniform_tax_neg_2460", c("line_2300", "line_2410_uniform_tax", "line_2460"), flag_imputation = F) +# +# +# +# ### In 2018 +# russian_financials_full[year == 2018 & (!is.na(line_2411) | !is.na(line_2412)), line_2411_uniform_tax := line_2411] +# russian_financials_full[year == 2018 & (!is.na(line_2411) | !is.na(line_2412)), line_2412_uniform_tax := line_2412] +# russian_financials_full[year == 2018 & (!is.na(line_2411) | !is.na(line_2412)), line_2411 := -line_2411] +# impute(russian_financials_full[year == 2018 & (!is.na(line_2411) | !is.na(line_2412))], "line_2410_uniform_tax", c("line_2411", "line_2412"), flag_imputation = F) +# +# russian_financials_full[year == 2018 & (is.na(line_2411) & is.na(line_2412)), line_2410 := -line_2410] +# impute(russian_financials_full[year == 2018 & (is.na(line_2411) & is.na(line_2412))], "line_2410_uniform_tax", c("line_2410", "line_2430", "line_2450")) +# russian_financials_full[year == 2018 & (is.na(line_2411) & is.na(line_2412)), line_2411_uniform_tax := line_2410] +# impute(russian_financials_full[year == 2018 & (is.na(line_2411) & is.na(line_2412))], "line_2412_uniform_tax", c("line_2430", "line_2450"), flag_imputation = F) +# impute(russian_financials_full[year == 2018 & (is.na(line_2411) & is.na(line_2412))], "line_2400_uniform_tax", c("line_2300", "line_2410_uniform_tax", "line_2460"), flag_imputation = F) +# +# ### After 2018 +# impute(russian_financials_full[year > 2018], "line_2400_uniform_tax", c("line_2300", "line_2410", "line_2460"), flag_imputation = F) +# # Impute 2500 -impute(russian_financials_full[year < 2019], "line_2500", c("line_2400", "line_2510", "line_2520")) -impute(russian_financials_full[year < 2019], "line_2500_uniform_tax", c("line_2410_uniform_tax", "line_2510", "line_2520"), flag_imputation = F) -impute(russian_financials_full[year >= 2019], "line_2500", c("line_2410", "line_2510", "line_2520", "line_2530")) -impute(russian_financials_full[year >= 2019], "line_2500_uniform_tax", c("line_2410_uniform_tax", "line_2510", "line_2520", "line_2530"), flag_imputation = F) +impute(russian_financials_full, "line_2500", c("line_2400", "line_2510", "line_2520", "line_2530")) +# impute(russian_financials_full[year < 2019], "line_2500_uniform_tax", c("line_2410_uniform_tax", "line_2510", "line_2520"), flag_imputation = F) +# impute(russian_financials_full[year >= 2019], "line_2500_uniform_tax", c("line_2410_uniform_tax", "line_2510", "line_2520", "line_2530"), flag_imputation = F) # Impute 3XXX and 4XXX -impute(russian_financials_full, "line_3230", c("line_3210", "line_3220_neg")) -impute(russian_financials_full, "line_3200", c("line_3100", "line_3210", "line_3220_neg")) -impute(russian_financials_full, "line_3300", c("line_3200", "line_3310", "line_3320_neg")) +impute(russian_financials_full, "line_3230", c("line_3210", "line_3220")) +impute(russian_financials_full, "line_3200", c("line_3100", "line_3210", "line_3220")) +impute(russian_financials_full, "line_3300", c("line_3200", "line_3310", "line_3320")) # line_3400? impute(russian_financials_full, "line_3500", c("line_3400", "line_3410", "line_3420")) impute(russian_financials_full, "line_3600", c("line_1300")) -impute(russian_financials_full, "line_4100", c("line_4110", "line_4120_neg")) -impute(russian_financials_full, "line_4200", c("line_4210", "line_4220_neg")) -impute(russian_financials_full, "line_4300", c("line_4310", "line_4320_neg")) +impute(russian_financials_full, "line_4100", c("line_4110", "line_4120")) +impute(russian_financials_full, "line_4200", c("line_4210", "line_4220")) +impute(russian_financials_full, "line_4300", c("line_4310", "line_4320")) impute(russian_financials_full, "line_4400", c("line_4100", "line_4200", "line_4300")) impute(russian_financials_full, "line_4500", c("line_4400", "line_4450", "line_4490")) ## Simplified statements -for(l in c("line_2120", "line_2330", "line_2350", "line_2410")) { - russian_financials_simple[, l_neg := -l, env = list(l = l, l_neg = paste0(l, "_neg"))] +### Prepare some lines +# for(l in c("line_2120", "line_2330", "line_2350", "line_2410")) { + # russian_financials_simple[, l_neg := -l, env = list(l = l, l_neg = paste0(l, "_neg"))] +# } +### Impute impute(russian_financials_simple, "line_1600", c("line_1150", "line_1170", "line_1210", "line_1250", "line_1230")) impute(russian_financials_simple, "line_1700", c("line_1300", "line_1350", "line_1360", "line_1410", "line_1450", "line_1510", "line_1520", "line_1550")) -impute(russian_financials_simple, "line_2200", c("line_2110", "line_2120_neg")) -impute(russian_financials_simple, "line_2300", c("line_2200", "line_2330_neg", "line_2340", "line_2350_neg")) -impute(russian_financials_simple, "line_2400", c("line_2110", "line_2120_neg", "line_2330_neg", "line_2340", "line_2350_neg", "line_2410_neg")) +impute(russian_financials_simple, "line_2200", c("line_2110", "line_2120")) +impute(russian_financials_simple, "line_2300", c("line_2200", "line_2330", "line_2340", "line_2350")) +impute(russian_financials_simple, "line_2400", c("line_2110", "line_2120", "line_2330", "line_2340", "line_2350", "line_2410")) russian_financials_simple[, line_2500 := line_2400] -russian_financials_simple[, line_2500_uniform_tax := line_2500] +# russian_financials_simple[, line_2500_uniform_tax := line_2500] # Combine data russian_financials <- rbindlist(list(russian_financials_full, russian_financials_simple), use.names=T, fill=T) -lines_to_delete <- grep("\\d_neg", names(russian_financials), value = T) -russian_financials[, (lines_to_delete) := NULL] +# lines_to_delete <- grep("\\d_neg", names(russian_financials), value = T) +# russian_financials[, (lines_to_delete) := NULL] -print(russian_financials[, .(adj_any = mean(adj_any)), keyby = year]) -print(russian_financials[, .N, keyby = year]) +# print(russian_financials[, .(adj_any = mean(adj_any)), keyby = year]) +# print(russian_financials[, .N, keyby = year]) # Tidy up and save setorderv(russian_financials, c("inn", "year")) -write_fst(russian_financials, "output/russian_financials_2011_2023_imp_adj.fst") +maxyear <- last(russian_financials$year) +write_fst(russian_financials, glue::glue("output/russian_financials_2011_{maxyear}.fst")) diff --git a/code/1_financials/helpers/check_articulation_functions.R b/code/1_financials/helpers/check_articulation_functions.R index 4966445..02c25ee 100644 --- a/code/1_financials/helpers/check_articulation_functions.R +++ b/code/1_financials/helpers/check_articulation_functions.R @@ -31,9 +31,9 @@ check_finres_full <- function(dt) { # conditions from https://www.consultant.ru/document/cons_doc_LAW_331182/dc2cbdd712256af48282919fd9edef3d1cec9f65/ neg_lines <- c("line_2120", "line_2210", "line_2220", "line_2330", "line_2350") dt[, paste0(neg_lines, "_neg") := lapply(.SD, function(x) -x), .SDcols = neg_lines] - dt[, finres_fine_2100 := as.numeric(abs(line_2100 - rowSums(.SD, na.rm = T)) <= 4), .SDcols = c("line_2110", "line_2120_neg")] - dt[, finres_fine_2200 := as.numeric(abs(line_2200 - rowSums(.SD, na.rm = T)) <= 4), .SDcols = c("line_2100", "line_2210_neg", "line_2220_neg")] - dt[, finres_fine_2300 := as.numeric(abs(line_2300 - rowSums(.SD, na.rm = T)) <= 4), .SDcols = c("line_2200", "line_2310", "line_2320", "line_2330_neg", "line_2340", "line_2350_neg")] + dt[, finres_fine_2100 := as.numeric(abs(line_2100 - rowSums(.SD, na.rm = T)) <= 4), .SDcols = c("line_2110", "line_2120")] + dt[, finres_fine_2200 := as.numeric(abs(line_2200 - rowSums(.SD, na.rm = T)) <= 4), .SDcols = c("line_2100", "line_2210", "line_2220")] + dt[, finres_fine_2300 := as.numeric(abs(line_2300 - rowSums(.SD, na.rm = T)) <= 4), .SDcols = c("line_2200", "line_2310", "line_2320", "line_2330", "line_2340", "line_2350")] dt[, paste0(neg_lines, "_neg") := NULL] check_cols <- grep("finres_fine_", names(dt), value = T) setnafill(dt, cols = check_cols, fill = 1) @@ -46,7 +46,7 @@ check_finres_simple <- function(dt) { # conditions from https://www.consultant.ru/document/cons_doc_LAW_331182/ee42e3ade8001260d90e2aa589c9a4ec243c5a70/ neg_lines <- c("line_2120", "line_2330", "line_2350", "line_2410") dt[, paste0(neg_lines, "_neg") := lapply(.SD, function(x) -x), .SDcols = neg_lines] - dt[, finres_fine_2400 := as.numeric(abs(line_2400 - rowSums(.SD, na.rm = T)) <= 4), .SDcols = c("line_2110", "line_2120_neg", "line_2330_neg", "line_2340", "line_2350_neg", "line_2410_neg")] + dt[, finres_fine_2400 := as.numeric(abs(line_2400 - rowSums(.SD, na.rm = T)) <= 4), .SDcols = c("line_2110", "line_2120", "line_2330", "line_2340", "line_2350", "line_2410")] dt[, paste0(neg_lines, "_neg") := NULL] setnafill(dt, cols = "finres_fine_2400", fill = 1) dt[, finres_fine := finres_fine_2400] @@ -58,13 +58,13 @@ check_cashflow_full <- function(dt) { neg_lines <- c("line_4120", "line_4220", "line_4320") dt[, paste0(neg_lines, "_neg") := lapply(.SD, function(x) -x), .SDcols = neg_lines] - dt[, cashflow_fine_4100 := as.numeric((line_4100 - rowSums(.SD, na.rm = T)) <= 4), .SDcols = c("line_4110", "line_4120_neg")] + dt[, cashflow_fine_4100 := as.numeric((line_4100 - rowSums(.SD, na.rm = T)) <= 4), .SDcols = c("line_4110", "line_4120")] dt[, cashflow_fine_4110 := as.numeric((line_4110 - rowSums(.SD, na.rm = T)) <= 4), .SDcols = patterns("line_411[1234569x]")] dt[, cashflow_fine_4120 := as.numeric((line_4120 - rowSums(.SD, na.rm = T)) <= 4), .SDcols = patterns("line_412[1234569x]")] - dt[, cashflow_fine_4200 := as.numeric((line_4200 - rowSums(.SD, na.rm = T)) <= 4), .SDcols = c("line_4210", "line_4220_neg")] + dt[, cashflow_fine_4200 := as.numeric((line_4200 - rowSums(.SD, na.rm = T)) <= 4), .SDcols = c("line_4210", "line_4220")] dt[, cashflow_fine_4210 := as.numeric((line_4210 - rowSums(.SD, na.rm = T)) <= 4), .SDcols = patterns("line_421[12345679x]")] dt[, cashflow_fine_4220 := as.numeric((line_4220 - rowSums(.SD, na.rm = T)) <= 4), .SDcols = patterns("line_422[12345679x]")] - dt[, cashflow_fine_4300 := as.numeric((line_4300 - rowSums(.SD, na.rm = T)) <= 4), .SDcols = c("line_4310", "line_4320_neg")] + dt[, cashflow_fine_4300 := as.numeric((line_4300 - rowSums(.SD, na.rm = T)) <= 4), .SDcols = c("line_4310", "line_4320")] dt[, cashflow_fine_4310 := as.numeric((line_4310 - rowSums(.SD, na.rm = T)) <= 4), .SDcols = patterns("line_431[12345679x]")] dt[, cashflow_fine_4320 := as.numeric((line_4320 - rowSums(.SD, na.rm = T)) <= 4), .SDcols = patterns("line_432[1234569x]")] dt[, cashflow_fine_4400 := as.numeric((line_4400 - rowSums(.SD, na.rm = T)) <= 4), .SDcols = patterns("line_4[123]00")] diff --git a/code/1_financials/helpers/lines_tags_dict.R b/code/1_financials/helpers/lines_tags_dict.R index 6d2221c..fd23a13 100644 --- a/code/1_financials/helpers/lines_tags_dict.R +++ b/code/1_financials/helpers/lines_tags_dict.R @@ -172,7 +172,7 @@ changes_in_equity_cur_tags <- c( # Value attr == Итог line_3325 = "ДвиженКап/ОтчетГод/УменКапитал/УменКолАкций", # Уменьшение количества акций line_3326 = "ДвиженКап/ОтчетГод/УменКапитал/Реорганизация", # Реорганизация юридического лица line_3327 = "ДвиженКап/ОтчетГод/УменКапитал/Дивиденды", # Дивиденды - line_332x = "ДвиженКап/ПредГод/УменКапитал/ВПокДвижКап", # Вписываемый показатель + line_332x = "ДвиженКап/ПредГод/УменКапитал/ВПокДвижКап", # Дивиденды line_3330 = "ДвиженКап/ОтчетГод/ИзмДобавКап", # !!! Изменение добавочного капитала line_3340 = "ДвиженКап/ОтчетГод/ИзмРезервКап", # !!! Изменение резервного капитала line_3300 = "ДвиженКап/ОтчетГод/Кап31дек" # Величина капитала на 31 декабря отчетного года diff --git a/code/1_financials/helpers/parsing_functions.R b/code/1_financials/helpers/parsing_functions.R index f1c4e87..05a9a73 100644 --- a/code/1_financials/helpers/parsing_functions.R +++ b/code/1_financials/helpers/parsing_functions.R @@ -24,7 +24,7 @@ extr_val <- function(doc, node_name, att) { # Gets values and writes to CSV -parse_xml <- function(xml_path, temp_output_dir) { +parse_xml <- function(xml_path, dir_year, temp_output_dir) { process_id <- Sys.getpid() # to use in file names so that every process writes in its own file @@ -75,14 +75,14 @@ parse_xml <- function(xml_path, temp_output_dir) { designated_use) # Append new row to file on disk - output_file <- glue::glue("{temp_output_dir}/{year}_{process_id}_cur_result.csv") + output_file <- glue::glue("{temp_output_dir}/{dir_year}_{process_id}_cur_result.csv") fwrite(as.data.table(row), output_file, append = file.exists(output_file)) TRUE }, error = function(e) { - failed_xmls <- glue::glue("{temp_output_dir}/{year}_corrupted_xmls_{process_id}.csv") + failed_xmls <- glue::glue("{temp_output_dir}/{dir_year}_corrupted_xmls_{process_id}.csv") fwrite(data.table(path = xml_path), failed_xmls, append = file.exists(failed_xmls)) message("cur year") message(e$message) @@ -142,7 +142,7 @@ parse_xml <- function(xml_path, temp_output_dir) { cashflow_lag1, designated_use_lag1) - output_file <- glue::glue("{temp_output_dir}/{year}_{process_id}_lag1_result.csv") + output_file <- glue::glue("{temp_output_dir}/{dir_year}_{process_id}_lag1_result.csv") fwrite(as.data.table(row_lag1), output_file, append = file.exists(output_file)) }, @@ -168,7 +168,7 @@ parse_xml <- function(xml_path, temp_output_dir) { balance_lag2, list(line_3600 = net_assets_lag2)) - output_file <- glue::glue("{temp_output_dir}/{year}_{process_id}_lag2_result.csv") + output_file <- glue::glue("{temp_output_dir}/{dir_year}_{process_id}_lag2_result.csv") fwrite(as.data.table(row_lag2), output_file, append = file.exists(output_file)) }, @@ -184,7 +184,7 @@ parse_xml <- function(xml_path, temp_output_dir) { ) # Log that this XML has been processed - processed_xmls_log <- glue::glue("{temp_output_dir}/{year}_{process_id}_done.csv") + processed_xmls_log <- glue::glue("{temp_output_dir}/{dir_year}_{process_id}_done.csv") fwrite(data.table(path = xml_path), processed_xmls_log, append = file.exists(processed_xmls_log)) } From 87eaacea20b5259aef9f7f001c5610338bdd8297 Mon Sep 17 00:00:00 2001 From: Sergey Bondarkov Date: Thu, 1 May 2025 13:31:30 +0000 Subject: [PATCH 2/4] upd: fixes a bug in the revised adjustment procedure --- code/1_financials/6_adjust_values.R | 46 +++++++++++------------------ 1 file changed, 18 insertions(+), 28 deletions(-) diff --git a/code/1_financials/6_adjust_values.R b/code/1_financials/6_adjust_values.R index 5762dd4..97cbfc4 100644 --- a/code/1_financials/6_adjust_values.R +++ b/code/1_financials/6_adjust_values.R @@ -8,20 +8,32 @@ russian_financials[, adj_any := 0] # Declare imputation function impute <- function(dt, imp_target, lines_to_sum, flag_imputation = T) { + + # Transform list of lines to regex to be passed to patterns() in .SDcols regex <- paste(lines_to_sum, collapse = "|") - env <- list(imp_target = imp_target, imp_value = paste0(imp_target, "_imp")) - dt[, useless := pmax(rowSums(is.na(.SD)), rowSums(.SD == 0)) == length(lines_to_sum), .SDcols = patterns(regex) ] + # Create environment for DT to address vars without overloading the code with paste0() and get() + env <- list(imp_target = imp_target, imp_value = paste0(imp_target, "_imp")) + + # Calculate new value skipping lines where lines_to_sum are all NA or 0 + dt[, useless := pmax(rowSums(is.na(.SD)), rowSums(.SD == 0), na.rm = T) == length(lines_to_sum), .SDcols = patterns(regex)] dt[useless == F, imp_value := rowSums(.SD, na.rm = T), .SDcols = patterns(regex), env = env] - + + # Save original value to a temp column dt[, orig_value := imp_target, env = env] + + # Always impute if original value is NA dt[is.na(imp_target), imp_target := imp_value, env = env] + + # If original value is present change it to the calculate only where their abs diff is > 4 thousand roubles dt[!is.na(imp_target) & !is.na(imp_value), imp_target := fifelse(abs(imp_target - imp_value) > 4, imp_value, imp_target), env = env] + # If adjustment indicator column was 0 change it to 1 if(flag_imputation == T) { dt[, adj_any := pmax(adj_any, imp_target != orig_value | (!is.na(imp_target) & is.na(orig_value)), na.rm = T), env = env] } - + + # Delete temp columns dt[, imp_value := NULL, env = env] dt[, useless := NULL] dt[, orig_value := NULL] @@ -39,7 +51,7 @@ impute <- function(dt, imp_target, lines_to_sum, flag_imputation = T) { # 4: 1 NA 0 -# Lower levels imputation: sum of XXX[1-9] lines --------------------------- +# Lower levels imputation: sum of XXX[1-9x] lines --------------------------- XXX0_lines_for_simple_imp <- c("line_3210", "line_3220", "line_3310", "line_3320", "line_4110", "line_4120", @@ -76,16 +88,9 @@ russian_financials_simple <- russian_financials[simplified == 1] rm(russian_financials) gc() - ## Full statements -### Prepare some lines -# for(l in c("line_1320", "line_2120", "line_2210", "line_2220", "line_2330", "line_2350", - # "line_3220", "line_3320", "line_4120", "line_4220", "line_4320")) { - # russian_financials_full[, l_neg := -l, env = list(l = l, l_neg = paste0(l, "_neg"))] -# } ### Impute impute(russian_financials_full, "line_1300", c("line_1310", "line_1320", "line_1340", "line_1350", "line_1360", "line_1370")) -# TODO: in NGOs case imputation of line 1300 should be different: line 1320 included as positive, impute(russian_financials_full, "line_1600", c("line_1100", "line_1200")) impute(russian_financials_full, "line_1700", c("line_1300", "line_1400", "line_1500")) impute(russian_financials_full, "line_2100", c("line_2110", "line_2120")) @@ -94,13 +99,7 @@ impute(russian_financials_full, "line_2300", c("line_2200", "line_2310", "line_2 # Impute 24XX -## Imputing 2412 and 2410 is theoretically right but in practice it adds erroneous values: -## 2411, 2412 as well as 2430 and 2450 often contain mistakes which affect -## the otherwise correct 2410 and, in turn, 2400. -# impute(russian_financials_full[year >= 2019], "line_2412", c("line_2430", "line_2450")) -# russian_financials_full[, line_2411 := -line_2411] -# impute(russian_financials_full[year >= 2019], "line_2410", c("line_2411", "line_2412")) - +## Imputing 2412 and 2410 is theoretically right but in practice it adds erroneous values impute(russian_financials_full, "line_2400", c("line_2300", "line_2410", "line_2460")) # # Check @@ -160,10 +159,6 @@ impute(russian_financials_full, "line_4400", c("line_4100", "line_4200", "line_4 impute(russian_financials_full, "line_4500", c("line_4400", "line_4450", "line_4490")) ## Simplified statements -### Prepare some lines -# for(l in c("line_2120", "line_2330", "line_2350", "line_2410")) { - # russian_financials_simple[, l_neg := -l, env = list(l = l, l_neg = paste0(l, "_neg"))] -# } ### Impute impute(russian_financials_simple, "line_1600", c("line_1150", "line_1170", "line_1210", "line_1250", "line_1230")) impute(russian_financials_simple, "line_1700", c("line_1300", "line_1350", "line_1360", "line_1410", "line_1450", "line_1510", "line_1520", "line_1550")) @@ -176,11 +171,6 @@ russian_financials_simple[, line_2500 := line_2400] # Combine data russian_financials <- rbindlist(list(russian_financials_full, russian_financials_simple), use.names=T, fill=T) -# lines_to_delete <- grep("\\d_neg", names(russian_financials), value = T) -# russian_financials[, (lines_to_delete) := NULL] - -# print(russian_financials[, .(adj_any = mean(adj_any)), keyby = year]) -# print(russian_financials[, .N, keyby = year]) # Tidy up and save setorderv(russian_financials, c("inn", "year")) From 90dc230fa6a0e81d2908582f1a7f6f68e0b76524 Mon Sep 17 00:00:00 2001 From: Dmitriy Skougarevskiy Date: Tue, 13 May 2025 12:03:16 +0300 Subject: [PATCH 3/4] Update README.md --- README.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 0d01e96..27bef28 100644 --- a/README.md +++ b/README.md @@ -216,7 +216,8 @@ The below figure explains how we constructed the dataset. An annotated `Makefile ``` ## Version and Update Policy -Version (SemVer): `1.0.0`. +Version (SemVer): `1.0.1`. + We intend to update the RFSD annualy as the data becomes available, in other words when most of the firms have their statements filed with the Federal Tax Service. The official deadline for filing of previous year statements is April, 1. However, every year a portion of firms either fails to meet the deadline or submits corrections afterwards. As the figure below shows, filing continues up to the very end of the year but after the end of April this stream quickly thins out. Nevertheless, there is obviously a trade-off between minimization of data completeness and version availability. We find it a reasonable compromise to query new data in early June, since on average by the end of May 96.7% statements are already filed, including 86.4% of all the correcting filings. We plan to make a new version of RFSD available by July, 2025. @@ -225,6 +226,17 @@ We intend to update the RFSD annualy as the data becomes available, in other wor
+## Changelog + +All notable changes to this project will be documented below. The format is based on [Keep a Changelog](http://keepachangelog.com/). + +## [1.0.1] - 2025-05-13 + +### Fixed + +- Fixed a bug in summation of negative lines when calculating line 2400 (net profit). The bug was identified in #7 and the fix is explained in #8. +- Fixed a bug in adjustment of line 1300 (total capital and reserves) and 2500 (result of the period). See #9 for an explanation. + ## ToDo and Known Deficiencies Below is our To-Do list, we will be grateful for any contributions you can make. If you spot a bug, just raise it as a GitHub issue. From 5061d9d858a2e784c5810ed33f55f134b41124bb Mon Sep 17 00:00:00 2001 From: Dmitriy Skougarevskiy Date: Tue, 13 May 2025 12:13:43 +0300 Subject: [PATCH 4/4] Update README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 27bef28..3255ee6 100644 --- a/README.md +++ b/README.md @@ -237,6 +237,8 @@ All notable changes to this project will be documented below. The format is base - Fixed a bug in summation of negative lines when calculating line 2400 (net profit). The bug was identified in #7 and the fix is explained in #8. - Fixed a bug in adjustment of line 1300 (total capital and reserves) and 2500 (result of the period). See #9 for an explanation. +The updated lines 2400 are quite different from the original values. The value of line 2400 changed in 6-11% of observations in 2011-2018 and in about 25% observations in 2019-2023, the summed difference in the original and new values ranges from 5% to 110% depending on year. The fix for sign inconsistency implies revising scripts for all calculations where negative-only, those ()-ed in statement forms, variables were used. + ## ToDo and Known Deficiencies Below is our To-Do list, we will be grateful for any contributions you can make. If you spot a bug, just raise it as a GitHub issue.