diff --git a/README.md b/README.md index 0d01e96..3255ee6 100644 --- a/README.md +++ b/README.md @@ -216,7 +216,8 @@ The below figure explains how we constructed the dataset. An annotated `Makefile ``` ## Version and Update Policy -Version (SemVer): `1.0.0`. +Version (SemVer): `1.0.1`. + We intend to update the RFSD annualy as the data becomes available, in other words when most of the firms have their statements filed with the Federal Tax Service. The official deadline for filing of previous year statements is April, 1. However, every year a portion of firms either fails to meet the deadline or submits corrections afterwards. As the figure below shows, filing continues up to the very end of the year but after the end of April this stream quickly thins out. Nevertheless, there is obviously a trade-off between minimization of data completeness and version availability. We find it a reasonable compromise to query new data in early June, since on average by the end of May 96.7% statements are already filed, including 86.4% of all the correcting filings. We plan to make a new version of RFSD available by July, 2025. @@ -225,6 +226,19 @@ We intend to update the RFSD annualy as the data becomes available, in other wor
+## Changelog + +All notable changes to this project will be documented below. The format is based on [Keep a Changelog](http://keepachangelog.com/). + +## [1.0.1] - 2025-05-13 + +### Fixed + +- Fixed a bug in summation of negative lines when calculating line 2400 (net profit). The bug was identified in #7 and the fix is explained in #8. +- Fixed a bug in adjustment of line 1300 (total capital and reserves) and 2500 (result of the period). See #9 for an explanation. + +The updated lines 2400 are quite different from the original values. The value of line 2400 changed in 6-11% of observations in 2011-2018 and in about 25% observations in 2019-2023, the summed difference in the original and new values ranges from 5% to 110% depending on year. The fix for sign inconsistency implies revising scripts for all calculations where negative-only, those ()-ed in statement forms, variables were used. + ## ToDo and Known Deficiencies Below is our To-Do list, we will be grateful for any contributions you can make. If you spot a bug, just raise it as a GitHub issue. diff --git a/code/1_financials/4_combine_rosstat_fns_panels.R b/code/1_financials/4_combine_rosstat_fns_panels.R index 471033d..ea9380e 100644 --- a/code/1_financials/4_combine_rosstat_fns_panels.R +++ b/code/1_financials/4_combine_rosstat_fns_panels.R @@ -5,58 +5,81 @@ setDTthreads(0) # Load data ============================================================================== rosstat_financials <- read_fst("temp/rosstat_financials_impFrNY.fst", as.data.table = T) fns_financials <- read_fst("temp/fns_financials_impFrNY.fst", as.data.table = T) -fns_financials_20172018 <- fns_financials[year < 2019] - -rosstat_financials_20172018_new_obs <- fns_financials_20172018[!rosstat_financials, on = c("inn", "year")] - -rosstat_financials <- rbindlist(list( - rosstat_financials, - rosstat_financials_20172018_new_obs - ), - use.names = T, fill = T -) - # Combine Rosstat and GIR BO (BFO) ========================================================== combined_financials <- rbindlist(list( rosstat_financials, + fns_financials[year < 2019][!rosstat_financials, on = c("inn", "year")], fns_financials[year >= 2019] ), fill = T, use.names = T +) +rm(rosstat_financials) +rm(fns_financials) +gc() # Check -print(dcast(combined_financials[, .N, keyby = .(year, all_na)], year ~ paste0("all_na_", all_na))) -print(dcast(combined_financials[, .N, keyby = .(year, new_obs)], year ~ paste0("new_obs_", new_obs))) -print(dcast(combined_financials[, .N, keyby = .(new_obs, all_na)], new_obs ~ paste0("all_na_", all_na))) -print(dcast(combined_financials[, .N, keyby = .(simplified, all_na)], simplified ~ paste0("all_na_", all_na))) -print(combined_financials[, lapply(.SD, mean), .SDcols = patterns("imp_"), keyby = year]) +# print(dcast(combined_financials[, .N, keyby = .(year, all_na)], year ~ paste0("all_na_", all_na))) +# print(dcast(combined_financials[, .N, keyby = .(year, new_obs)], year ~ paste0("new_obs_", new_obs))) +# print(dcast(combined_financials[, .N, keyby = .(new_obs, all_na)], new_obs ~ paste0("all_na_", all_na))) +# print(dcast(combined_financials[, .N, keyby = .(simplified, all_na)], simplified ~ paste0("all_na_", all_na))) +# print(combined_financials[, lapply(.SD, mean), .SDcols = patterns("imp_"), keyby = year]) -# Negative lines values to positive ================================================= +# Bracketed negative lines values to positive ================================================= -neg_lines <- paste0("line_", c(1320:1323, - 2120:2123, - 2210:2213, - 2220:2223, - 2330:2333, - 2350:2353, - 2411, - 3220:3227, - 3320:3327, - 4120:4129, - 4220:4229, - 4320:4329, - 6310:6313, - 6320:6326, - 6330, - 6350:6359, - 6300 - )) +neg_lines_full <- paste0("line_", c( + 1320:1323, + 2120:2123, + 2210:2213, + 2220:2223, + 2330:2333, + 2350:2353, + 2411, + 3220:3227, + 3320:3327, + 4120:4129, + 4220:4229, + 4320:4329, + 6310:6313, + 6320:6326, + 6330, + 6350:6359, + 6300 + )) # Not all the lines are present in data so we repack them as regex -neg_lines_pattern <- paste(neg_lines, collapse="|") -neg_lines_present <- grep(neg_lines_pattern, names(combined_financials), value = T) +neg_lines_pattern_full <- paste(neg_lines_full, collapse="|") +neg_lines_present_full <- grep(neg_lines_pattern_full, names(combined_financials), value = T) +combined_financials[simplified == 0, (neg_lines_present_full) := lapply(.SD, function(l) fifelse(l > 0, -l, l)), .SDcols = neg_lines_present_full] + +combined_financials[simplified == 0 & year < 2019 & line_2410 > 0, line_2410 := -line_2410] +combined_financials[simplified == 0 & year >= 2019 + & line_2410 > 0 + & (abs(line_2411) > line_2412 + | (!is.na(line_2411) & line_2411 != 0 & is.na(line_2412))), + line_2410 := -line_2410] -combined_financials[, (neg_lines_present) := lapply(.SD, function(l) fifelse(l < 0, -l, l)), .SDcols = neg_lines_present] +neg_lines_simple <- paste0("line_", c( + 2120, + 2330, + 2350, + 2410, + 3220:3227, + 3320:3327, + 4120:4129, + 4220:4229, + 4320:4329, + 6310, + 6320, + 6330, + 6350, + 6300 + )) + +# Not all the lines are present in data so we repack them as regex +neg_lines_pattern_simple <- paste(neg_lines_simple, collapse="|") +neg_lines_present_simple <- grep(neg_lines_pattern_simple, names(combined_financials), value = T) +combined_financials[simplified == 1, (neg_lines_present_simple) := lapply(.SD, function(l) fifelse(l > 0, -l, l)), .SDcols = neg_lines_present_simple] # Save setorderv(combined_financials, c("inn", "year")) diff --git a/code/1_financials/5_build_articulation_panel.R b/code/1_financials/5_build_articulation_panel.R index 902c54e..1040289 100644 --- a/code/1_financials/5_build_articulation_panel.R +++ b/code/1_financials/5_build_articulation_panel.R @@ -2,27 +2,29 @@ library(fst) library(data.table) setDTthreads(0) library(ggplot2) -source("financials/helpers/check_articulation_functions.R") +source("code/data_processing/helpers/check_articulation_functions.R") + +articulation_path <- file.path("output", "articulation") +dir.create(articulation_path, recursive = T, showWarnings = F) # Load panel =================================================================== russian_financials <- read_fst("temp/combined_financials_impFrNY_negLinesCorr.fst", as.data.table = T) - russian_financials[, articulation_basic := as.numeric(line_1600 == line_1700)] # Separate checks ================================================================= -rosneft <- russian_financials[inn == "7706107510"] -gazprom <- russian_financials[inn == "7736050003"] - -check_balance_full(rosneft) -check_finres_full(rosneft) -check_cashflow_full(rosneft) -rosneft[, .SD, .SDcols = patterns("year|fine")] - -check_balance_full(gazprom) -check_finres_full(gazprom) -check_cashflow_full(gazprom) -gazprom[, .SD, .SDcols = patterns("year|fine")] +# rosneft <- russian_financials[inn == "7706107510"] +# gazprom <- russian_financials[inn == "7736050003"] +# +# check_balance_full(rosneft) +# check_finres_full(rosneft) +# check_cashflow_full(rosneft) +# rosneft[, .SD, .SDcols = patterns("year|fine")] +# +# check_balance_full(gazprom) +# check_finres_full(gazprom) +# check_cashflow_full(gazprom) +# gazprom[, .SD, .SDcols = patterns("year|fine")] # ================================================================================ russian_financials_full <- russian_financials[simplified == 0] @@ -64,31 +66,31 @@ plot_check_failed_share <- function(dt, check_name) { labs(y = "Failed checks share", y = "") print(pl) +} ## Full statements check_balance_full(russian_financials_full) plot_check_failed_share(russian_financials_full, "balance") -ggsave("output/balance_check_full_impFrNY.pdf", width = 12, height = 7) +ggsave(file.path(articulation_path, "balance_check_full_impFrNY.pdf"), width = 12, height = 7) check_finres_full(russian_financials_full) plot_check_failed_share(russian_financials_full, "finres") -ggsave("output/finres_check_full_impFrNY.pdf", width = 12, height = 7) +ggsave(file.path(articulation_path, "finres_check_full_impFrNY.pdf"), width = 12, height = 7) check_cashflow_full(russian_financials_full) plot_check_failed_share(russian_financials_full, "cashflow") -ggsave("output/cashflow_check_full_impFrNY.pdf", width = 12, height = 7) - +ggsave(file.path(articulation_path, "cashflow_check_full_impFrNY.pdf"), width = 12, height = 7) ## Simplified statements check_balance_simple(russian_financials_simple) plot_check_failed_share(russian_financials_simple, "balance") -ggsave("output/balance_check_simple_impFrNY.pdf", width = 12, height = 7) +ggsave(file.path(articulation_path, "balance_check_simple_impFrNY.pdf"), width = 12, height = 7) check_finres_simple(russian_financials_simple) plot_check_failed_share(russian_financials_simple, "finres") -ggsave("output/finres_check_simple_impFrNY.pdf", width = 12, height = 7) +ggsave(file.path(articulation_path, "finres_check_simple_impFrNY.pdf"), width = 12, height = 7) # Check articulation =============================================================== @@ -99,4 +101,6 @@ russian_financials <- rbindlist(list(russian_financials_full, russian_financials # Save ============================================================================= setorderv(russian_financials, c("inn", "year")) -write_fst(russian_financials[, .(inn, year, new_obs, simplified, articulation, articulation_basic, imp_any_from_future)], "output/articulation_impFrNY.fst") +write_fst(russian_financials[, .(inn, year, new_obs, simplified, + articulation, articulation_basic, imp_any_from_future)], + file.path(articulation_path, "articulation_panel.fst")) diff --git a/code/1_financials/6_adjust_values.R b/code/1_financials/6_adjust_values.R index cf174cb..97cbfc4 100644 --- a/code/1_financials/6_adjust_values.R +++ b/code/1_financials/6_adjust_values.R @@ -5,28 +5,38 @@ setDTthreads(0) # Load data russian_financials <- read_fst("temp/combined_financials_impFrNY_negLinesCorr.fst", as.data.table = T) russian_financials[, adj_any := 0] -# russian_financials[inn == "0100000621", .SD, .SDcols = patterns("line_1")] # Declare imputation function impute <- function(dt, imp_target, lines_to_sum, flag_imputation = T) { + + # Transform list of lines to regex to be passed to patterns() in .SDcols regex <- paste(lines_to_sum, collapse = "|") - env <- list(imp_target = imp_target, imp_value = paste0(imp_target, "_imp")) - dt[, useless := rowSums(is.na(.SD)) == length(lines_to_sum), .SDcols = patterns(regex) ] + # Create environment for DT to address vars without overloading the code with paste0() and get() + env <- list(imp_target = imp_target, imp_value = paste0(imp_target, "_imp")) + + # Calculate new value skipping lines where lines_to_sum are all NA or 0 + dt[, useless := pmax(rowSums(is.na(.SD)), rowSums(.SD == 0), na.rm = T) == length(lines_to_sum), .SDcols = patterns(regex)] dt[useless == F, imp_value := rowSums(.SD, na.rm = T), .SDcols = patterns(regex), env = env] - + + # Save original value to a temp column dt[, orig_value := imp_target, env = env] + + # Always impute if original value is NA dt[is.na(imp_target), imp_target := imp_value, env = env] + + # If original value is present change it to the calculate only where their abs diff is > 4 thousand roubles dt[!is.na(imp_target) & !is.na(imp_value), imp_target := fifelse(abs(imp_target - imp_value) > 4, imp_value, imp_target), env = env] + # If adjustment indicator column was 0 change it to 1 if(flag_imputation == T) { dt[, adj_any := pmax(adj_any, imp_target != orig_value | (!is.na(imp_target) & is.na(orig_value)), na.rm = T), env = env] } - + + # Delete temp columns dt[, imp_value := NULL, env = env] dt[, useless := NULL] dt[, orig_value := NULL] - # return(dt) } # Function demo: @@ -41,33 +51,36 @@ impute <- function(dt, imp_target, lines_to_sum, flag_imputation = T) { # 4: 1 NA 0 -# Lower levels imputation: sum of XXX[1-9] lines --------------------------- +# Lower levels imputation: sum of XXX[1-9x] lines --------------------------- XXX0_lines_for_simple_imp <- c("line_3210", "line_3220", "line_3310", "line_3320", - "line_3410", "line_3420", "line_4110", "line_4120", "line_4210", "line_4220", "line_4310", "line_4320", "line_6310", "line_6320") for(imp_target in XXX0_lines_for_simple_imp) { - regex <- paste0(stringi::stri_sub(imp_target, 1, -2), "[1-9x]") # e.g. "line_1230" > "line_123[1-9]" + regex <- paste0(stringi::stri_sub(imp_target, 1, -2), "[1-9x]") # e.g. "line_1230" > "line_123[1-9x]" lines_to_sum <- grep(regex, names(russian_financials), value = T) - # message(imp_target, " = ", paste(lines_to_sum, collapse = " + ")) impute(russian_financials, imp_target, lines_to_sum) +} # Higher level imputation: sum of XX[1-9]0 lines ----------------------------- -XX00_lines_for_simple_imp <- c("line_1100", "line_1200", - "line_1300", "line_1400", - "line_1500", "line_6200", +XX00_lines_for_simple_imp <- c("line_1100", + "line_1200", + "line_1400", + "line_1500", + "line_6200", "line_6300") for(imp_target in XX00_lines_for_simple_imp) { regex <- paste0(stringi::stri_sub(imp_target, 1, -3), "[1-9]0") # e.g. "line_1200" > "line_12[1-9]0" + lines_to_sum <- grep(regex, names(russian_financials), value = T) + impute(russian_financials, imp_target, lines_to_sum) +} # Imputation using fomulas common to simplified and non-simplifieds --------- -russian_financials[, line_6300_neg := -line_6300] -impute(russian_financials, "line_6400", c("line_6100", "line_6200", "line_6300_neg")) +impute(russian_financials, "line_6400", c("line_6100", "line_6200", "line_6300")) # Imputation using specific formulas for simplifieds and non-simplifieds ---- russian_financials_full <- russian_financials[simplified == 0] @@ -75,102 +88,94 @@ russian_financials_simple <- russian_financials[simplified == 1] rm(russian_financials) gc() - ## Full statements -### Prepare some lines -for(l in c("line_2120", "line_2210", "line_2220", "line_2330", "line_2350", - "line_3220", "line_3320", "line_4120", "line_4220", "line_4320")) { - russian_financials_full[, l_neg := -l, env = list(l = l, l_neg = paste0(l, "_neg"))] ### Impute +impute(russian_financials_full, "line_1300", c("line_1310", "line_1320", "line_1340", "line_1350", "line_1360", "line_1370")) impute(russian_financials_full, "line_1600", c("line_1100", "line_1200")) impute(russian_financials_full, "line_1700", c("line_1300", "line_1400", "line_1500")) -impute(russian_financials_full, "line_2100", c("line_2110", "line_2120_neg")) -impute(russian_financials_full, "line_2200", c("line_2100", "line_2210_neg", "line_2220_neg")) -impute(russian_financials_full, "line_2300", c("line_2200", "line_2310", "line_2320", "line_2330_neg", "line_2340", "line_2350_neg")) - - +impute(russian_financials_full, "line_2100", c("line_2110", "line_2120")) +impute(russian_financials_full, "line_2200", c("line_2100", "line_2210", "line_2220")) +impute(russian_financials_full, "line_2300", c("line_2200", "line_2310", "line_2320", "line_2330", "line_2340", "line_2350")) # Impute 24XX -russian_financials_full[, line_2410_neg := -line_2410] -impute(russian_financials_full[year >= 2020], "line_2410", c("line_2411", "line_2412")) -impute(russian_financials_full, "line_2400", c("line_2300", "line_2410_neg", "line_2460")) - -## Construct 24XX lines with the same meaning across different periods -russian_financials_full[, line_2410_uniform_tax := NA_real_] -russian_financials_full[, line_2411_uniform_tax := NA_real_] -russian_financials_full[, line_2412_uniform_tax := NA_real_] -russian_financials_full[, line_2400_uniform_tax := NA_real_] -russian_financials_full[, line_2500_uniform_tax := NA_real_] - -### Before 2018 -russian_financials_full[year < 2018, line_2411_uniform_tax := line_2410] -impute(russian_financials_full[year < 2018], "line_2410_uniform_tax", c("line_2410_neg", "line_2430", "line_2450"), flag_imputation = F) -impute(russian_financials_full[year < 2020], "line_2412_uniform_tax", c("line_2430", "line_2450"), flag_imputation = F) -russian_financials_full[year < 2018, line_2460_neg := -line_2460] -impute(russian_financials_full[year < 2018], "line_2400_uniform_tax", c("line_2300", "line_2410_uniform_tax", "line_2460_neg"), flag_imputation = F) -# impute(russian_financials_full[year == 2017], "line_2400_uniform_tax", c("line_2300", "line_2410_uniform_tax", "line_2460"), flag_imputation = F) -russian_financials_full[year == 2017, line_2400_uniform_tax_neg_2460 := line_2400] -impute(russian_financials_full[year == 2017], "line_2400_uniform_tax_neg_2460", c("line_2300", "line_2410_uniform_tax", "line_2460_neg"), flag_imputation = F) - - - -### In 2018 -russian_financials_full[year == 2018 & (!is.na(line_2411) | !is.na(line_2412)), line_2411_uniform_tax := line_2411] -russian_financials_full[year == 2018 & (!is.na(line_2411) | !is.na(line_2412)), line_2412_uniform_tax := line_2412] -russian_financials_full[year == 2018 & (!is.na(line_2411) | !is.na(line_2412)), line_2411_neg := -line_2411] -impute(russian_financials_full[year == 2018 & (!is.na(line_2411) | !is.na(line_2412))], "line_2410_uniform_tax", c("line_2411_neg", "line_2412"), flag_imputation = F) - -russian_financials_full[year == 2018 & (is.na(line_2411) & is.na(line_2412)), line_2410_neg := -line_2410] -impute(russian_financials_full[year == 2018 & (is.na(line_2411) & is.na(line_2412))], "line_2410_uniform_tax", c("line_2410_neg", "line_2430", "line_2450")) -russian_financials_full[year == 2018 & (is.na(line_2411) & is.na(line_2412)), line_2411_uniform_tax := line_2410] -impute(russian_financials_full[year == 2018 & (is.na(line_2411) & is.na(line_2412))], "line_2412_uniform_tax", c("line_2430", "line_2450"), flag_imputation = F) -impute(russian_financials_full[year == 2018 & (is.na(line_2411) & is.na(line_2412))], "line_2400_uniform_tax", c("line_2300", "line_2410_uniform_tax", "line_2460"), flag_imputation = F) - -### After 2018 -impute(russian_financials_full[year > 2018], "line_2400_uniform_tax", c("line_2300", "line_2410", "line_2460"), flag_imputation = F) +## Imputing 2412 and 2410 is theoretically right but in practice it adds erroneous values +impute(russian_financials_full, "line_2400", c("line_2300", "line_2410", "line_2460")) + +# # Check +# russian_financials_full[inn == "7703443256", .(year, line_2400, line_2300, line_2410, line_2411, line_2460)] +# russian_financials_full[year >= 2019][inn %in% sample(inn, 5), .(inn, year, line_2400, line_2300, line_2410, line_2411, line_2460)] + +# ## Construct 24XX lines with the same meaning across different periods +# russian_financials_full[, line_2410_uniform_tax := NA_real_] +# russian_financials_full[, line_2411_uniform_tax := NA_real_] +# russian_financials_full[, line_2412_uniform_tax := NA_real_] +# russian_financials_full[, line_2400_uniform_tax := NA_real_] +# russian_financials_full[, line_2500_uniform_tax := NA_real_] +# +# ### Before 2018 +# russian_financials_full[year < 2018, line_2411_uniform_tax := line_2410] +# impute(russian_financials_full[year < 2018], "line_2410_uniform_tax", c("line_2410", "line_2430", "line_2450"), flag_imputation = F) +# impute(russian_financials_full[year < 2020], "line_2412_uniform_tax", c("line_2430", "line_2450"), flag_imputation = F) +# russian_financials_full[year < 2018, line_2460 := -line_2460] +# impute(russian_financials_full[year < 2018], "line_2400_uniform_tax", c("line_2300", "line_2410_uniform_tax", "line_2460"), flag_imputation = F) +# # impute(russian_financials_full[year == 2017], "line_2400_uniform_tax", c("line_2300", "line_2410_uniform_tax", "line_2460"), flag_imputation = F) +# russian_financials_full[year == 2017, line_2400_uniform_tax_neg_2460 := line_2400] +# impute(russian_financials_full[year == 2017], "line_2400_uniform_tax_neg_2460", c("line_2300", "line_2410_uniform_tax", "line_2460"), flag_imputation = F) +# +# +# +# ### In 2018 +# russian_financials_full[year == 2018 & (!is.na(line_2411) | !is.na(line_2412)), line_2411_uniform_tax := line_2411] +# russian_financials_full[year == 2018 & (!is.na(line_2411) | !is.na(line_2412)), line_2412_uniform_tax := line_2412] +# russian_financials_full[year == 2018 & (!is.na(line_2411) | !is.na(line_2412)), line_2411 := -line_2411] +# impute(russian_financials_full[year == 2018 & (!is.na(line_2411) | !is.na(line_2412))], "line_2410_uniform_tax", c("line_2411", "line_2412"), flag_imputation = F) +# +# russian_financials_full[year == 2018 & (is.na(line_2411) & is.na(line_2412)), line_2410 := -line_2410] +# impute(russian_financials_full[year == 2018 & (is.na(line_2411) & is.na(line_2412))], "line_2410_uniform_tax", c("line_2410", "line_2430", "line_2450")) +# russian_financials_full[year == 2018 & (is.na(line_2411) & is.na(line_2412)), line_2411_uniform_tax := line_2410] +# impute(russian_financials_full[year == 2018 & (is.na(line_2411) & is.na(line_2412))], "line_2412_uniform_tax", c("line_2430", "line_2450"), flag_imputation = F) +# impute(russian_financials_full[year == 2018 & (is.na(line_2411) & is.na(line_2412))], "line_2400_uniform_tax", c("line_2300", "line_2410_uniform_tax", "line_2460"), flag_imputation = F) +# +# ### After 2018 +# impute(russian_financials_full[year > 2018], "line_2400_uniform_tax", c("line_2300", "line_2410", "line_2460"), flag_imputation = F) +# # Impute 2500 -impute(russian_financials_full[year < 2019], "line_2500", c("line_2400", "line_2510", "line_2520")) -impute(russian_financials_full[year < 2019], "line_2500_uniform_tax", c("line_2410_uniform_tax", "line_2510", "line_2520"), flag_imputation = F) -impute(russian_financials_full[year >= 2019], "line_2500", c("line_2410", "line_2510", "line_2520", "line_2530")) -impute(russian_financials_full[year >= 2019], "line_2500_uniform_tax", c("line_2410_uniform_tax", "line_2510", "line_2520", "line_2530"), flag_imputation = F) +impute(russian_financials_full, "line_2500", c("line_2400", "line_2510", "line_2520", "line_2530")) +# impute(russian_financials_full[year < 2019], "line_2500_uniform_tax", c("line_2410_uniform_tax", "line_2510", "line_2520"), flag_imputation = F) +# impute(russian_financials_full[year >= 2019], "line_2500_uniform_tax", c("line_2410_uniform_tax", "line_2510", "line_2520", "line_2530"), flag_imputation = F) # Impute 3XXX and 4XXX -impute(russian_financials_full, "line_3230", c("line_3210", "line_3220_neg")) -impute(russian_financials_full, "line_3200", c("line_3100", "line_3210", "line_3220_neg")) -impute(russian_financials_full, "line_3300", c("line_3200", "line_3310", "line_3320_neg")) +impute(russian_financials_full, "line_3230", c("line_3210", "line_3220")) +impute(russian_financials_full, "line_3200", c("line_3100", "line_3210", "line_3220")) +impute(russian_financials_full, "line_3300", c("line_3200", "line_3310", "line_3320")) # line_3400? impute(russian_financials_full, "line_3500", c("line_3400", "line_3410", "line_3420")) impute(russian_financials_full, "line_3600", c("line_1300")) -impute(russian_financials_full, "line_4100", c("line_4110", "line_4120_neg")) -impute(russian_financials_full, "line_4200", c("line_4210", "line_4220_neg")) -impute(russian_financials_full, "line_4300", c("line_4310", "line_4320_neg")) +impute(russian_financials_full, "line_4100", c("line_4110", "line_4120")) +impute(russian_financials_full, "line_4200", c("line_4210", "line_4220")) +impute(russian_financials_full, "line_4300", c("line_4310", "line_4320")) impute(russian_financials_full, "line_4400", c("line_4100", "line_4200", "line_4300")) impute(russian_financials_full, "line_4500", c("line_4400", "line_4450", "line_4490")) ## Simplified statements -for(l in c("line_2120", "line_2330", "line_2350", "line_2410")) { - russian_financials_simple[, l_neg := -l, env = list(l = l, l_neg = paste0(l, "_neg"))] +### Impute impute(russian_financials_simple, "line_1600", c("line_1150", "line_1170", "line_1210", "line_1250", "line_1230")) impute(russian_financials_simple, "line_1700", c("line_1300", "line_1350", "line_1360", "line_1410", "line_1450", "line_1510", "line_1520", "line_1550")) -impute(russian_financials_simple, "line_2200", c("line_2110", "line_2120_neg")) -impute(russian_financials_simple, "line_2300", c("line_2200", "line_2330_neg", "line_2340", "line_2350_neg")) -impute(russian_financials_simple, "line_2400", c("line_2110", "line_2120_neg", "line_2330_neg", "line_2340", "line_2350_neg", "line_2410_neg")) +impute(russian_financials_simple, "line_2200", c("line_2110", "line_2120")) +impute(russian_financials_simple, "line_2300", c("line_2200", "line_2330", "line_2340", "line_2350")) +impute(russian_financials_simple, "line_2400", c("line_2110", "line_2120", "line_2330", "line_2340", "line_2350", "line_2410")) russian_financials_simple[, line_2500 := line_2400] -russian_financials_simple[, line_2500_uniform_tax := line_2500] +# russian_financials_simple[, line_2500_uniform_tax := line_2500] # Combine data russian_financials <- rbindlist(list(russian_financials_full, russian_financials_simple), use.names=T, fill=T) -lines_to_delete <- grep("\\d_neg", names(russian_financials), value = T) -russian_financials[, (lines_to_delete) := NULL] - -print(russian_financials[, .(adj_any = mean(adj_any)), keyby = year]) -print(russian_financials[, .N, keyby = year]) # Tidy up and save setorderv(russian_financials, c("inn", "year")) -write_fst(russian_financials, "output/russian_financials_2011_2023_imp_adj.fst") +maxyear <- last(russian_financials$year) +write_fst(russian_financials, glue::glue("output/russian_financials_2011_{maxyear}.fst")) diff --git a/code/1_financials/helpers/check_articulation_functions.R b/code/1_financials/helpers/check_articulation_functions.R index 4966445..02c25ee 100644 --- a/code/1_financials/helpers/check_articulation_functions.R +++ b/code/1_financials/helpers/check_articulation_functions.R @@ -31,9 +31,9 @@ check_finres_full <- function(dt) { # conditions from https://www.consultant.ru/document/cons_doc_LAW_331182/dc2cbdd712256af48282919fd9edef3d1cec9f65/ neg_lines <- c("line_2120", "line_2210", "line_2220", "line_2330", "line_2350") dt[, paste0(neg_lines, "_neg") := lapply(.SD, function(x) -x), .SDcols = neg_lines] - dt[, finres_fine_2100 := as.numeric(abs(line_2100 - rowSums(.SD, na.rm = T)) <= 4), .SDcols = c("line_2110", "line_2120_neg")] - dt[, finres_fine_2200 := as.numeric(abs(line_2200 - rowSums(.SD, na.rm = T)) <= 4), .SDcols = c("line_2100", "line_2210_neg", "line_2220_neg")] - dt[, finres_fine_2300 := as.numeric(abs(line_2300 - rowSums(.SD, na.rm = T)) <= 4), .SDcols = c("line_2200", "line_2310", "line_2320", "line_2330_neg", "line_2340", "line_2350_neg")] + dt[, finres_fine_2100 := as.numeric(abs(line_2100 - rowSums(.SD, na.rm = T)) <= 4), .SDcols = c("line_2110", "line_2120")] + dt[, finres_fine_2200 := as.numeric(abs(line_2200 - rowSums(.SD, na.rm = T)) <= 4), .SDcols = c("line_2100", "line_2210", "line_2220")] + dt[, finres_fine_2300 := as.numeric(abs(line_2300 - rowSums(.SD, na.rm = T)) <= 4), .SDcols = c("line_2200", "line_2310", "line_2320", "line_2330", "line_2340", "line_2350")] dt[, paste0(neg_lines, "_neg") := NULL] check_cols <- grep("finres_fine_", names(dt), value = T) setnafill(dt, cols = check_cols, fill = 1) @@ -46,7 +46,7 @@ check_finres_simple <- function(dt) { # conditions from https://www.consultant.ru/document/cons_doc_LAW_331182/ee42e3ade8001260d90e2aa589c9a4ec243c5a70/ neg_lines <- c("line_2120", "line_2330", "line_2350", "line_2410") dt[, paste0(neg_lines, "_neg") := lapply(.SD, function(x) -x), .SDcols = neg_lines] - dt[, finres_fine_2400 := as.numeric(abs(line_2400 - rowSums(.SD, na.rm = T)) <= 4), .SDcols = c("line_2110", "line_2120_neg", "line_2330_neg", "line_2340", "line_2350_neg", "line_2410_neg")] + dt[, finres_fine_2400 := as.numeric(abs(line_2400 - rowSums(.SD, na.rm = T)) <= 4), .SDcols = c("line_2110", "line_2120", "line_2330", "line_2340", "line_2350", "line_2410")] dt[, paste0(neg_lines, "_neg") := NULL] setnafill(dt, cols = "finres_fine_2400", fill = 1) dt[, finres_fine := finres_fine_2400] @@ -58,13 +58,13 @@ check_cashflow_full <- function(dt) { neg_lines <- c("line_4120", "line_4220", "line_4320") dt[, paste0(neg_lines, "_neg") := lapply(.SD, function(x) -x), .SDcols = neg_lines] - dt[, cashflow_fine_4100 := as.numeric((line_4100 - rowSums(.SD, na.rm = T)) <= 4), .SDcols = c("line_4110", "line_4120_neg")] + dt[, cashflow_fine_4100 := as.numeric((line_4100 - rowSums(.SD, na.rm = T)) <= 4), .SDcols = c("line_4110", "line_4120")] dt[, cashflow_fine_4110 := as.numeric((line_4110 - rowSums(.SD, na.rm = T)) <= 4), .SDcols = patterns("line_411[1234569x]")] dt[, cashflow_fine_4120 := as.numeric((line_4120 - rowSums(.SD, na.rm = T)) <= 4), .SDcols = patterns("line_412[1234569x]")] - dt[, cashflow_fine_4200 := as.numeric((line_4200 - rowSums(.SD, na.rm = T)) <= 4), .SDcols = c("line_4210", "line_4220_neg")] + dt[, cashflow_fine_4200 := as.numeric((line_4200 - rowSums(.SD, na.rm = T)) <= 4), .SDcols = c("line_4210", "line_4220")] dt[, cashflow_fine_4210 := as.numeric((line_4210 - rowSums(.SD, na.rm = T)) <= 4), .SDcols = patterns("line_421[12345679x]")] dt[, cashflow_fine_4220 := as.numeric((line_4220 - rowSums(.SD, na.rm = T)) <= 4), .SDcols = patterns("line_422[12345679x]")] - dt[, cashflow_fine_4300 := as.numeric((line_4300 - rowSums(.SD, na.rm = T)) <= 4), .SDcols = c("line_4310", "line_4320_neg")] + dt[, cashflow_fine_4300 := as.numeric((line_4300 - rowSums(.SD, na.rm = T)) <= 4), .SDcols = c("line_4310", "line_4320")] dt[, cashflow_fine_4310 := as.numeric((line_4310 - rowSums(.SD, na.rm = T)) <= 4), .SDcols = patterns("line_431[12345679x]")] dt[, cashflow_fine_4320 := as.numeric((line_4320 - rowSums(.SD, na.rm = T)) <= 4), .SDcols = patterns("line_432[1234569x]")] dt[, cashflow_fine_4400 := as.numeric((line_4400 - rowSums(.SD, na.rm = T)) <= 4), .SDcols = patterns("line_4[123]00")] diff --git a/code/1_financials/helpers/lines_tags_dict.R b/code/1_financials/helpers/lines_tags_dict.R index 6d2221c..fd23a13 100644 --- a/code/1_financials/helpers/lines_tags_dict.R +++ b/code/1_financials/helpers/lines_tags_dict.R @@ -172,7 +172,7 @@ changes_in_equity_cur_tags <- c( # Value attr == Итог line_3325 = "ДвиженКап/ОтчетГод/УменКапитал/УменКолАкций", # Уменьшение количества акций line_3326 = "ДвиженКап/ОтчетГод/УменКапитал/Реорганизация", # Реорганизация юридического лица line_3327 = "ДвиженКап/ОтчетГод/УменКапитал/Дивиденды", # Дивиденды - line_332x = "ДвиженКап/ПредГод/УменКапитал/ВПокДвижКап", # Вписываемый показатель + line_332x = "ДвиженКап/ПредГод/УменКапитал/ВПокДвижКап", # Дивиденды line_3330 = "ДвиженКап/ОтчетГод/ИзмДобавКап", # !!! Изменение добавочного капитала line_3340 = "ДвиженКап/ОтчетГод/ИзмРезервКап", # !!! Изменение резервного капитала line_3300 = "ДвиженКап/ОтчетГод/Кап31дек" # Величина капитала на 31 декабря отчетного года diff --git a/code/1_financials/helpers/parsing_functions.R b/code/1_financials/helpers/parsing_functions.R index f1c4e87..05a9a73 100644 --- a/code/1_financials/helpers/parsing_functions.R +++ b/code/1_financials/helpers/parsing_functions.R @@ -24,7 +24,7 @@ extr_val <- function(doc, node_name, att) { # Gets values and writes to CSV -parse_xml <- function(xml_path, temp_output_dir) { +parse_xml <- function(xml_path, dir_year, temp_output_dir) { process_id <- Sys.getpid() # to use in file names so that every process writes in its own file @@ -75,14 +75,14 @@ parse_xml <- function(xml_path, temp_output_dir) { designated_use) # Append new row to file on disk - output_file <- glue::glue("{temp_output_dir}/{year}_{process_id}_cur_result.csv") + output_file <- glue::glue("{temp_output_dir}/{dir_year}_{process_id}_cur_result.csv") fwrite(as.data.table(row), output_file, append = file.exists(output_file)) TRUE }, error = function(e) { - failed_xmls <- glue::glue("{temp_output_dir}/{year}_corrupted_xmls_{process_id}.csv") + failed_xmls <- glue::glue("{temp_output_dir}/{dir_year}_corrupted_xmls_{process_id}.csv") fwrite(data.table(path = xml_path), failed_xmls, append = file.exists(failed_xmls)) message("cur year") message(e$message) @@ -142,7 +142,7 @@ parse_xml <- function(xml_path, temp_output_dir) { cashflow_lag1, designated_use_lag1) - output_file <- glue::glue("{temp_output_dir}/{year}_{process_id}_lag1_result.csv") + output_file <- glue::glue("{temp_output_dir}/{dir_year}_{process_id}_lag1_result.csv") fwrite(as.data.table(row_lag1), output_file, append = file.exists(output_file)) }, @@ -168,7 +168,7 @@ parse_xml <- function(xml_path, temp_output_dir) { balance_lag2, list(line_3600 = net_assets_lag2)) - output_file <- glue::glue("{temp_output_dir}/{year}_{process_id}_lag2_result.csv") + output_file <- glue::glue("{temp_output_dir}/{dir_year}_{process_id}_lag2_result.csv") fwrite(as.data.table(row_lag2), output_file, append = file.exists(output_file)) }, @@ -184,7 +184,7 @@ parse_xml <- function(xml_path, temp_output_dir) { ) # Log that this XML has been processed - processed_xmls_log <- glue::glue("{temp_output_dir}/{year}_{process_id}_done.csv") + processed_xmls_log <- glue::glue("{temp_output_dir}/{dir_year}_{process_id}_done.csv") fwrite(data.table(path = xml_path), processed_xmls_log, append = file.exists(processed_xmls_log)) }