From 12c1f1a9cb40aea4a949fed5e04bd578eab582a3 Mon Sep 17 00:00:00 2001 From: Ashley Burdett <76621383+ANBurdett@users.noreply.github.com> Date: Wed, 8 Jul 2026 18:07:05 +0100 Subject: [PATCH 1/5] Added SILC panel constrcution files and READMEs --- input_processing/data_construction/.DS_Store | Bin 0 -> 6148 bytes .../README_data_construction.txt | 15 + .../2005_2023_panel/.DS_Store | Bin 0 -> 10244 bytes .../2005_2023_panel/do_files/.DS_Store | Bin 0 -> 6148 bytes .../00_master_construct_eusilc_panel.do | 112 + .../do_files/01_create_masterD.do | 2819 +++++++++++++++++ .../do_files/02_create_masterH.do | 711 +++++ .../do_files/03_create_masterR.do | 869 +++++ .../do_files/04_create_masterP.do | 758 +++++ .../2005_2023_panel/do_files/05_weights.do | 270 ++ .../README_SILC_panel_construction.txt | 33 + .../UID_construction/.DS_Store | Bin 0 -> 6148 bytes .../PL/00_master_data_set_construction_PL.do | 3 +- .../PL/01_prepare_pooled_data_PL.do | 0 .../PL/02_create_variables_PL.do | 0 .../PL/03_create_benefit_units_PL.do | 0 .../PL/04_reweight_PL.do | 0 ...05_drop_hholds_slice_and_refactoring_PL.do | 0 .../PL/06_check_yearly_data_PL.do | 0 .../PL/07_version_comparison_PL.do | 0 .../prepare_pooled_data _05_20.do | 0 .../PL/extra_var_info/vars_05_20_PL2.do | 0 .../PL/00_master_regression_estimates_PL.do | 4 +- 23 files changed, 5592 insertions(+), 2 deletions(-) create mode 100644 input_processing/data_construction/.DS_Store create mode 100644 input_processing/data_construction/README_data_construction.txt create mode 100644 input_processing/data_construction/SILC_panel_construction/2005_2023_panel/.DS_Store create mode 100644 input_processing/data_construction/SILC_panel_construction/2005_2023_panel/do_files/.DS_Store create mode 100644 input_processing/data_construction/SILC_panel_construction/2005_2023_panel/do_files/00_master_construct_eusilc_panel.do create mode 100644 input_processing/data_construction/SILC_panel_construction/2005_2023_panel/do_files/01_create_masterD.do create mode 100644 input_processing/data_construction/SILC_panel_construction/2005_2023_panel/do_files/02_create_masterH.do create mode 100644 input_processing/data_construction/SILC_panel_construction/2005_2023_panel/do_files/03_create_masterR.do create mode 100644 input_processing/data_construction/SILC_panel_construction/2005_2023_panel/do_files/04_create_masterP.do create mode 100644 input_processing/data_construction/SILC_panel_construction/2005_2023_panel/do_files/05_weights.do create mode 100644 input_processing/data_construction/SILC_panel_construction/README_SILC_panel_construction.txt create mode 100644 input_processing/data_construction/UID_construction/.DS_Store rename input_processing/data_construction/{ => UID_construction}/PL/00_master_data_set_construction_PL.do (95%) rename input_processing/data_construction/{ => UID_construction}/PL/01_prepare_pooled_data_PL.do (100%) rename input_processing/data_construction/{ => UID_construction}/PL/02_create_variables_PL.do (100%) rename input_processing/data_construction/{ => UID_construction}/PL/03_create_benefit_units_PL.do (100%) rename input_processing/data_construction/{ => UID_construction}/PL/04_reweight_PL.do (100%) rename input_processing/data_construction/{ => UID_construction}/PL/05_drop_hholds_slice_and_refactoring_PL.do (100%) rename input_processing/data_construction/{ => UID_construction}/PL/06_check_yearly_data_PL.do (100%) rename input_processing/data_construction/{ => UID_construction}/PL/07_version_comparison_PL.do (100%) rename input_processing/data_construction/{ => UID_construction}/PL/extra_var_info/prepare_pooled_data _05_20.do (100%) rename input_processing/data_construction/{ => UID_construction}/PL/extra_var_info/vars_05_20_PL2.do (100%) diff --git a/input_processing/data_construction/.DS_Store b/input_processing/data_construction/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..47fd5903a1c169446bc93b17c382f20bf1e840bd GIT binary patch literal 6148 zcmeHK!AiqG5Pe&FhG=6(M z>;4{PHT#BB7Wb`e1{jl%lx*zIz?U)2oafJnr<(H)aUWTBiQAG_X9doB`Hs zk?0_xjmCg6U<|AoknclG6-*TCfd13L!M6a!KFwY@mtI0*V#P$U4#*vf3!%gi>a@jh zA)NMD{Sw7GUR!G6sx+ zKgEFSr&&5;N?Kc6$w{q^sn=8y@v8%_LpbrJn7&eqPpDpKk7Ytk6zhPrQ0zy*(_n)! I@S_a81L9wE5C8xG literal 0 HcmV?d00001 diff --git a/input_processing/data_construction/README_data_construction.txt b/input_processing/data_construction/README_data_construction.txt new file mode 100644 index 0000000..0e49b56 --- /dev/null +++ b/input_processing/data_construction/README_data_construction.txt @@ -0,0 +1,15 @@ +Construction of the SimPaths Input Datasets + +Date: 6/6/26 +Author: A Burdett + +Workflow summary: + +1. Obtain the required longitudinal EU-SILC releases. + +2. Run the GESIS compilation program using the earlier longitudinal release to produce the harmonised 2005-2020 D, H, R, and P datasets. + +3. Run the extended version of the GESIS compilation program using the 2024 longitudinal release to produce the harmonised 2005-2023 D, H, R, and P datasets. (Compilation files contained in …./data_construction/SILC_construction/2005_2023_panel/.) + +4. Run the SimPaths UID construction files for the country of interest, contained in the ”data_construction/UID_construction/” folder. During this process, the variables affected by post-2020 coding changes are merged from the 2005-2020 dataset into the 2005-2023 dataset. This creates the UID file (“…ipop.dta”) which is used to both process estimation and for internal validation of SimPaths, as well as the annual cross-sections used to initialise the model. + diff --git a/input_processing/data_construction/SILC_panel_construction/2005_2023_panel/.DS_Store b/input_processing/data_construction/SILC_panel_construction/2005_2023_panel/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..ba5594226f8225c059fee837924cbd47a67d1021 GIT binary patch literal 10244 zcmeHMU2GIp6uxJ6u`>+NDFOv{k*$RYO@S5&Ey~aCwt!HerMu8=p<} zx%d3sd(XG$+}nFg2!Uu`X(l8@2;uRgRGNj=6oty^S5Y{ZBOwK}C;DJ>ENy186`CHy zL%>78L%>78L%>7edO-lsY+87cQLpw8@DT727)OBa46z5&@bnexeYW*hDVBzlUi`jmt7-dwWVZQ^@0!hOg4o!R; z=t5J>c!iiGNsbVW=tLz}YQ)OAbDU3Qb>pd^b6vf$rL~q~0L;Xof%3Er;s>`-kP}^l&wUSb`g<$MV zn?_eoEg%RzX;aT<43yNQWj&ooWi_JfG3_|hPUlQr?=mgT)=Z-}VXH@NDx(9Hi4uL5 znX}uR!4evd&$Q;!IXbh{*gBx*ETn?)(wy3*%T}yf+p;OLeb?UZOVv`%+#BbKVjtpc z*xC_I&Fvr26nm&ww$iGhXvSbqL9?`!uEq^Hqt^I+f^cc!qQ%vLVBn^bXtD_})$@sb zO1m?!*~gqYLUjT0kb(~GF%>$CBAw+0N3y&w5I9IBZ8;Tdh36gi%7%a#x6oZlb`@8z z5ykj028d+g66nI$trr9PbvdQ#k|Hp(){SDIr@*|GI$Y-FEn*;nuU;8+(W-$)mmO^j z1P18LyJgGn%&G>i>V0?m9VqLwhUA>uIf60jayyXg{Eo<{M%xt4HmT47q%N?J=X>dZ z5k*mzOVx`OFO0>hg`^n;4RZp20+t%mMV{yDO95FQZeY#&chW_tJ1B!*MtPV{j7gfirMFJOB^Eqwpj=1y92>@G`s( zZ^B#fHe7&@;WAu-tMDa!2j9aF@C!GS6S-O3d~N|(%hhoW+-7bo7v>^dlc4d2_%)dX%*aEhkIq}ybIpGb63aiJ=bmCU7f=+ZT^DV(DH^g>svQR z$0L%3a4Bjh=7N&|WohFifD;{)BA13>Ah<+adaaz?kC~$$ox{g^O&U)=XAy9;5q?wQSz~FW3WZwC19Of#Z3ZcrQ~Pbtztx=8-+xi zey}~boyW|Q{;`n%fLtV3$QR^W4EaC62Rzhbn6H4<&=hM7M_O};6*qGufZE|-c5jy zF#JD*uO@}`-tv%!pQj4xiH9{I#pk;IbPnr^(9f8EAKKIDl^x!`B3(koScO>5&`o0y zWi3%=M!o)W)z$YL!R(3vs z=d*ji2Ji~kElyT9jIVy1H&AVbO{TDCv>*Ye(f!PYL$fHU9> ztPBIJnRYf;CSy<38E^)c#ejSt0#qgfg!a3z&6ZDFqd9JVuE327!|PsVGRXp zC|irc8V-9fzsxWyYB;eqA8aSHbts%p$NnL?6K9G(Is?u?oq;vI>`MJVUEKfIgZ#=F za0dPr1KiFBc^|iAwY70`QfmYB94aDyQSl=LlT?b~E2VfFngsSB6JTZ-6=8w+k3gcq K2WMcZ47>xBi&jVg literal 0 HcmV?d00001 diff --git a/input_processing/data_construction/SILC_panel_construction/2005_2023_panel/do_files/00_master_construct_eusilc_panel.do b/input_processing/data_construction/SILC_panel_construction/2005_2023_panel/do_files/00_master_construct_eusilc_panel.do new file mode 100644 index 0000000..f007f64 --- /dev/null +++ b/input_processing/data_construction/SILC_panel_construction/2005_2023_panel/do_files/00_master_construct_eusilc_panel.do @@ -0,0 +1,112 @@ +******************************************************************************** +* PROJECT: SimPaths EU +* DO-FILE NAME: 00_master_construct_eusilc_panel +* DESCRIPTION: Imports the EU-SILC csv files. +******************************************************************************** +* COUNTRY: - +* DATA: EU-SILC longitudinal data +* AUTHORS: Ashley Burdett +* LAST UPDATE: Jan 2025 +******************************************************************************** +* NOTE: This is a formatted, extended (to include 2023 data) +* and organized version of the procedure constrcuted by +* GESIS to combine the SILC longitudinal data panels in a +* systematic way to idenitfy individuals across +* observations. +* +* In addition to the included do-files you will need to +* obtain the set-up files that transform csv files +* containing the SILC data from GESIS. They can be found +* here: https://www.gesis.org/en/missy/materials/EU-SILC/setups#:~:text=Routines%20to%20transform%20*.csv%20to,document%20of%20every%20data%20release. +* +* The version of the set-up files used should coincide +* with the data release being use. +* +* Population files merged added in "05_weights" need to be +* uploaded into the input_data subfolder. The 2005-2020 +* information was obtained from GESIS, whilst the +* 2021-2023 information was obtained from EUROSTAT. +* +* Make sure the necessary subfolders are created for the +* set-up file outputs. +* Currently called in do-file 01-04 using the following +* pattern +* ${do_dir}/EU-SILC/19 L-2023/ +* ${do_dir}/EU-SILC/18 L-2022/ +* ${do_dir}/EU-SILC/17 L-2021/ ... +* +******************************************************************************** + +* Initalization +clear all +capture log close +set more off + + +/******************************************************************************* +* DEFINE DIRECTORIES +*******************************************************************************/ + +global main_dir "/Users/aburdett/Library/CloudStorage/Box-Box/ESPON - OVERLAP/_countries/Cumulative Longitudional Dataset (all countries)/2005_2023_panel" // master folder for processing + +global log_dir "$main_dir/logs" // folder in main_dir that contains logs + +global do_files_dir "$main_dir/do_files" // folder in main_dir that contains do files + +global data_dir "$main_dir/data" // folder in main_dir that contains output data + +global input_data_dir "$main_dir/input_data" + +//for set-up files +global do_dir "$main_dir/data" // folder in main_dir that contains output data for merging do file + +global datapath "$main_dir/data" // folder in main_dir that contains output data for merging do file + +global csv_path "$main_dir/input_data" + + +/******************************************************************************* +* EXECUTE FILES +*******************************************************************************/ +* Call set-up files to import the EU-SILC data +forvalues i = 2005/2023 { + + do "$do_files_dir/Long_allcountries_Dfiles_`i'.do" +} + +forvalues i = 2005/2023 { + + do "$do_files_dir/Long_allcountries_Hfiles_`i'.do" +} + + +forvalues i = 2005/2023 { + + do "$do_files_dir/Long_allcountries_Pfiles_`i'.do" +} + + +forvalues i = 2005/2023 { + + do "$do_files_dir/Long_allcountries_Rfiles_`i'.do" +} +*/ + +* Create panel using D files +do "$do_files_dir/01_create_masterD.do" + + +* Create panel using H files +do "$do_files_dir/02_create_masterH.do" + + +* Create panel using R files +do "$do_files_dir/03_create_masterR.do" + + +* Create panel using P files +do "$do_files_dir/04_create_masterP.do" + + +* Adjust weight variables +do "$do_files_dir/05_weights.do" diff --git a/input_processing/data_construction/SILC_panel_construction/2005_2023_panel/do_files/01_create_masterD.do b/input_processing/data_construction/SILC_panel_construction/2005_2023_panel/do_files/01_create_masterD.do new file mode 100644 index 0000000..f580e55 --- /dev/null +++ b/input_processing/data_construction/SILC_panel_construction/2005_2023_panel/do_files/01_create_masterD.do @@ -0,0 +1,2819 @@ +/******************************************************************************* +SUMMARY — 01_create_masterD.do (2024 release, extended to 2023) + +This file is based on the program from the above. It extends the logic beyond +2020 and no longer uses a program but a series of do files called in the master. +This was only to facilitate understanding of the code logic. + +This file builds the master household register (D) panel for EU-SILC 2005–2023. +Unlike the H, R, and P files, the masterD is assembled incrementally: each +annual block appends its records directly into masterD.dta building the sample +rather than saving intermediate files that are merged at the end. + +NOTE: The data in each release comes with all of the historic data for each +current individual. Therefore there will be repeated observations across +annual datasets if the individual has been observed more than once. This +requires a procedure to identify and drop the duplicates. here the logic is to +take the most recent data release that contains all avaialble information for +individuals currently in the survey. Then moving backwards in time, add data +from the rotation group that is just about to exit the survey, i.e. the rotation +group with the longest individual panels. By doing this each year you avoid +adding duplicates. + +The file works as follows: + +2023 block (master year): + Reads all rotation groups. All groups are selected + (no filtering). Builds unique household (uhid) and rotation group (urtgrp) + identifiers that encode country, rotation group number, planned dropout year, + and household ID. Saves masterD.dta and 2023D.dta to the 19 L-2023 folder. + +2022 block (and all earlier blocks): + Reads the raw D data for the release year. + Counts rotation groups (nrtgrpYYYY) and identifies the longest rotation + groups (lgstgrp). + Builds uhid/urtgrp using the same logic as the 2023 block. + Merges against 2023D.dta (and for older blocks, against additional more + recent release files) to isolate records not already captured in a later + release (keep if _merge == 1). + Updates drpout_year, urtgrp, and uhid to be consistent with the most recent + release conventions. + Runs balance and duplicate checks to catch cross-release inconsistencies + (e.g. rotation groups changing numbers across releases). + Merges the surviving records into masterD.dta and saves both the updated + masterD.dta and a YYYYD.dta for use by downstream files (02, 03, 04). + +Rolling window rule: + Each block merges against the three releases immediately following it, + consistent with the 4-year rotation design (a group can extend at most 3 + years beyond any given release). + +Output: + masterD.dta — full panel 2005–2023, saved in 20 L-2023 + YYYYD.dta — one file per release year, saved in the corresponding release + folder. + Used by 02_create_masterH.do, 03_create_masterR.do, + 04_create_masterP.do to select the correct sample for the H, + R, and P files. +*/ + + +/* D-FILES */ + + +/* 2023 */ + /* open the 2023 Household register to get list of rotation groups with max obs and starting point for masterfile */ + clear + set more off + local filen : dir "${do_dir}/EU-SILC/19 L-2023/" files "udb_l23d_ver*" + display `filen' + local filel "${datapath}/EU-SILC/19 L-2023/" + display `"`filel'"' + local fileln : display `"`filel'"' `filen' + + use `"`fileln'"', clear + + local new_var = lower("`var'") + + foreach var of varlist _all { + local new_var = lower("`var'") + cap rename `var' `new_var' + } + + /* block Stata from displaying IDs in exponential format */ + tostring db030, replace + gen year = db010 + gen country = db020 + gen hid = db030 + gen rotation_group = db075 + replace country = "EL" if country == "GR" + sort country hid year + + /* count the number of rotational groups for each country in this release */ + bysort country rotation_group: gen nvals = _n == 1 + bysort country : egen nrtgrp2023 = total(nvals) + drop nvals + + /* mark selected rotation group (this is the most recent release, so all rotationgroups are selected) */ + gen slctd_rtgrp = rotation_group + + /* build household and rotationgroup IDs, unique across releases. contains country, rotation group, drouput year and hid */ + tostring rotation_group, generate(rotation_groupstr) + egen maxyear = max(year) + bysort country rotation_group : egen minyear = min(year) + gen years_cov = maxyear - minyear + 1 + gen drpout_year = maxyear + replace drpout_year = drpout_year + 4 - years_cov + + /* check for rotation groups that ended earlier than the latest year*/ + bysort country rotation_group : egen maxyear_grp = max(year) + replace drpout_year = maxyear_grp if maxyear_grp < maxyear + + tostring drpout_year, replace + gen uhid = country + rotation_groupstr + drpout_year + hid + gen urtgrp = country + rotation_groupstr + drpout_year + drop rotation_groupstr maxyear_grp minyear years_cov maxyear_grp maxyear + destring drpout_year, replace + + /* for later checks */ + gen drpout_year2023 = drpout_year + gen slctd_urtgrp2023 = urtgrp + gen slctd_uhid2023 = uhid + + /* this is the masterfile to build the dataset on */ + gen merge2023 = 1 + save "${datapath}/EU-SILC/19 L-2023/masterD.dta", replace + /* this is the the 2023D file for later control.*/ + drop merge2023 + save "${datapath}/EU-SILC/19 L-2023/2023D.dta", replace + + +/* 2022 */ + /* open the 2022 Household register to get data from rotational groups that are inactive in 2020 and their uhid */ + clear + local filen : dir "${datapath}/EU-SILC/18 L-2022/" files "udb_l22d_ver*" + display `filen' + local filel "${datapath}/EU-SILC/18 L-2022/" + display `"`filel'"' + local fileln : display `"`filel'"' `filen' + + use `"`fileln'"', clear + + local new_var = lower("`var'") + + foreach var of varlist _all { + local new_var = lower("`var'") + cap rename `var' `new_var' + } + + /* block Stata from displaying IDs in exponential format */ + tostring db030, replace + gen year = db010 + gen country = db020 + gen hid = db030 + gen rotation_group = db075 + sort country hid year + replace country = "EL" if country == "GR" + + /* count the number of rotational groups for each country in this release */ + bysort country rotation_group: gen nvals = _n == 1 + bysort country : egen nrtgrp2022 = total(nvals) + drop nvals + + /* get the rotation group(s) that cover most years - rotationl group to be added to create panel */ + bysort country rotation_group : egen maxyear = max(year) + bysort country rotation_group : egen minyear = min(year) + gen years_cov = maxyear - minyear + 1 + bysort country : egen maxgrp = max(years_cov) + gen slctd_rtgrp = 0 + bysort country rotation_group: replace slctd_rtgrp = rotation_group if years_cov == maxgrp + gen lgstgrp = 0 + bysort country rotation_group: replace lgstgrp = 1 if years_cov == maxgrp + + /* build household and rotationgroup IDs that are unique across releases */ + tostring rotation_group, generate(rotation_groupstr) + gen drpout_year = 0 + replace drpout_year = maxyear + 4 - years_cov + + /* check for rotation groups that dropped out before planned */ + bysort country rotation_group : egen maxyear_grp = max(year) + replace drpout_year = maxyear_grp if maxyear_grp < maxyear + + tostring drpout_year, replace + gen uhid = country + rotation_groupstr + drpout_year + hid + gen urtgrp = country + rotation_groupstr + drpout_year + destring drpout_year, replace + + /* make sure all groups not observed in later releases are included. this accounts for daps in the data releases */ + sort year country rotation_group hid + merge m:m year country rotation_group using "${datapath}/EU-SILC/19 L-2023/2023D.dta" //merging on all 3 variables permits the inclusion of extended panels + replace slctd_rtgrp = rotation_group if _merge == 1 + + /* update urtgrp and uhid to account for later drop out year, allows for extended rotation panels */ + drop maxyear rotation_groupstr + bysort country rotation_group: egen maxyear = max(drpout_year) + replace drpout_year = maxyear + tostring drpout_year, replace + tostring rotation_group, generate(rotation_groupstr) + replace uhid = country + rotation_groupstr + drpout_year + hid + replace urtgrp = country + rotation_groupstr + drpout_year + + /* preserve observations to be appended (observations not found in later data releases whether from new households or due to extended panel) */ + keep if _merge == 1 + drop rotation_groupstr + drop _merge maxyear_grp minyear years_cov maxyear_grp maxyear + destring drpout_year, replace + drop if slctd_rtgrp == 0 + + /* for later checks */ + gen drpout_year2022 = drpout_year + gen slctd_urtgrp2022 = urtgrp + gen slctd_uhid2022 = uhid + + /* merge to masterfile */ + merge 1:1 year uhid using "${datapath}/EU-SILC/19 L-2023/masterD.dta" + + /* check for duplicates caused by same rotational groups changing numbers across releases - in same hid, different rotation_group. Catches the LV cases 2020-2019.*/ + duplicates report year country hid + duplicates tag year country hid, generate(test2) + tab country year if test2 != 0 + tab urtgrp year if test2 != 0 + + /* check if the problem regards a large portion of the rotational group, if the portion is small ignore */ + gen tag1 = 0 + replace tag1 = 1 if test2 != 0 + bysort year country urtgrp : egen ndups = total(tag1) + gen tag2 = 1 + bysort year country urtgrp : egen tot = total(tag2) + gen rdups = ndups / tot + gen tag3 = 0 + replace tag3 = 1 if rdups > 0.5 + drop tag1 tag2 ndups tot rdups + + /* extend selection to whole rotational group */ + bysort year country urtgrp : egen taga = max(tag3) + drop tag3 + + /* check if the rotational group with duplicates is the one covering most years in current release. If yes ignore. Still leaves us with duplicates? E.g. Pt */ + tab urtgrp year if taga == 1 & lgstgrp == 0 & _merge == 1 + drop if taga == 1 & lgstgrp == 0 & _merge == 1 + drop test2 taga + + /* check for unbalances */ + bysort country : egen test3 = total(nrtgrp2023) + replace test3 = 0 if test3 == . + tab urtgrp year if ( test3 != 0 & lgstgrp == 0 & _merge == 1 ) + drop if (test3 != 0 & lgstgrp == 0 & _merge == 1) + drop test3 + + /* this is the updated masterfile */ + gen merge2022 = _merge + drop _merge + save "${datapath}/EU-SILC/19 L-2023/masterD.dta", replace + + /* this is 2022D file for later control */ + keep if merge2022 == 1 + drop merge2022 + save "${datapath}/EU-SILC/18 L-2022/2022D.dta", replace + + +/* 2021 */ + /* open the 2021 Household register to get data from rotational groups inactive in 2019 and their uhid */ + clear + local filen : dir "${datapath}/EU-SILC/17 L-2021/" files "udb_l21d_ver*" + display `filen' + local filel "${datapath}/EU-SILC/17 L-2021/" + display `"`filel'"' + local fileln : display `"`filel'"' `filen' + + use `"`fileln'"', clear + + local new_var = lower("`var'") + + foreach var of varlist _all { + local new_var = lower("`var'") + cap rename `var' `new_var' + } + + /* block Stata from displaying IDs in exponential format */ + tostring db030, replace + gen year = db010 + gen country = db020 + gen hid = db030 + gen rotation_group = db075 + sort country hid year + replace country = "EL" if country == "GR" + + /* count the number of rotational groups for each country in this release */ + bysort country rotation_group: gen nvals = _n == 1 + bysort country : egen nrtgrp2021 = total(nvals) + drop nvals + + /* get the rotation group(s) that cover most years - rotational group to be added to create panel */ + bysort country rotation_group : egen maxyear = max(year) + bysort country rotation_group : egen minyear = min(year) + gen years_cov = maxyear - minyear + 1 + bysort country : egen maxgrp = max(years_cov) + gen slctd_rtgrp = 0 + bysort country rotation_group: replace slctd_rtgrp = rotation_group if years_cov == maxgrp + gen lgstgrp = 0 + bysort country rotation_group: replace lgstgrp = 1 if years_cov == maxgrp + + /* build household and rotationgroup IDs that are unique across releases */ + tostring rotation_group, generate(rotation_groupstr) + gen drpout_year = 0 + replace drpout_year = maxyear + 4 - years_cov + + /* check for rotation groups that dropped out before planned */ + bysort country rotation_group : egen maxyear_grp = max(year) + replace drpout_year = maxyear_grp if maxyear_grp < maxyear + + tostring drpout_year, replace + gen uhid = country + rotation_groupstr + drpout_year + hid + gen urtgrp = country + rotation_groupstr + drpout_year + destring drpout_year, replace + + /* check if any groups have been selected in the more recent 2019 release ("prolonged rotation groups") Ensure rotation groups that are not in next data release are selected to be preserved */ + sort year country rotation_group hid + merge m:m year country rotation_group using "${datapath}/EU-SILC/18 L-2022/2022D.dta" + replace slctd_rtgrp = rotation_group if _merge == 1 + + /* tag rotation groups that have already been selected in the more recent release in some years (overlapping) [How France is dealt with - 9 yr rotational panel but onyl 4 years worth of data released each year] */ + gen atag = 0 + replace atag = 1 if _merge == 3 + bysort country rotation_group : egen btag = max(atag) + bysort country rotation_group : egen ndrpoy = max(drpout_year2022) + replace drpout_year = ndrpoy if btag == 1 + drop atag btag ndrpoy + + /* update IDs urtgrp and uhid so that "prolonged rotation groups" maintain their urtgrp across different releases - update the drpout_year to the latest recorded */ + tostring drpout_year, replace + replace uhid = country + rotation_groupstr + drpout_year + hid + replace urtgrp = country + rotation_groupstr + drpout_year + destring drpout_year, replace + drop rotation_groupstr maxyear_grp minyear years_cov maxgrp maxyear_grp maxyear + + /* drop overlapping rotation group years and data from the more recent release */ + keep if _merge == 1 + drop _merge + + /* check if any selected groups have been selected in the more recent 2020 release ("prolonged rotation groups") */ + merge m:m year country urtgrp using "${datapath}/EU-SILC/19 L-2023/2023D.dta" + replace slctd_rtgrp = rotation_group if _merge == 1 + keep if _merge == 1 + drop _merge + + /* preserve observations to be appended */ + destring drpout_year, replace + drop if slctd_rtgrp == 0 + + /* for later checks */ + gen drpout_year2021 = drpout_year + gen slctd_urtgrp2021 = urtgrp + gen slctd_uhid2021 = uhid + + /* merge to masterfile */ + merge 1:1 year uhid using "${datapath}/EU-SILC/19 L-2023/masterD.dta" + + /* check for duplicates caused by same rotational groups changing code across releases - in same hid, different rotation_group */ + duplicates report year country hid + duplicates tag year country hid, generate(test2) + tab country year if test2 != 0 + tab urtgrp year if test2 != 0 + + /* check if the problem regards a large portion of the rotational group, if the portion is small ignor e */ + gen tag1 = 0 + replace tag1 = 1 if test2 != 0 + bysort year country urtgrp : egen ndups = total(tag1) + gen tag2 = 1 + bysort year country urtgrp : egen tot = total(tag2) + gen rdups = ndups / tot + gen tag3 = 0 + replace tag3 = 1 if rdups > 0.5 + drop tag1 tag2 ndups tot rdups + + /* extend selection to whole rotational group */ + bysort year country urtgrp : egen taga = max(tag3) + drop tag3 + + /* check if the rotational group with duplicates is the one covering most years in current release. If yes ignore */ + tab urtgrp year if taga == 1 & lgstgrp == 0 & _merge == 1 + drop if taga == 1 & lgstgrp == 0 & _merge == 1 + drop test2 taga + + /* check for unbalances */ + bysort country : egen test3 = total(nrtgrp2022) + bysort country : egen test4 = total(nrtgrp2023) + replace test3 = 0 if test3 == . + replace test4 = 0 if test4 == . + tab urtgrp year if ( test3 != 0 & lgstgrp == 0 & _merge == 1 ) & ( test4 != 0 & lgstgrp == 0 & _merge == 1 ) + drop if ( test3 != 0 & lgstgrp == 0 & _merge == 1 ) & ( test4 != 0 & lgstgrp == 0 & _merge == 1 ) + drop test3 test4 + + /* this is the updated masterfile */ + gen merge2021 = _merge + drop _merge + save "${datapath}/EU-SILC/19 L-2023/masterD.dta", replace + + /* this is 2021D file for later control */ + keep if merge2021 == 1 + drop merge2021 + save "${datapath}/EU-SILC/17 L-2021/2021D.dta", replace + +/* 2020 */ + /* open the 2020 Household register to get data from rotational groups inactive in more recent releases and their uhid */ + clear + local filen : dir "${datapath}/EU-SILC/16 L-2020/" files "udb_l20d_ver*" + display `filen' + local filel "${datapath}/EU-SILC/16 L-2020/" + display `"`filel'"' + local fileln : display `"`filel'"' `filen' + + use `"`fileln'"', clear + + local new_var = lower("`var'") + + foreach var of varlist _all { + local new_var = lower("`var'") + cap rename `var' `new_var' + } + + /* block Stata from displaying IDs in exponential format */ + tostring db030, replace + gen year = db010 + gen country = db020 + gen hid = db030 + gen rotation_group = db075 + sort country hid year + replace country = "EL" if country == "GR" + + /* count the number of rotational groups for each country in this release */ + bysort country rotation_group: gen nvals = _n == 1 + bysort country : egen nrtgrp2020 = total(nvals) + drop nvals + + /* get the rotation group(s) that cover most years */ + bysort country rotation_group : egen maxyear = max(year) + bysort country rotation_group : egen minyear = min(year) + gen years_cov = maxyear - minyear + 1 + bysort country : egen maxgrp = max(years_cov) + gen slctd_rtgrp = 0 + bysort country rotation_group: replace slctd_rtgrp = rotation_group if years_cov == maxgrp + gen lgstgrp = 0 + bysort country rotation_group: replace lgstgrp = 1 if years_cov == maxgrp + + /* build household and rotationgroup IDs that are unique across releases */ + tostring rotation_group, generate(rotation_groupstr) + gen drpout_year = 0 + replace drpout_year = maxyear + 4 - years_cov + + /* check for rotation groups that dropped out before planned */ + bysort country rotation_group : egen maxyear_grp = max(year) + replace drpout_year = maxyear_grp if maxyear_grp < maxyear + + tostring drpout_year, replace + gen uhid = country + rotation_groupstr + drpout_year + hid + gen urtgrp = country + rotation_groupstr + drpout_year + destring drpout_year, replace + + /* check if any selected groups have been selected in the more recent 2021 release ("prolonged rotation groups") */ + sort year country rotation_group hid + merge m:m year country rotation_group using "${datapath}/EU-SILC/17 L-2021/2021D.dta" + replace slctd_rtgrp = rotation_group if _merge == 1 + + /* tag rotation groups that have already been selected in the more recent release in some years (overlapping) */ + gen atag = 0 + replace atag = 1 if _merge == 3 + bysort country rotation_group : egen btag = max(atag) + bysort country rotation_group : egen ndrpoy = max(drpout_year2021) + replace drpout_year = ndrpoy if btag == 1 + drop atag btag ndrpoy + + /* update urtgrp and uhid so that "prolonged rotation groups" maintain their urtgrp across different releases */ + tostring drpout_year, replace + replace uhid = country + rotation_groupstr + drpout_year + hid + replace urtgrp = country + rotation_groupstr + drpout_year + destring drpout_year, replace + drop rotation_groupstr maxyear_grp minyear years_cov maxgrp maxyear_grp maxyear + + /* drop overlapping rotation group years and data from the more recent release */ + keep if _merge == 1 + drop _merge + + /* check if any selected groups have been selected in the more recent 2022 release ("prolonged rotation groups") */ + merge m:m year country urtgrp using "${datapath}/EU-SILC/18 L-2022/2022D.dta" + replace slctd_rtgrp = rotation_group if _merge == 1 + keep if _merge == 1 + drop _merge + + /* check if any selected groups have been selected in the more recent 2023 release ("prolonged rotation groups") */ + merge m:m year country urtgrp using "${datapath}/EU-SILC/19 L-2023/2023D.dta" + replace slctd_rtgrp = rotation_group if _merge == 1 + keep if _merge == 1 + drop _merge + destring drpout_year, replace + drop if slctd_rtgrp == 0 + + /* for later checks */ + gen drpout_year2020 = drpout_year + gen slctd_urtgrp2020 = urtgrp + gen slctd_uhid2020 = uhid + + /* merge to masterfile */ + merge 1:1 year uhid using "${datapath}/EU-SILC/19 L-2023/masterD.dta" + + /* check for duplicates caused by same rotational groups in terms of same hid, different rotation_group */ + duplicates report year country hid + duplicates tag year country hid, generate(test2) + tab country year if test2 != 0 + tab urtgrp year if test2 != 0 + + /* check if the problem regards a large portion of the rotational group, if the portion is small ignore */ + gen tag1 = 0 + replace tag1 = 1 if test2 != 0 + bysort year country urtgrp : egen ndups = total(tag1) + gen tag2 = 1 + bysort year country urtgrp : egen tot = total(tag2) + gen rdups = ndups / tot + gen tag3 = 0 + replace tag3 = 1 if rdups > 0.5 + drop tag1 tag2 ndups tot rdups + + /* extend selection to whole rotational group */ + bysort year country urtgrp : egen taga = max(tag3) + drop tag3 + + /* check if the rotational group with duplicates is the one covering most years in current release. If yes ignore */ + tab urtgrp year if taga == 1 & lgstgrp == 0 & _merge == 1 + drop if taga == 1 & lgstgrp == 0 & _merge == 1 + drop test2 taga + + /* check for unbalances */ + bysort country : egen test3 = total(nrtgrp2021) + bysort country : egen test4 = total(nrtgrp2022) + bysort country : egen test5 = total(nrtgrp2023) + replace test3 = 0 if test3 == . + replace test4 = 0 if test4 == . + replace test5 = 0 if test5 == . + tab urtgrp year if ( test3 != 0 & lgstgrp == 0 & _merge == 1 ) & ( test4 != 0 & lgstgrp == 0 & _merge == 1 ) &( test5 != 0 & lgstgrp == 0 & _merge == 1 ) + drop if ( test3 != 0 & lgstgrp == 0 & _merge == 1 ) & ( test4 != 0 & lgstgrp == 0 & _merge == 1 ) & ( test5 != 0 & lgstgrp == 0 & _merge == 1 ) + drop test3 test4 test5 + + /* this is the updated masterfile */ + gen merge2020 = _merge + drop _merge + save "${datapath}/EU-SILC/19 L-2023/masterD.dta", replace + + /* this is 2020D file for later control */ + keep if merge2020 == 1 + drop merge2020 + save "${datapath}/EU-SILC/16 L-2020/2020D.dta", replace + + +/* 2019 */ + /* open the 2019 Household register to get data from rotational groups inactive in more recent releases and their uhid */ + clear + local filen : dir "${datapath}/EU-SILC/15 L-2019/" files "udb_l19d_ver*" + display `filen' + local filel "${datapath}/EU-SILC/15 L-2019/" + display `"`filel'"' + local fileln : display `"`filel'"' `filen' + + + use `"`fileln'"', clear + + local new_var = lower("`var'") + + foreach var of varlist _all { + local new_var = lower("`var'") + cap rename `var' `new_var' + } + + /* block Stata from displaying IDs in exponential format */ + tostring db030, replace + gen year = db010 + gen country = db020 + gen hid = db030 + gen rotation_group = db075 + sort country hid year + replace country = "EL" if country == "GR" + + /* count the number of rotational groups for each country in this release */ + bysort country rotation_group: gen nvals = _n == 1 + bysort country : egen nrtgrp2019 = total(nvals) + drop nvals + + /* get the rotation group(s) that cover most years */ + bysort country rotation_group : egen maxyear = max(year) + bysort country rotation_group : egen minyear = min(year) + gen years_cov = maxyear - minyear + 1 + bysort country : egen maxgrp = max(years_cov) + gen slctd_rtgrp = 0 + bysort country rotation_group: replace slctd_rtgrp = rotation_group if years_cov == maxgrp + gen lgstgrp = 0 + bysort country rotation_group: replace lgstgrp = 1 if years_cov == maxgrp + + /* build household and rotationgroup IDs that are unique across releases */ + tostring rotation_group, generate(rotation_groupstr) + gen drpout_year = 0 + replace drpout_year = maxyear + 4 - years_cov + + /* check for rotation groups that dropped out before planned */ + bysort country rotation_group : egen maxyear_grp = max(year) + replace drpout_year = maxyear_grp if maxyear_grp < maxyear + + tostring drpout_year, replace + gen uhid = country + rotation_groupstr + drpout_year + hid + gen urtgrp = country + rotation_groupstr + drpout_year + destring drpout_year, replace + + /* check if any selected groups have been selected in the more recent 2020 release ("prolonged rotation groups") */ + sort year country rotation_group hid + merge m:m year country rotation_group using "${datapath}/EU-SILC/16 L-2020/2020D.dta" + replace slctd_rtgrp = rotation_group if _merge == 1 + + /* tag rotation groups that have already been selected in the more recent release in some years (overlapping) */ + gen atag = 0 + replace atag = 1 if _merge == 3 + bysort country rotation_group : egen btag = max(atag) + bysort country rotation_group : egen ndrpoy = max(drpout_year2020) + replace drpout_year = ndrpoy if btag == 1 + drop atag btag ndrpoy + + /* update urtgrp and uhid so that "prolonged rotation groups" maintain their urtgrp across different releases */ + tostring drpout_year, replace + replace uhid = country + rotation_groupstr + drpout_year + hid + replace urtgrp = country + rotation_groupstr + drpout_year + destring drpout_year, replace + drop rotation_groupstr maxyear_grp minyear years_cov maxgrp maxyear_grp maxyear + + /* drop overlapping rotation group years and data from the more recent release */ + keep if _merge == 1 + drop _merge + + /* check if any selected groups have been selected in the more recent 2021 release ("prolonged rotation groups") */ + merge m:m year country urtgrp using "${datapath}/EU-SILC/17 L-2021/2021D.dta" + replace slctd_rtgrp = rotation_group if _merge == 1 + keep if _merge == 1 + drop _merge + + /* check if any selected groups have been selected in the more recent 2022 release ("prolonged rotation groups") */ + merge m:m year country urtgrp using "${datapath}/EU-SILC/18 L-2022/2022D.dta" + replace slctd_rtgrp = rotation_group if _merge == 1 + keep if _merge == 1 + drop _merge + destring drpout_year, replace + drop if slctd_rtgrp == 0 + + /* for later checks */ + gen drpout_year2019 = drpout_year + gen slctd_urtgrp2019 = urtgrp + gen slctd_uhid2019 = uhid + + /* merge to masterfile */ + merge 1:1 year uhid using "${datapath}/EU-SILC/19 L-2023/masterD.dta" + + /* check for duplicates caused by same rotational groups in terms of same hid, different rotation_group */ + duplicates report year country hid + duplicates tag year country hid, generate(test2) + tab country year if test2 != 0 + tab urtgrp year if test2 != 0 + + /* check if the problem regards a large portion of the rotational group, if the portion is small ignore */ + gen tag1 = 0 + replace tag1 = 1 if test2 != 0 + bysort year country urtgrp : egen ndups = total(tag1) + gen tag2 = 1 + bysort year country urtgrp : egen tot = total(tag2) + gen rdups = ndups / tot + gen tag3 = 0 + replace tag3 = 1 if rdups > 0.5 + drop tag1 tag2 ndups tot rdups + + /* extend selection to whole rotational group */ + bysort year country urtgrp : egen taga = max(tag3) + drop tag3 + + /* check if the rotational group with duplicates is the one covering most years in current release. If yes ignore */ + tab urtgrp year if taga == 1 & lgstgrp == 0 & _merge == 1 + drop if taga == 1 & lgstgrp == 0 & _merge == 1 + drop test2 taga + + /* check for unbalances */ + bysort country : egen test3 = total(nrtgrp2020) + bysort country : egen test4 = total(nrtgrp2021) + bysort country : egen test5 = total(nrtgrp2022) + replace test3 = 0 if test3 == . + replace test4 = 0 if test4 == . + replace test5 = 0 if test5 == . + tab urtgrp year if ( test3 != 0 & lgstgrp == 0 & _merge == 1 ) & ( test4 != 0 & lgstgrp == 0 & _merge == 1 ) &( test5 != 0 & lgstgrp == 0 & _merge == 1 ) + drop if ( test3 != 0 & lgstgrp == 0 & _merge == 1 ) & ( test4 != 0 & lgstgrp == 0 & _merge == 1 ) &( test5 != 0 & lgstgrp == 0 & _merge == 1 ) + drop test3 test4 test5 + + /* this is the updated masterfile */ + gen merge2019 = _merge + drop _merge + save "${datapath}/EU-SILC/19 L-2023/masterD.dta", replace + + /* this is 2016D file for later control */ + keep if merge2019 == 1 + drop merge2019 + save "${datapath}/EU-SILC/15 L-2019/2019D.dta", replace + + +/* 2018 */ + /* open the 2018 Household register to get data from rotational groups inactive in 2019 and their uhid */ + clear + local filen : dir "${datapath}/EU-SILC/14 L-2018/" files "udb_l18d_ver*" + display `filen' + local filel "${datapath}/EU-SILC/14 L-2018/" + display `"`filel'"' + local fileln : display `"`filel'"' `filen' + + use `"`fileln'"', clear + + local new_var = lower("`var'") + + foreach var of varlist _all { + local new_var = lower("`var'") + cap rename `var' `new_var' + } + + /* block Stata from displaying IDs in exponential format */ + tostring db030, replace + gen year = db010 + gen country = db020 + gen hid = db030 + gen rotation_group = db075 + sort country hid year + replace country = "EL" if country == "GR" + + /* count the number of rotational groups for each country in this release */ + bysort country rotation_group: gen nvals = _n == 1 + bysort country : egen nrtgrp2018 = total(nvals) + drop nvals + + /* get the rotation group(s) that cover most years - rotational group to be added to create panel */ + bysort country rotation_group : egen maxyear = max(year) + bysort country rotation_group : egen minyear = min(year) + gen years_cov = maxyear - minyear + 1 + bysort country : egen maxgrp = max(years_cov) + gen slctd_rtgrp = 0 + bysort country rotation_group: replace slctd_rtgrp = rotation_group if years_cov == maxgrp + gen lgstgrp = 0 + bysort country rotation_group: replace lgstgrp = 1 if years_cov == maxgrp + + /* build household and rotationgroup IDs that are unique across releases */ + tostring rotation_group, generate(rotation_groupstr) + gen drpout_year = 0 + replace drpout_year = maxyear + 4 - years_cov + + /* check for rotation groups that dropped out before planned */ + bysort country rotation_group : egen maxyear_grp = max(year) + replace drpout_year = maxyear_grp if maxyear_grp < maxyear + + tostring drpout_year, replace + gen uhid = country + rotation_groupstr + drpout_year + hid + gen urtgrp = country + rotation_groupstr + drpout_year + destring drpout_year, replace + + /* check if any groups have been selected in the more recent 2019 release ("prolonged rotation groups") Ensure rotation groups that are not in next data release are selected to be preserved */ + sort year country rotation_group hid + merge m:m year country rotation_group using "${datapath}/EU-SILC/15 L-2019/2019D.dta" + replace slctd_rtgrp = rotation_group if _merge == 1 + + /* tag rotation groups that have already been selected in the more recent release in some years (overlapping) [How France is dealt with - 9 yr rotational panel but onyl 4 years worth of data released each year] */ + gen atag = 0 + replace atag = 1 if _merge == 3 + bysort country rotation_group : egen btag = max(atag) + bysort country rotation_group : egen ndrpoy = max(drpout_year2019) + replace drpout_year = ndrpoy if btag == 1 + drop atag btag ndrpoy + + /* update IDs urtgrp and uhid so that "prolonged rotation groups" maintain their urtgrp across different releases - update the drpout_year to the latest recorded */ + tostring drpout_year, replace + replace uhid = country + rotation_groupstr + drpout_year + hid + replace urtgrp = country + rotation_groupstr + drpout_year + destring drpout_year, replace + drop rotation_groupstr maxyear_grp minyear years_cov maxgrp maxyear_grp maxyear + + /* drop overlapping rotation group years and data from the more recent release */ + keep if _merge == 1 + drop _merge + + /* check if any selected groups have been selected in the more recent 2020 release ("prolonged rotation groups") */ + merge m:m year country urtgrp using "${datapath}/EU-SILC/16 L-2020/2020D.dta" + replace slctd_rtgrp = rotation_group if _merge == 1 + keep if _merge == 1 + drop _merge + + /* check if any selected groups have been selected in the more recent 2021 release ("prolonged rotation groups") */ + merge m:m year country urtgrp using "${datapath}/EU-SILC/17 L-2021/2021D.dta" + replace slctd_rtgrp = rotation_group if _merge == 1 + keep if _merge == 1 + drop _merge + + /* preserve observations to be appended */ + destring drpout_year, replace + drop if slctd_rtgrp == 0 + + /* for later checks */ + gen drpout_year2018 = drpout_year + gen slctd_urtgrp2018 = urtgrp + gen slctd_uhid2018 = uhid + + /* merge to masterfile */ + merge 1:1 year uhid using "${datapath}/EU-SILC/19 L-2023/masterD.dta" + + /* check for duplicates caused by same rotational groups changing code across releases - in same hid, different rotation_group */ + duplicates report year country hid + duplicates tag year country hid, generate(test2) + tab country year if test2 != 0 + tab urtgrp year if test2 != 0 + + /* check if the problem regards a large portion of the rotational group, if the portion is small ignore */ + gen tag1 = 0 + replace tag1 = 1 if test2 != 0 + bysort year country urtgrp : egen ndups = total(tag1) + gen tag2 = 1 + bysort year country urtgrp : egen tot = total(tag2) + gen rdups = ndups / tot + gen tag3 = 0 + replace tag3 = 1 if rdups > 0.5 + drop tag1 tag2 ndups tot rdups + + /* extend selection to whole rotational group */ + bysort year country urtgrp : egen taga = max(tag3) + drop tag3 + + /* check if the rotational group with duplicates is the one covering most years in current release. If yes ignore */ + tab urtgrp year if taga == 1 & lgstgrp == 0 & _merge == 1 + drop if taga == 1 & lgstgrp == 0 & _merge == 1 + drop test2 taga + + /* check for unbalances */ + bysort country : egen test3 = total(nrtgrp2019) + bysort country : egen test4 = total(nrtgrp2020) + bysort country : egen test5 = total(nrtgrp2021) + replace test3 = 0 if test3 == . + replace test4 = 0 if test4 == . + replace test5 = 0 if test5 == . + tab urtgrp year if ( test3 != 0 & lgstgrp == 0 & _merge == 1 ) & ( test4 != 0 & lgstgrp == 0 & _merge == 1 ) &( test5 != 0 & lgstgrp == 0 & _merge == 1 ) + drop if ( test3 != 0 & lgstgrp == 0 & _merge == 1 ) & ( test4 != 0 & lgstgrp == 0 & _merge == 1 ) &( test5 != 0 & lgstgrp == 0 & _merge == 1 ) + drop test3 test4 test5 + + /* this is the updated masterfile */ + gen merge2018 = _merge + drop _merge + save "${datapath}/EU-SILC/19 L-2023/masterD.dta", replace + + /* this is 2018D file for later control */ + keep if merge2018 == 1 + drop merge2018 + save "${datapath}/EU-SILC/14 L-2018/2018D.dta", replace + + +/* 2017 */ + /* open the 2017 Household register to get data from rotational groups inactive in more recent releases and their uhid */ + clear + local filen : dir "${datapath}/EU-SILC/13 L-2017/" files "udb_l17d_ver*" + display `filen' + local filel "${datapath}/EU-SILC/13 L-2017/" + display `"`filel'"' + local fileln : display `"`filel'"' `filen' + + use `"`fileln'"', clear + + local new_var = lower("`var'") + + foreach var of varlist _all { + local new_var = lower("`var'") + cap rename `var' `new_var' + } + + /* block Stata from displaying IDs in exponential format */ + tostring db030, replace + gen year = db010 + gen country = db020 + gen hid = db030 + gen rotation_group = db075 + sort country hid year + replace country = "EL" if country == "GR" + + /* count the number of rotational groups for each country in this release */ + bysort country rotation_group: gen nvals = _n == 1 + bysort country : egen nrtgrp2017 = total(nvals) + drop nvals + + /* get the rotation group(s) that cover most years */ + bysort country rotation_group : egen maxyear = max(year) + bysort country rotation_group : egen minyear = min(year) + gen years_cov = maxyear - minyear + 1 + bysort country : egen maxgrp = max(years_cov) + gen slctd_rtgrp = 0 + bysort country rotation_group: replace slctd_rtgrp = rotation_group if years_cov == maxgrp + gen lgstgrp = 0 + bysort country rotation_group: replace lgstgrp = 1 if years_cov == maxgrp + + /* build household and rotationgroup IDs that are unique across releases */ + tostring rotation_group, generate(rotation_groupstr) + gen drpout_year = 0 + replace drpout_year = maxyear + 4 - years_cov + + /* check for rotation groups that dropped out before planned */ + bysort country rotation_group : egen maxyear_grp = max(year) + replace drpout_year = maxyear_grp if maxyear_grp < maxyear + + tostring drpout_year, replace + gen uhid = country + rotation_groupstr + drpout_year + hid + gen urtgrp = country + rotation_groupstr + drpout_year + destring drpout_year, replace + + /* check if any selected groups have been selected in the more recent 2018 release ("prolonged rotation groups") */ + sort year country rotation_group hid + merge m:m year country rotation_group using "${datapath}/EU-SILC/14 L-2018/2018D.dta" + replace slctd_rtgrp = rotation_group if _merge == 1 + + /* tag rotation groups that have already been selected in the more recent release in some years (overlapping) */ + gen atag = 0 + replace atag = 1 if _merge == 3 + bysort country rotation_group : egen btag = max(atag) + bysort country rotation_group : egen ndrpoy = max(drpout_year2018) + replace drpout_year = ndrpoy if btag == 1 + drop atag btag ndrpoy + + /* update urtgrp and uhid so that "prolonged rotation groups" maintain their urtgrp across different releases */ + tostring drpout_year, replace + replace uhid = country + rotation_groupstr + drpout_year + hid + replace urtgrp = country + rotation_groupstr + drpout_year + destring drpout_year, replace + drop rotation_groupstr maxyear_grp minyear years_cov maxgrp maxyear_grp maxyear + + /* drop overlapping rotation group years and data from the more recent release */ + keep if _merge == 1 + drop _merge + + /* check if any selected groups have been selected in the more recent 2019 release ("prolonged rotation groups") */ + merge m:m year country urtgrp using "${datapath}/EU-SILC/15 L-2019/2019D.dta" + replace slctd_rtgrp = rotation_group if _merge == 1 + keep if _merge == 1 + drop _merge + + /* check if any selected groups have been selected in the more recent 2020 release ("prolonged rotation groups") */ + merge m:m year country urtgrp using "${datapath}/EU-SILC/16 L-2020/2020D.dta" + replace slctd_rtgrp = rotation_group if _merge == 1 + keep if _merge == 1 + drop _merge + destring drpout_year, replace + drop if slctd_rtgrp == 0 + + /* for later checks */ + gen drpout_year2017 = drpout_year + gen slctd_urtgrp2017 = urtgrp + gen slctd_uhid2017 = uhid + + /* merge to masterfile */ + merge 1:1 year uhid using "${datapath}/EU-SILC/19 L-2023/masterD.dta" + + /* check for duplicates caused by same rotational groups in terms of same hid, different rotation_group */ + duplicates report year country hid + duplicates tag year country hid, generate(test2) + tab country year if test2 != 0 + tab urtgrp year if test2 != 0 + + /* check if the problem regards a large portion of the rotational group, if the portion is small ignore */ + gen tag1 = 0 + replace tag1 = 1 if test2 != 0 + bysort year country urtgrp : egen ndups = total(tag1) + gen tag2 = 1 + bysort year country urtgrp : egen tot = total(tag2) + gen rdups = ndups / tot + gen tag3 = 0 + replace tag3 = 1 if rdups > 0.5 + drop tag1 tag2 ndups tot rdups + + /* extend selection to whole rotational group */ + bysort year country urtgrp : egen taga = max(tag3) + drop tag3 + + /* check if the rotational group with duplicates is the one covering most years in current release. If yes ignore */ + tab urtgrp year if taga == 1 & lgstgrp == 0 & _merge == 1 + drop if taga == 1 & lgstgrp == 0 & _merge == 1 + drop test2 taga + + /* check for unbalances */ + bysort country : egen test3 = total(nrtgrp2018) + bysort country : egen test4 = total(nrtgrp2019) + bysort country : egen test5 = total(nrtgrp2020) + replace test3 = 0 if test3 == . + replace test4 = 0 if test4 == . + replace test5 = 0 if test5 == . + tab urtgrp year if ( test3 != 0 & lgstgrp == 0 & _merge == 1 ) & ( test4 != 0 & lgstgrp == 0 & _merge == 1 ) &( test5 != 0 & lgstgrp == 0 & _merge == 1 ) + drop if ( test3 != 0 & lgstgrp == 0 & _merge == 1 ) & ( test4 != 0 & lgstgrp == 0 & _merge == 1 ) & ( test5 != 0 & lgstgrp == 0 & _merge == 1 ) + drop test3 test4 test5 + + /* this is the updated masterfile */ + gen merge2017 = _merge + drop _merge + save "${datapath}/EU-SILC/19 L-2023/masterD.dta", replace + + /* this is 2017D file for later control */ + keep if merge2017 == 1 + drop merge2017 + save "${datapath}/EU-SILC/13 L-2017/2017D.dta", replace + + +/* 2016 */ + /* open the 2016 Household register to get data from rotational groups inactive in more recent releases and their uhid */ + clear + local filen : dir "${datapath}/EU-SILC/12 L-2016/" files "udb_l16d_ver*" + display `filen' + local filel "${datapath}/EU-SILC/12 L-2016/" + display `"`filel'"' + local fileln : display `"`filel'"' `filen' + + + use `"`fileln'"', clear + + local new_var = lower("`var'") + + foreach var of varlist _all { + local new_var = lower("`var'") + cap rename `var' `new_var' + } + + /* block Stata from displaying IDs in exponential format */ + tostring db030, replace + gen year = db010 + gen country = db020 + gen hid = db030 + gen rotation_group = db075 + sort country hid year + replace country = "EL" if country == "GR" + + /* count the number of rotational groups for each country in this release */ + bysort country rotation_group: gen nvals = _n == 1 + bysort country : egen nrtgrp2016 = total(nvals) + drop nvals + + /* get the rotation group(s) that cover most years */ + bysort country rotation_group : egen maxyear = max(year) + bysort country rotation_group : egen minyear = min(year) + gen years_cov = maxyear - minyear + 1 + bysort country : egen maxgrp = max(years_cov) + gen slctd_rtgrp = 0 + bysort country rotation_group: replace slctd_rtgrp = rotation_group if years_cov == maxgrp + gen lgstgrp = 0 + bysort country rotation_group: replace lgstgrp = 1 if years_cov == maxgrp + + /* build household and rotationgroup IDs that are unique across releases */ + tostring rotation_group, generate(rotation_groupstr) + gen drpout_year = 0 + replace drpout_year = maxyear + 4 - years_cov + + /* check for rotation groups that dropped out before planned */ + bysort country rotation_group : egen maxyear_grp = max(year) + replace drpout_year = maxyear_grp if maxyear_grp < maxyear + + tostring drpout_year, replace + gen uhid = country + rotation_groupstr + drpout_year + hid + gen urtgrp = country + rotation_groupstr + drpout_year + destring drpout_year, replace + + /* check if any selected groups have been selected in the more recent 2017 release ("prolonged rotation groups") */ + sort year country rotation_group hid + merge m:m year country rotation_group using "${datapath}/EU-SILC/13 L-2017/2017D.dta" + replace slctd_rtgrp = rotation_group if _merge == 1 + + /* tag rotation groups that have already been selected in the more recent release in some years (overlapping) */ + gen atag = 0 + replace atag = 1 if _merge == 3 + bysort country rotation_group : egen btag = max(atag) + bysort country rotation_group : egen ndrpoy = max(drpout_year2017) + replace drpout_year = ndrpoy if btag == 1 + drop atag btag ndrpoy + + /* update urtgrp and uhid so that "prolonged rotation groups" maintain their urtgrp across different releases */ + tostring drpout_year, replace + replace uhid = country + rotation_groupstr + drpout_year + hid + replace urtgrp = country + rotation_groupstr + drpout_year + destring drpout_year, replace + drop rotation_groupstr maxyear_grp minyear years_cov maxgrp maxyear_grp maxyear + + /* drop overlapping rotation group years and data from the more recent release */ + keep if _merge == 1 + drop _merge + + /* check if any selected groups have been selected in the more recent 2018 release ("prolonged rotation groups") */ + merge m:m year country urtgrp using "${datapath}/EU-SILC/14 L-2018/2018D.dta" + replace slctd_rtgrp = rotation_group if _merge == 1 + keep if _merge == 1 + drop _merge + + /* check if any selected groups have been selected in the more recent 2019 release ("prolonged rotation groups") */ + merge m:m year country urtgrp using "${datapath}/EU-SILC/15 L-2019/2019D.dta" + replace slctd_rtgrp = rotation_group if _merge == 1 + keep if _merge == 1 + drop _merge + + destring drpout_year, replace + drop if slctd_rtgrp == 0 + + /* for later checks */ + gen drpout_year2016 = drpout_year + gen slctd_urtgrp2016 = urtgrp + gen slctd_uhid2016 = uhid + + /* merge to masterfile */ + merge 1:1 year uhid using "${datapath}/EU-SILC/19 L-2023/masterD.dta" + + /* check for duplicates caused by same rotational groups in terms of same hid, different rotation_group */ + duplicates report year country hid + duplicates tag year country hid, generate(test2) + tab country year if test2 != 0 + tab urtgrp year if test2 != 0 + + /* check if the problem regards a large portion of the rotational group, if the portion is small ignore */ + gen tag1 = 0 + replace tag1 = 1 if test2 != 0 + bysort year country urtgrp : egen ndups = total(tag1) + gen tag2 = 1 + bysort year country urtgrp : egen tot = total(tag2) + gen rdups = ndups / tot + gen tag3 = 0 + replace tag3 = 1 if rdups > 0.5 + drop tag1 tag2 ndups tot rdups + + /* extend selection to whole rotational group */ + bysort year country urtgrp : egen taga = max(tag3) + drop tag3 + + /* check if the rotational group with duplicates is the one covering most years in current release. If yes ignore */ + tab urtgrp year if taga == 1 & lgstgrp == 0 & _merge == 1 + drop if taga == 1 & lgstgrp == 0 & _merge == 1 + drop test2 taga + + /* check for unbalances */ + bysort country : egen test3 = total(nrtgrp2017) + bysort country : egen test4 = total(nrtgrp2018) + bysort country : egen test5 = total(nrtgrp2019) + replace test3 = 0 if test3 == . + replace test4 = 0 if test4 == . + replace test5 = 0 if test5 == . + tab urtgrp year if ( test3 != 0 & lgstgrp == 0 & _merge == 1 ) & ( test4 != 0 & lgstgrp == 0 & _merge == 1 ) &( test5 != 0 & lgstgrp == 0 & _merge == 1 ) + drop if ( test3 != 0 & lgstgrp == 0 & _merge == 1 ) & ( test4 != 0 & lgstgrp == 0 & _merge == 1 ) &( test5 != 0 & lgstgrp == 0 & _merge == 1 ) + drop test3 test4 test5 + + /* this is the updated masterfile */ + gen merge2016 = _merge + drop _merge + save "${datapath}/EU-SILC/19 L-2023/masterD.dta", replace + + /* this is 2016D file for later control */ + keep if merge2016 == 1 + drop merge2016 + save "${datapath}/EU-SILC/12 L-2016/2016D.dta", replace + + +/* 2015 */ + /* open the 2015 Household register to get data from rotational groups inactive in more recent releases and their uhid */ + clear + local filen : dir "${datapath}/EU-SILC/11 L-2015/" files "udb_l15d_ver*" + display `filen' + local filel "${datapath}/EU-SILC/11 L-2015/" + display `"`filel'"' + local fileln : display `"`filel'"' `filen' + use `"`fileln'"', clear + + local new_var = lower("`var'") + + foreach var of varlist _all { + local new_var = lower("`var'") + cap rename `var' `new_var' + } + + /* block Stata from displaying IDs in exponential format */ + tostring db030, replace + gen year = db010 + gen country = db020 + gen hid = db030 + gen rotation_group = db075 + sort country hid year + replace country = "EL" if country == "GR" + + /* count the number of rotational groups for each country in this release */ + bysort country rotation_group: gen nvals = _n == 1 + bysort country : egen nrtgrp2015 = total(nvals) + drop nvals + + /* get the rotation group(s) that cover most years */ + bysort country rotation_group : egen maxyear = max(year) + bysort country rotation_group : egen minyear = min(year) + gen years_cov = maxyear - minyear + 1 + bysort country : egen maxgrp = max(years_cov) + gen slctd_rtgrp = 0 + bysort country rotation_group: replace slctd_rtgrp = rotation_group if years_cov == maxgrp + gen lgstgrp = 0 + bysort country rotation_group: replace lgstgrp = 1 if years_cov == maxgrp + + /* build household and rotationgroup IDs that are unique across releases */ + tostring rotation_group, generate(rotation_groupstr) + gen drpout_year = 0 + replace drpout_year = maxyear + 4 - years_cov + + /* check for rotation groups that dropped out before planned */ + bysort country rotation_group : egen maxyear_grp = max(year) + replace drpout_year = maxyear_grp if maxyear_grp < maxyear + + tostring drpout_year, replace + gen uhid = country + rotation_groupstr + drpout_year + hid + gen urtgrp = country + rotation_groupstr + drpout_year + destring drpout_year, replace + + /* check if any selected groups have been selected in the more recent 2016 release ("prolonged rotation groups") */ + sort year country rotation_group hid + merge m:m year country rotation_group using "${datapath}/EU-SILC/12 L-2016/2016D.dta" + replace slctd_rtgrp = rotation_group if _merge == 1 + + /* tag rotation groups that have already been selected in the more recent release in some years (overlapping) */ + gen atag = 0 + replace atag = 1 if _merge == 3 + bysort country rotation_group : egen btag = max(atag) + bysort country rotation_group : egen ndrpoy = max(drpout_year2016) + replace drpout_year = ndrpoy if btag == 1 + drop atag btag ndrpoy + + /* update urtgrp and uhid so that "prolonged rotation groups" maintain their urtgrp across different releases */ + tostring drpout_year, replace + replace uhid = country + rotation_groupstr + drpout_year + hid + replace urtgrp = country + rotation_groupstr + drpout_year + destring drpout_year, replace + drop rotation_groupstr maxyear_grp minyear years_cov maxgrp maxyear_grp maxyear + + /* drop overlapping rotation group years and data from the more recent release */ + keep if _merge == 1 + drop _merge + + /* check if any selected groups have been selected in the more recent 2017 release ("prolonged rotation groups") */ + merge m:m year country urtgrp using "${datapath}/EU-SILC/13 L-2017/2017D.dta" + replace slctd_rtgrp = rotation_group if _merge == 1 + keep if _merge == 1 + drop _merge + + /* check if any selected groups have been selected in the more recent 2018 release ("prolonged rotation groups") */ + merge m:m year country urtgrp using "${datapath}/EU-SILC/14 L-2018/2018D.dta" + replace slctd_rtgrp = rotation_group if _merge == 1 + keep if _merge == 1 + drop _merge + + destring drpout_year, replace + drop if slctd_rtgrp == 0 + + /* for later checks */ + gen drpout_year2015 = drpout_year + gen slctd_urtgrp2015 = urtgrp + gen slctd_uhid2015 = uhid + + /* merge to masterfile */ + merge 1:1 year uhid using "${datapath}/EU-SILC/19 L-2023/masterD.dta" + + /* check for duplicates caused by same rotational groups in terms of same hid, different rotation_group */ + duplicates report year country hid + duplicates tag year country hid, generate(test2) + tab country year if test2 != 0 + tab urtgrp year if test2 != 0 + /* check if the problem regards a large portion of the rotational group, if the portion is small ignore */ + gen tag1 = 0 + replace tag1 = 1 if test2 != 0 + bysort year country urtgrp : egen ndups = total(tag1) + gen tag2 = 1 + bysort year country urtgrp : egen tot = total(tag2) + gen rdups = ndups / tot + gen tag3 = 0 + replace tag3 = 1 if rdups > 0.5 + drop tag1 tag2 ndups tot rdups + /* extend selection to whole rotational group */ + bysort year country urtgrp : egen taga = max(tag3) + drop tag3 + /* check if the rotational group with duplicates is the one covering most years in current release. If yes ignore */ + tab urtgrp year if taga == 1 & lgstgrp == 0 & _merge == 1 + drop if taga == 1 & lgstgrp == 0 & _merge == 1 + drop test2 taga + /* check for unbalances */ + bysort country : egen test3 = total(nrtgrp2016) + bysort country : egen test4 = total(nrtgrp2017) + bysort country : egen test5 = total(nrtgrp2018) + replace test3 = 0 if test3 == . + replace test4 = 0 if test4 == . + replace test5 = 0 if test5 == . + tab urtgrp year if ( test3 != 0 & lgstgrp == 0 & _merge == 1 ) & ( test4 != 0 & lgstgrp == 0 & _merge == 1 ) &( test5 != 0 & lgstgrp == 0 & _merge == 1 ) + drop if ( test3 != 0 & lgstgrp == 0 & _merge == 1 ) & ( test4 != 0 & lgstgrp == 0 & _merge == 1 ) &( test5 != 0 & lgstgrp == 0 & _merge == 1 ) + drop test3 test4 test5 + /* this is the updated masterfile */ + gen merge2015 = _merge + drop _merge + save "${datapath}/EU-SILC/19 L-2023/masterD.dta", replace + /* this is 2015D file for later control */ + keep if merge2015 == 1 + drop merge2015 + save "${datapath}/EU-SILC/11 L-2015/2015D.dta", replace + +/* 2014 */ + /* open the 2014 Household register to get data from rotational groups inactive in more recent releases and their uhid */ + clear + local filen : dir "${datapath}/EU-SILC/10 L-2014/" files "udb_l14d_ver*" + display `filen' + local filel "${datapath}/EU-SILC/10 L-2014/" + display `"`filel'"' + local fileln : display `"`filel'"' `filen' + use `"`fileln'"', clear + + local new_var = lower("`var'") + + foreach var of varlist _all { + local new_var = lower("`var'") + cap rename `var' `new_var' + } + + */ block Stata from displaying IDs in exponential format */ + tostring db030, replace + gen year = db010 + gen country = db020 + gen hid = db030 + gen rotation_group = db075 + sort country hid year + replace country = "EL" if country == "GR" + + /* count the number of rotational groups for each country in this release */ + bysort country rotation_group: gen nvals = _n == 1 + bysort country : egen nrtgrp2014 = total(nvals) + drop nvals + + /* get the rotation group(s) that cover most years */ + bysort country rotation_group : egen maxyear = max(year) + bysort country rotation_group : egen minyear = min(year) + gen years_cov = maxyear - minyear + 1 + bysort country : egen maxgrp = max(years_cov) + gen slctd_rtgrp = 0 + bysort country rotation_group: replace slctd_rtgrp = rotation_group if years_cov == maxgrp + gen lgstgrp = 0 + bysort country rotation_group: replace lgstgrp = 1 if years_cov == maxgrp + + /* build household and rotationgroup IDs that are unique across releases */ + tostring rotation_group, generate(rotation_groupstr) + gen drpout_year = 0 + replace drpout_year = maxyear + 4 - years_cov + + /* check for rotation groups that dropped out before planned */ + bysort country rotation_group : egen maxyear_grp = max(year) + replace drpout_year = maxyear_grp if maxyear_grp < maxyear + + tostring drpout_year, replace + gen uhid = country + rotation_groupstr + drpout_year + hid + gen urtgrp = country + rotation_groupstr + drpout_year + destring drpout_year, replace + + /* check if any selected groups have been selected in the more recent 2016 release ("prolonged rotation groups") */ + sort year country rotation_group hid + merge m:m year country rotation_group using "${datapath}/EU-SILC/12 L-2016/2016D.dta" + replace slctd_rtgrp = rotation_group if _merge == 1 + + /* tag rotation groups that have already been selected in the more recent release in some years (overlapping) */ + gen atag = 0 + replace atag = 1 if _merge == 3 + bysort country rotation_group : egen btag = max(atag) + bysort country rotation_group : egen ndrpoy = max(drpout_year2016) + replace drpout_year = ndrpoy if btag == 1 + drop atag btag ndrpoy + + /* update urtgrp and uhid so that "prolonged rotation groups" maintain their urtgrp across different releases */ + tostring drpout_year, replace + replace uhid = country + rotation_groupstr + drpout_year + hid + replace urtgrp = country + rotation_groupstr + drpout_year + destring drpout_year, replace + drop rotation_groupstr maxyear_grp minyear years_cov maxgrp maxyear_grp maxyear + + /* drop overlapping rotation group years and data from the more recent release */ + keep if _merge == 1 + drop _merge + + /* check if any selected groups have been selected in the more recent 2017 release ("prolonged rotation groups") */ + merge m:m year country urtgrp using "${datapath}/EU-SILC/13 L-2017/2017D.dta" + replace slctd_rtgrp = rotation_group if _merge == 1 + keep if _merge == 1 + drop _merge + + /* check if any selected groups have been selected in the more recent 2015 release ("prolonged rotation groups") */ + merge m:m year country urtgrp using "${datapath}/EU-SILC/11 L-2015/2015D.dta" + replace slctd_rtgrp = rotation_group if _merge == 1 + keep if _merge == 1 + drop _merge + + destring drpout_year, replace + drop if slctd_rtgrp == 0 + + /* for later checks */ + gen drpout_year2014 = drpout_year + gen slctd_urtgrp2014 = urtgrp + gen slctd_uhid2014 = uhid + + /* merge to masterfile */ + merge 1:1 year uhid using "${datapath}/EU-SILC/19 L-2023/masterD.dta" + + /* check for duplicates caused by same rotational groups in terms of same hid, different rotation_group */ + duplicates report year country hid + duplicates tag year country hid, generate(test2) + tab country year if test2 != 0 + tab urtgrp year if test2 != 0 + + /* check if the problem regards a large portion of the rotational group, if the portion is small ignore */ + gen tag1 = 0 + replace tag1 = 1 if test2 != 0 + bysort year country urtgrp : egen ndups = total(tag1) + gen tag2 = 1 + bysort year country urtgrp : egen tot = total(tag2) + gen rdups = ndups / tot + gen tag3 = 0 + replace tag3 = 1 if rdups > 0.5 + drop tag1 tag2 ndups tot rdups + + /* extend selection to whole rotational group */ + bysort year country urtgrp : egen taga = max(tag3) + drop tag3 + + /* check if the rotational group with duplicates is the one covering most years in current release. If yes ignore */ + tab urtgrp year if taga == 1 & lgstgrp == 0 & _merge == 1 + drop if taga == 1 & lgstgrp == 0 & _merge == 1 + drop test2 taga + + /* check for unbalances */ + bysort country : egen test3 = total(nrtgrp2016) + bysort country : egen test4 = total(nrtgrp2017) + bysort country : egen test5 = total(nrtgrp2015) + replace test3 = 0 if test3 == . + replace test4 = 0 if test4 == . + replace test5 = 0 if test5 == . + tab urtgrp year if ( test3 != 0 & lgstgrp == 0 & _merge == 1 ) & ( test4 != 0 & lgstgrp == 0 & _merge == 1 ) &( test5 != 0 & lgstgrp == 0 & _merge == 1 ) + drop if ( test3 != 0 & lgstgrp == 0 & _merge == 1 ) & ( test4 != 0 & lgstgrp == 0 & _merge == 1 ) &( test5 != 0 & lgstgrp == 0 & _merge == 1 ) + drop test3 test4 test5 + + /* this is the updated masterfile */ + gen merge2014 = _merge + drop _merge + save "${datapath}/EU-SILC/19 L-2023/masterD.dta", replace + + /* this is 2014D file for later control */ + keep if merge2014 == 1 + drop merge2014 + save "${datapath}/EU-SILC/10 L-2014/2014D.dta", replace + +/* 2013 */ + /* open the 2013 Household register to get data from rotational groups inactive in more recent releases and their uhid */ + clear + local filen : dir "${datapath}/EU-SILC/9 L-2013/" files "udb_l13d_ver*" + display `filen' + local filel "${datapath}/EU-SILC/9 L-2013/" + display `"`filel'"' + local fileln : display `"`filel'"' `filen' + use `"`fileln'"', clear + + local new_var = lower("`var'") + + foreach var of varlist _all { + local new_var = lower("`var'") + cap rename `var' `new_var' + } + + */ block Stata from displaying IDs in exponential format */ + tostring db030, replace + gen year = db010 + gen country = db020 + gen hid = db030 + gen rotation_group = db075 + sort country hid year + replace country = "EL" if country == "GR" + + /* count the number of rotational groups for each country in this release */ + bysort country rotation_group: gen nvals = _n == 1 + bysort country : egen nrtgrp2013 = total(nvals) + drop nvals + + /* get the rotation group(s) that cover most years */ + bysort country rotation_group : egen maxyear = max(year) + bysort country rotation_group : egen minyear = min(year) + gen years_cov = maxyear - minyear + 1 + bysort country : egen maxgrp = max(years_cov) + gen slctd_rtgrp = 0 + bysort country rotation_group: replace slctd_rtgrp = rotation_group if years_cov == maxgrp + gen lgstgrp = 0 + bysort country rotation_group: replace lgstgrp = 1 if years_cov == maxgrp + + /* build household and rotationgroup IDs that are unique across releases */ + tostring rotation_group, generate(rotation_groupstr) + gen drpout_year = 0 + replace drpout_year = maxyear + 4 - years_cov + + /* check for rotation groups that dropped out before planned */ + bysort country rotation_group : egen maxyear_grp = max(year) + replace drpout_year = maxyear_grp if maxyear_grp < maxyear + tostring drpout_year, replace + gen uhid = country + rotation_groupstr + drpout_year + hid + gen urtgrp = country + rotation_groupstr + drpout_year + destring drpout_year, replace + + /* check if any selected groups have been selected in the more recent 2016 release ("prolonged rotation groups") */ + sort year country rotation_group hid + merge m:m year country rotation_group using "${datapath}/EU-SILC/12 L-2016/2016D.dta" + replace slctd_rtgrp = rotation_group if _merge == 1 + + /* tag rotation groups that have already been selected in the more recent release in some years (overlapping) */ + gen atag = 0 + replace atag = 1 if _merge == 3 + bysort country rotation_group : egen btag = max(atag) + bysort country rotation_group : egen ndrpoy = max(drpout_year2016) + replace drpout_year = ndrpoy if btag == 1 + drop atag btag ndrpoy + /* update urtgrp and uhid so that "prolonged rotation groups" maintain their urtgrp across different releases */ + tostring drpout_year, replace + replace uhid = country + rotation_groupstr + drpout_year + hid + replace urtgrp = country + rotation_groupstr + drpout_year + destring drpout_year, replace + drop rotation_groupstr maxyear_grp minyear years_cov maxgrp maxyear_grp maxyear + + /* drop overlapping rotation group years and data from the more recent release */ + keep if _merge == 1 + drop _merge + + /* check if any selected groups have been selected in the more recent 2014 release ("prolonged rotation groups") */ + merge m:m year country urtgrp using "${datapath}/EU-SILC/10 L-2014/2014D.dta" + replace slctd_rtgrp = rotation_group if _merge == 1 + keep if _merge == 1 + drop _merge + + /* check if any selected groups have been selected in the more recent 2015 release ("prolonged rotation groups") */ + merge m:m year country urtgrp using "${datapath}/EU-SILC/11 L-2015/2015D.dta" + replace slctd_rtgrp = rotation_group if _merge == 1 + keep if _merge == 1 + drop _merge + + destring drpout_year, replace + drop if slctd_rtgrp == 0 + + /* for later checks */ + gen drpout_year2013 = drpout_year + gen slctd_urtgrp2013 = urtgrp + gen slctd_uhid2013 = uhid + + /* merge to masterfile */ + merge 1:1 year uhid using "${datapath}/EU-SILC/19 L-2023/masterD.dta" + + /* check for duplicates caused by same rotational groups in terms of same hid, different rotation_group */ + duplicates report year country hid + duplicates tag year country hid, generate(test2) + tab country year if test2 != 0 + tab urtgrp year if test2 != 0 + + /* check if the problem regards a large portion of the rotational group, if the portion is small ignore */ + gen tag1 = 0 + replace tag1 = 1 if test2 != 0 + bysort year country urtgrp : egen ndups = total(tag1) + gen tag2 = 1 + bysort year country urtgrp : egen tot = total(tag2) + gen rdups = ndups / tot + gen tag3 = 0 + replace tag3 = 1 if rdups > 0.5 + drop tag1 tag2 ndups tot rdups + + /* extend selection to whole rotational group */ + bysort year country urtgrp : egen taga = max(tag3) + drop tag3 + + /* check if the rotational group with duplicates is the one covering most years in current release. If yes ignore */ + tab urtgrp year if taga == 1 & lgstgrp == 0 & _merge == 1 + drop if taga == 1 & lgstgrp == 0 & _merge == 1 + drop test2 taga + + /* check for unbalances */ + bysort country : egen test3 = total(nrtgrp2016) + bysort country : egen test4 = total(nrtgrp2014) + bysort country : egen test5 = total(nrtgrp2015) + replace test3 = 0 if test3 == . + replace test4 = 0 if test4 == . + replace test5 = 0 if test5 == . + tab urtgrp year if ( test3 != 0 & lgstgrp == 0 & _merge == 1 ) & ( test4 != 0 & lgstgrp == 0 & _merge == 1 ) &( test5 != 0 & lgstgrp == 0 & _merge == 1 ) + drop if ( test3 != 0 & lgstgrp == 0 & _merge == 1 ) & ( test4 != 0 & lgstgrp == 0 & _merge == 1 ) &( test5 != 0 & lgstgrp == 0 & _merge == 1 ) + drop test3 test4 test5 + + /* this is the updated masterfile */ + gen merge2013 = _merge + drop _merge + save "${datapath}/EU-SILC/19 L-2023/masterD.dta", replace + + /* this is 2013D file for later control */ + keep if merge2013 == 1 + drop merge2013 + save "${datapath}/EU-SILC/9 L-2013/2013D.dta", replace + +/* 2012 */ + /* open the 2012 Household register to get data from rotational groups inactive in more recent releases and their uhid */ + clear + local filen : dir "${datapath}/EU-SILC/8 L-2012/" files "udb_l12d_ver*" + display `filen' + local filel "${datapath}/EU-SILC/8 L-2012/" + display `"`filel'"' + local fileln : display `"`filel'"' `filen' +use `"`fileln'"', clear + + + local new_var = lower("`var'") + + foreach var of varlist _all { + local new_var = lower("`var'") + cap rename `var' `new_var' + } + + */ block Stata from displaying IDs in exponential format */ + tostring db030, replace + gen year = db010 + gen country = db020 + gen hid = db030 + gen rotation_group = db075 + sort country hid year + replace country = "EL" if country == "GR" + + /* count the number of rotational groups for each country in this release */ + bysort country rotation_group: gen nvals = _n == 1 + bysort country : egen nrtgrp2012 = total(nvals) + drop nvals + + /* get the rotation group(s) that cover most years */ + bysort country rotation_group : egen maxyear = max(year) + bysort country rotation_group : egen minyear = min(year) + gen years_cov = maxyear - minyear + 1 + bysort country : egen maxgrp = max(years_cov) + gen slctd_rtgrp = 0 + bysort country rotation_group: replace slctd_rtgrp = rotation_group if years_cov == maxgrp + gen lgstgrp = 0 + bysort country rotation_group: replace lgstgrp = 1 if years_cov == maxgrp + + /* build household and rotationgroup IDs that are unique across releases */ + tostring rotation_group, generate(rotation_groupstr) + gen drpout_year = 0 + replace drpout_year = maxyear + 4 - years_cov + + /* check for rotation groups that dropped out before planned */ + bysort country rotation_group : egen maxyear_grp = max(year) + replace drpout_year = maxyear_grp if maxyear_grp < maxyear + tostring drpout_year, replace + + gen uhid = country + rotation_groupstr + drpout_year + hid + gen urtgrp = country + rotation_groupstr + drpout_year + destring drpout_year, replace + + /* check if any selected groups have been selected in the more recent 2013 release ("prolonged rotation groups") */ + sort year country rotation_group hid + merge m:m year country rotation_group using "${datapath}/EU-SILC/9 L-2013/2013D.dta" + replace slctd_rtgrp = rotation_group if _merge == 1 + + /* tag rotation groups that have already been selected in the more recent release in some years (overlapping) */ + gen atag = 0 + replace atag = 1 if _merge == 3 + bysort country rotation_group : egen btag = max(atag) + bysort country rotation_group : egen ndrpoy = max(drpout_year2013) + replace drpout_year = ndrpoy if btag == 1 + drop atag btag ndrpoy + + /* update urtgrp and uhid so that "prolonged rotation groups" maintain their urtgrp across different releases */ + tostring drpout_year, replace + replace uhid = country + rotation_groupstr + drpout_year + hid + replace urtgrp = country + rotation_groupstr + drpout_year + destring drpout_year, replace + drop rotation_groupstr maxyear_grp minyear years_cov maxgrp maxyear_grp maxyear + + /* drop overlapping rotation group years and data from the more recent release */ + keep if _merge == 1 + drop _merge + + /* check if any selected groups have been selected in the more recent 2014 release ("prolonged rotation groups") */ + merge m:m year country urtgrp using "${datapath}/EU-SILC/10 L-2014/2014D.dta" + replace slctd_rtgrp = rotation_group if _merge == 1 + keep if _merge == 1 + drop _merge + + /* check if any selected groups have been selected in the more recent 2015 release ("prolonged rotation groups") */ + merge m:m year country urtgrp using "${datapath}/EU-SILC/11 L-2015/2015D.dta" + replace slctd_rtgrp = rotation_group if _merge == 1 + keep if _merge == 1 + drop _merge + + destring drpout_year, replace + drop if slctd_rtgrp == 0 + + /* for later checks */ + gen drpout_year2012 = drpout_year + gen slctd_urtgrp2012 = urtgrp + gen slctd_uhid2012 = uhid + + /* merge to masterfile */ + merge 1:1 year uhid using "${datapath}/EU-SILC/19 L-2023/masterD.dta" + + /* check for duplicates caused by same rotational groups in terms of same hid, different rotation_group */ + duplicates report year country hid + duplicates tag year country hid, generate(test2) + tab country year if test2 != 0 + tab urtgrp year if test2 != 0 + + /* check if the problem regards a large portion of the rotational group, if the portion is small ignore */ + gen tag1 = 0 + replace tag1 = 1 if test2 != 0 + bysort year country urtgrp : egen ndups = total(tag1) + gen tag2 = 1 + bysort year country urtgrp : egen tot = total(tag2) + gen rdups = ndups / tot + gen tag3 = 0 + replace tag3 = 1 if rdups > 0.5 + drop tag1 tag2 ndups tot rdups + + /* extend selection to whole rotational group */ + bysort year country urtgrp : egen taga = max(tag3) + drop tag3 + + /* check if the rotational group with duplicates is the one covering most years in current release. If yes ignore */ + tab urtgrp year if taga == 1 & lgstgrp == 0 & _merge == 1 + drop if taga == 1 & lgstgrp == 0 & _merge == 1 + drop test2 taga + + /* check for unbalances */ + bysort country : egen test3 = total(nrtgrp2013) + bysort country : egen test4 = total(nrtgrp2014) + bysort country : egen test5 = total(nrtgrp2015) + replace test3 = 0 if test3 == . + replace test4 = 0 if test4 == . + replace test5 = 0 if test5 == . + tab urtgrp year if ( test3 != 0 & lgstgrp == 0 & _merge == 1 ) & ( test4 != 0 & lgstgrp == 0 & _merge == 1 ) &( test5 != 0 & lgstgrp == 0 & _merge == 1 ) + drop if ( test3 != 0 & lgstgrp == 0 & _merge == 1 ) & ( test4 != 0 & lgstgrp == 0 & _merge == 1 ) & ( test5 != 0 & lgstgrp == 0 & _merge == 1 ) + drop test3 test4 test5 + + /* this is the updated masterfile */ + gen merge2012 = _merge + drop _merge + save "${datapath}/EU-SILC/19 L-2023/masterD.dta", replace + + /* this is 2012D file for later control */ + keep if merge2012 == 1 + drop merge2012 + save "${datapath}/EU-SILC/8 L-2012/2012D.dta", replace + + +/* 2011 */ + /* open the 2011 Household register to get data from rotational groups inactive in more recent releases and their uhid */ + clear + local filen : dir "${datapath}/EU-SILC/7 L-2011/" files "udb_l11d_ver*" + display `filen' + local filel "${datapath}/EU-SILC/7 L-2011/" + display `"`filel'"' + local fileln : display `"`filel'"' `filen' + use `"`fileln'"', clear + + local new_var = lower("`var'") + + foreach var of varlist _all { + local new_var = lower("`var'") + cap rename `var' `new_var' + } + + */ block Stata from displaying IDs in exponential format */ + tostring db030, replace + gen year = db010 + gen country = db020 + gen hid = db030 + gen rotation_group = db075 + sort country hid year + replace country = "EL" if country == "GR" + + /* count the number of rotational groups for each country in this release */ + bysort country rotation_group: gen nvals = _n == 1 + bysort country : egen nrtgrp2011 = total(nvals) + drop nvals + + /* get the rotation group(s) that cover most years */ + bysort country rotation_group : egen maxyear = max(year) + bysort country rotation_group : egen minyear = min(year) + gen years_cov = maxyear - minyear + 1 + bysort country : egen maxgrp = max(years_cov) + gen slctd_rtgrp = 0 + bysort country rotation_group: replace slctd_rtgrp = rotation_group if years_cov == maxgrp + gen lgstgrp = 0 + bysort country rotation_group: replace lgstgrp = 1 if years_cov == maxgrp + + /* build household and rotationgroup IDs that are unique across releases */ + tostring rotation_group, generate(rotation_groupstr) + gen drpout_year = 0 + replace drpout_year = maxyear + 4 - years_cov + + /* check for rotation groups that dropped out before planned */ + bysort country rotation_group : egen maxyear_grp = max(year) + replace drpout_year = maxyear_grp if maxyear_grp < maxyear + + tostring drpout_year, replace + gen uhid = country + rotation_groupstr + drpout_year + hid + gen urtgrp = country + rotation_groupstr + drpout_year + destring drpout_year, replace + + /* check if any selected groups have been selected in the more recent 2012 release ("prolonged rotation groups") */ + sort year country rotation_group hid + merge m:m year country rotation_group using "${datapath}/EU-SILC/8 L-2012/2012D.dta" + replace slctd_rtgrp = rotation_group if _merge == 1 + + /* tag rotation groups that have already been selected in the more recent release in some years (overlapping) */ + gen atag = 0 + replace atag = 1 if _merge == 3 + bysort country rotation_group : egen btag = max(atag) + bysort country rotation_group : egen ndrpoy = max(drpout_year2012) + replace drpout_year = ndrpoy if btag == 1 + drop atag btag ndrpoy + + /* update urtgrp and uhid so that "prolonged rotation groups" maintain their urtgrp across different releases */ + tostring drpout_year, replace + replace uhid = country + rotation_groupstr + drpout_year + hid + replace urtgrp = country + rotation_groupstr + drpout_year + destring drpout_year, replace + drop rotation_groupstr maxyear_grp minyear years_cov maxgrp maxyear_grp maxyear + + /* drop overlapping rotation group years and data from the more recent release */ + keep if _merge == 1 + drop _merge + + /* check if any selected groups have been selected in the more recent 2013 release ("prolonged rotation groups") */ + merge m:m year country urtgrp using "${datapath}/EU-SILC/9 L-2013/2013D.dta" + replace slctd_rtgrp = rotation_group if _merge == 1 + keep if _merge == 1 + drop _merge + + /* check if any selected groups have been selected in the more recent 2014 release ("prolonged rotation groups") */ + merge m:m year country urtgrp using "${datapath}/EU-SILC/10 L-2014/2014D.dta" + replace slctd_rtgrp = rotation_group if _merge == 1 + keep if _merge == 1 + drop _merge + + destring drpout_year, replace + drop if slctd_rtgrp == 0 + + /* for later checks */ + gen drpout_year2011 = drpout_year + gen slctd_urtgrp2011 = urtgrp + gen slctd_uhid2011 = uhid + + /* merge to masterfile */ + merge 1:1 year uhid using "${datapath}/EU-SILC/19 L-2023/masterD.dta" + + /* check for duplicates caused by same rotational groups in terms of same hid, different rotation_group */ + duplicates report year country hid + duplicates tag year country hid, generate(test2) + tab country year if test2 != 0 + tab urtgrp year if test2 != 0 + + /* check if the problem regards a large portion of the rotational group, if the portion is small ignore */ + gen tag1 = 0 + replace tag1 = 1 if test2 != 0 + bysort year country urtgrp : egen ndups = total(tag1) + gen tag2 = 1 + bysort year country urtgrp : egen tot = total(tag2) + gen rdups = ndups / tot + gen tag3 = 0 + replace tag3 = 1 if rdups > 0.5 + drop tag1 tag2 ndups tot rdups + + /* extend selection to whole rotational group */ + bysort year country urtgrp : egen taga = max(tag3) + drop tag3 + + /* check if the rotational group with duplicates is the one covering most years in current release. If yes ignore */ + tab urtgrp year if taga == 1 & lgstgrp == 0 & _merge == 1 + drop if taga == 1 & lgstgrp == 0 & _merge == 1 + drop test2 taga + + /* check for unbalances */ + bysort country : egen test3 = total(nrtgrp2012) + bysort country : egen test4 = total(nrtgrp2013) + bysort country : egen test5 = total(nrtgrp2014) + replace test3 = 0 if test3 == . + replace test4 = 0 if test4 == . + replace test5 = 0 if test5 == . + tab urtgrp year if ( test3 != 0 & lgstgrp == 0 & _merge == 1 ) & ( test4 != 0 & lgstgrp == 0 & _merge == 1 ) &( test5 != 0 & lgstgrp == 0 & _merge == 1 ) + drop if ( test3 != 0 & lgstgrp == 0 & _merge == 1 ) & ( test4 != 0 & lgstgrp == 0 & _merge == 1 ) &( test5 != 0 & lgstgrp == 0 & _merge == 1 ) + drop test3 test4 test5 + + /* this is the updated masterfile */ + gen merge2011 = _merge + drop _merge + save "${datapath}/EU-SILC/19 L-2023/masterD.dta", replace + + /* this is 2011D file for later control */ + keep if merge2011 == 1 + drop merge2011 + save "${datapath}/EU-SILC/7 L-2011/2011D.dta", replace + + +/* 2010 */ + /* open the 2010 Household register to get data from rotational groups inactive in more recent releases and their uhid */ + clear + local filen : dir "${datapath}/EU-SILC/6 L-2010/" files "udb_l10d_ver*" + display `filen' + local filel "${datapath}/EU-SILC/6 L-2010/" + display `"`filel'"' + local fileln : display `"`filel'"' `filen' + use `"`fileln'"', clear + + local new_var = lower("`var'") + + foreach var of varlist _all { + local new_var = lower("`var'") + cap rename `var' `new_var' + } + + */ block Stata from displaying IDs in exponential format */ + tostring db030, replace + gen year = db010 + gen country = db020 + gen hid = db030 + gen rotation_group = db075 + sort country hid year + replace country = "EL" if country == "GR" + + /* count the number of rotational groups for each country in this release */ + bysort country rotation_group: gen nvals = _n == 1 + bysort country : egen nrtgrp2010 = total(nvals) + drop nvals + + /* get the rotation group(s) that cover most years */ + bysort country rotation_group : egen maxyear = max(year) + bysort country rotation_group : egen minyear = min(year) + gen years_cov = maxyear - minyear + 1 + bysort country : egen maxgrp = max(years_cov) + gen slctd_rtgrp = 0 + bysort country rotation_group: replace slctd_rtgrp = rotation_group if years_cov == maxgrp + gen lgstgrp = 0 + bysort country rotation_group: replace lgstgrp = 1 if years_cov == maxgrp + + /* build household and rotationgroup IDs that are unique across releases */ + tostring rotation_group, generate(rotation_groupstr) + gen drpout_year = 0 + replace drpout_year = maxyear + 4 - years_cov + + /* check for rotation groups that dropped out before planned */ + bysort country rotation_group : egen maxyear_grp = max(year) + replace drpout_year = maxyear_grp if maxyear_grp < maxyear + + tostring drpout_year, replace + gen uhid = country + rotation_groupstr + drpout_year + hid + gen urtgrp = country + rotation_groupstr + drpout_year + destring drpout_year, replace + + /* check if any selected groups have been selected in the more recent 2011 release ("prolonged rotation groups") */ + sort year country rotation_group hid + merge m:m year country rotation_group using "${datapath}/EU-SILC/7 L-2011/2011D.dta" + replace slctd_rtgrp = rotation_group if _merge == 1 + + /* tag rotation groups that have already been selected in the more recent release in some years (overlapping) */ + gen atag = 0 + replace atag = 1 if _merge == 3 + bysort country rotation_group : egen btag = max(atag) + bysort country rotation_group : egen ndrpoy = max(drpout_year2011) + replace drpout_year = ndrpoy if btag == 1 + drop atag btag ndrpoy + + /* update urtgrp and uhid so that "prolonged rotation groups" maintain their urtgrp across different releases */ + tostring drpout_year, replace + replace uhid = country + rotation_groupstr + drpout_year + hid + replace urtgrp = country + rotation_groupstr + drpout_year + destring drpout_year, replace + drop rotation_groupstr maxyear_grp minyear years_cov maxgrp maxyear_grp maxyear + + /* drop overlapping rotation group years and data from the more recent release */ + keep if _merge == 1 + drop _merge + + /* check if any selected groups have been selected in the more recent 2012 release ("prolonged rotation groups") */ + merge m:m year country urtgrp using "${datapath}/EU-SILC/8 L-2012/2012D.dta" + replace slctd_rtgrp = rotation_group if _merge == 1 + keep if _merge == 1 + drop _merge + + /* check if any selected groups have been selected in the more recent 2013 release ("prolonged rotation groups") */ + merge m:m year country urtgrp using "${datapath}/EU-SILC/9 L-2013/2013D.dta" + replace slctd_rtgrp = rotation_group if _merge == 1 + keep if _merge == 1 + drop _merge + + destring drpout_year, replace + drop if slctd_rtgrp == 0 + + /* for later checks */ + gen drpout_year2010 = drpout_year + gen slctd_urtgrp2010 = urtgrp + gen slctd_uhid2010 = uhid + + /* merge to masterfile */ + merge 1:1 year uhid using "${datapath}/EU-SILC/19 L-2023/masterD.dta" + + /* check for duplicates caused by same rotational groups in terms of same hid, different rotation_group */ + duplicates report year country hid + duplicates tag year country hid, generate(test2) + tab country year if test2 != 0 + tab urtgrp year if test2 != 0 + + /* check if the problem regards a large portion of the rotational group, if the portion is small ignore */ + gen tag1 = 0 + replace tag1 = 1 if test2 != 0 + bysort year country urtgrp : egen ndups = total(tag1) + gen tag2 = 1 + bysort year country urtgrp : egen tot = total(tag2) + gen rdups = ndups / tot + gen tag3 = 0 + replace tag3 = 1 if rdups > 0.5 + drop tag1 tag2 ndups tot rdups + + /* extend selection to whole rotational group */ + bysort year country urtgrp : egen taga = max(tag3) + drop tag3 + + /* check if the rotational group with duplicates is the one covering most years in current release. If yes ignore */ + tab urtgrp year if taga == 1 & lgstgrp == 0 & _merge == 1 + drop if taga == 1 & lgstgrp == 0 & _merge == 1 + drop test2 taga + + /* check for unbalances */ + bysort country : egen test3 = total(nrtgrp2011) + bysort country : egen test4 = total(nrtgrp2012) + bysort country : egen test5 = total(nrtgrp2013) + replace test3 = 0 if test3 == . + replace test4 = 0 if test4 == . + replace test5 = 0 if test5 == . + tab urtgrp year if ( test3 != 0 & lgstgrp == 0 & _merge == 1 ) & ( test4 != 0 & lgstgrp == 0 & _merge == 1 ) &( test5 != 0 & lgstgrp == 0 & _merge == 1 ) + drop if ( test3 != 0 & lgstgrp == 0 & _merge == 1 ) & ( test4 != 0 & lgstgrp == 0 & _merge == 1 ) &( test5 != 0 & lgstgrp == 0 & _merge == 1 ) + drop test3 test4 test5 + + /* this is the updated masterfile */ + gen merge2010 = _merge + drop _merge + save "${datapath}/EU-SILC/19 L-2023/masterD.dta", replace + + /* this is 2010D file for later control */ + keep if merge2010 == 1 + drop merge2010 + save "${datapath}/EU-SILC/6 L-2010/2010D.dta", replace + + +/* 2009 */ + /* open the 2009 Household register to get data from rotational groups inactive in more recent releases and their uhid */ + clear + local filen : dir "${datapath}/EU-SILC/5 L-2009/" files "udb_l09d_ver*" + display `filen' + local filel "${datapath}/EU-SILC/5 L-2009/" + display `"`filel'"' + local fileln : display `"`filel'"' `filen' + use `"`fileln'"', clear + + local new_var = lower("`var'") + + foreach var of varlist _all { + local new_var = lower("`var'") + cap rename `var' `new_var' + } + + */ block Stata from displaying IDs in exponential format */ + tostring db030, replace + gen year = db010 + gen country = db020 + gen hid = db030 + gen rotation_group = db075 + sort country hid year + replace country = "EL" if country == "GR" + + /* count the number of rotational groups for each country in this release */ + bysort country rotation_group: gen nvals = _n == 1 + bysort country : egen nrtgrp2009 = total(nvals) + drop nvals + + /* get the rotation group(s) that cover most years */ + bysort country rotation_group : egen maxyear = max(year) + bysort country rotation_group : egen minyear = min(year) + gen years_cov = maxyear - minyear + 1 + bysort country : egen maxgrp = max(years_cov) + gen slctd_rtgrp = 0 + bysort country rotation_group: replace slctd_rtgrp = rotation_group if years_cov == maxgrp + gen lgstgrp = 0 + bysort country rotation_group: replace lgstgrp = 1 if years_cov == maxgrp + + /* build household and rotationgroup IDs that are unique across releases */ + tostring rotation_group, generate(rotation_groupstr) + gen drpout_year = 0 + replace drpout_year = maxyear + 4 - years_cov + + /* check for rotation groups that dropped out before planned */ + bysort country rotation_group : egen maxyear_grp = max(year) + replace drpout_year = maxyear_grp if maxyear_grp < maxyear + + tostring drpout_year, replace + gen uhid = country + rotation_groupstr + drpout_year + hid + gen urtgrp = country + rotation_groupstr + drpout_year + destring drpout_year, replace + + /* check if any selected groups have been selected in the more recent 2010 release ("prolonged rotation groups") */ + sort year country rotation_group hid + merge m:m year country rotation_group using "${datapath}/EU-SILC/6 L-2010/2010D.dta" + replace slctd_rtgrp = rotation_group if _merge == 1 + + /* tag rotation groups that have already been selected in the more recent release in some years (overlapping) */ + gen atag = 0 + replace atag = 1 if _merge == 3 + bysort country rotation_group : egen btag = max(atag) + bysort country rotation_group : egen ndrpoy = max(drpout_year2010) + replace drpout_year = ndrpoy if btag == 1 + drop atag btag ndrpoy + + /* update urtgrp and uhid so that "prolonged rotation groups" maintain their urtgrp across different releases */ + tostring drpout_year, replace + replace uhid = country + rotation_groupstr + drpout_year + hid + replace urtgrp = country + rotation_groupstr + drpout_year + destring drpout_year, replace + drop rotation_groupstr maxyear_grp minyear years_cov maxgrp maxyear_grp maxyear + + /* drop overlapping rotation group years and data from the more recent release */ + keep if _merge == 1 + drop _merge + + /* check if any selected groups have been selected in the more recent 2011 release ("prolonged rotation groups") */ + merge m:m year country urtgrp using "${datapath}/EU-SILC/7 L-2011/2011D.dta" + replace slctd_rtgrp = rotation_group if _merge == 1 + keep if _merge == 1 + drop _merge + + /* check if any selected groups have been selected in the more recent 2012 release ("prolonged rotation groups") */ + merge m:m year country urtgrp using "${datapath}/EU-SILC/8 L-2012/2012D.dta" + replace slctd_rtgrp = rotation_group if _merge == 1 + keep if _merge == 1 + drop _merge + + destring drpout_year, replace + drop if slctd_rtgrp == 0 + + /* for later checks */ + gen drpout_year2009 = drpout_year + gen slctd_urtgrp2009 = urtgrp + gen slctd_uhid2009 = uhid + + /* merge to masterfile */ + merge 1:1 year uhid using "${datapath}/EU-SILC/19 L-2023/masterD.dta" + + /* check for duplicates caused by same rotational groups in terms of same hid, different rotation_group */ + duplicates report year country hid + duplicates tag year country hid, generate(test2) + tab country year if test2 != 0 + tab urtgrp year if test2 != 0 + + /* check if the problem regards a large portion of the rotational group, if the portion is small ignore */ + gen tag1 = 0 + replace tag1 = 1 if test2 != 0 + bysort year country urtgrp : egen ndups = total(tag1) + gen tag2 = 1 + bysort year country urtgrp : egen tot = total(tag2) + gen rdups = ndups / tot + gen tag3 = 0 + replace tag3 = 1 if rdups > 0.5 + drop tag1 tag2 ndups tot rdups + + /* extend selection to whole rotational group */ + bysort year country urtgrp : egen taga = max(tag3) + drop tag3 + + /* check if the rotational group with duplicates is the one covering most years in current release. If yes ignore */ + tab urtgrp year if taga == 1 & lgstgrp == 0 & _merge == 1 + drop if taga == 1 & lgstgrp == 0 & _merge == 1 + drop test2 taga + + /* check for unbalances */ + bysort country : egen test3 = total(nrtgrp2010) + bysort country : egen test4 = total(nrtgrp2011) + bysort country : egen test5 = total(nrtgrp2012) + replace test3 = 0 if test3 == . + replace test4 = 0 if test4 == . + replace test5 = 0 if test5 == . + tab urtgrp year if ( test3 != 0 & lgstgrp == 0 & _merge == 1 ) & ( test4 != 0 & lgstgrp == 0 & _merge == 1 ) &( test5 != 0 & lgstgrp == 0 & _merge == 1 ) + drop if ( test3 != 0 & lgstgrp == 0 & _merge == 1 ) & ( test4 != 0 & lgstgrp == 0 & _merge == 1 ) &( test5 != 0 & lgstgrp == 0 & _merge == 1 ) + drop test3 test4 test5 + + /* this is the updated masterfile */ + gen merge2009 = _merge + drop _merge + save "${datapath}/EU-SILC/19 L-2023/masterD.dta", replace + + /* this is 2009D file for later control */ + keep if merge2009 == 1 + drop merge2009 + save "${datapath}/EU-SILC/5 L-2009/2009D.dta", replace + + +/* 2008 */ + /* open the 2008 Household register to get data from rotational groups inactive in more recent releases and their uhid */ + clear + local filen : dir "${datapath}/EU-SILC/4 L-2008/" files "udb_l08d_ver*" + display `filen' + local filel "${datapath}/EU-SILC/4 L-2008/" + display `"`filel'"' + local fileln : display `"`filel'"' `filen' + use `"`fileln'"', clear + + local new_var = lower("`var'") + + foreach var of varlist _all { + local new_var = lower("`var'") + cap rename `var' `new_var' + } + + */ block Stata from displaying IDs in exponential format */ + tostring db030, replace + gen year = db010 + gen country = db020 + gen hid = db030 + gen rotation_group = db075 + sort country hid year + replace country = "EL" if country == "GR" + + /* count the number of rotational groups for each country in this release */ + bysort country rotation_group: gen nvals = _n == 1 + bysort country : egen nrtgrp2008 = total(nvals) + drop nvals + + /* get the rotation group(s) that cover most years */ + bysort country rotation_group : egen maxyear = max(year) + bysort country rotation_group : egen minyear = min(year) + gen years_cov = maxyear - minyear + 1 + bysort country : egen maxgrp = max(years_cov) + gen slctd_rtgrp = 0 + bysort country rotation_group: replace slctd_rtgrp = rotation_group if years_cov == maxgrp + gen lgstgrp = 0 + bysort country rotation_group: replace lgstgrp = 1 if years_cov == maxgrp + + /* build household and rotationgroup IDs that are unique across releases */ + tostring rotation_group, generate(rotation_groupstr) + gen drpout_year = 0 + replace drpout_year = maxyear + 4 - years_cov + + /* check for rotation groups that dropped out before planned */ + bysort country rotation_group : egen maxyear_grp = max(year) + replace drpout_year = maxyear_grp if maxyear_grp < maxyear + + tostring drpout_year, replace + gen uhid = country + rotation_groupstr + drpout_year + hid + gen urtgrp = country + rotation_groupstr + drpout_year + destring drpout_year, replace + + /* check if any selected groups have been selected in the more recent 2009 release ("prolonged rotation groups") */ + sort year country rotation_group hid + merge m:m year country rotation_group using "${datapath}/EU-SILC/5 L-2009/2009D.dta" + replace slctd_rtgrp = rotation_group if _merge == 1 + + /* tag rotation groups that have already been selected in the more recent release in some years (overlapping) */ + gen atag = 0 + replace atag = 1 if _merge == 3 + bysort country rotation_group : egen btag = max(atag) + bysort country rotation_group : egen ndrpoy = max(drpout_year2009) + replace drpout_year = ndrpoy if btag == 1 + drop atag btag ndrpoy + + /* update urtgrp and uhid so that "prolonged rotation groups" maintain their urtgrp across different releases */ + tostring drpout_year, replace + replace uhid = country + rotation_groupstr + drpout_year + hid + replace urtgrp = country + rotation_groupstr + drpout_year + destring drpout_year, replace + drop rotation_groupstr maxyear_grp minyear years_cov maxgrp maxyear_grp maxyear + + /* drop overlapping rotation group years and data from the more recent release */ + keep if _merge == 1 + drop _merge + + /* check if any selected groups have been selected in the more recent 2010 release ("prolonged rotation groups") */ + merge m:m year country urtgrp using "${datapath}/EU-SILC/6 L-2010/2010D.dta" + replace slctd_rtgrp = rotation_group if _merge == 1 + keep if _merge == 1 + drop _merge + + /* check if any selected groups have been selected in the more recent 2011 release ("prolonged rotation groups") */ + merge m:m year country urtgrp using "${datapath}/EU-SILC/7 L-2011/2011D.dta" + replace slctd_rtgrp = rotation_group if _merge == 1 + keep if _merge == 1 + drop _merge + + destring drpout_year, replace + drop if slctd_rtgrp == 0 + + /* for later checks */ + gen drpout_year2008 = drpout_year + gen slctd_urtgrp2008 = urtgrp + gen slctd_uhid2008 = uhid + + /* merge to masterfile */ + merge 1:1 year uhid using "${datapath}/EU-SILC/19 L-2023/masterD.dta" + + /* check for duplicates caused by same rotational groups in terms of same hid, different rotation_group */ + duplicates report year country hid + duplicates tag year country hid, generate(test2) + tab country year if test2 != 0 + tab urtgrp year if test2 != 0 + + /* check if the problem regards a large portion of the rotational group, if the portion is small ignore */ + gen tag1 = 0 + replace tag1 = 1 if test2 != 0 + bysort year country urtgrp : egen ndups = total(tag1) + gen tag2 = 1 + bysort year country urtgrp : egen tot = total(tag2) + gen rdups = ndups / tot + gen tag3 = 0 + replace tag3 = 1 if rdups > 0.5 + drop tag1 tag2 ndups tot rdups + + /* extend selection to whole rotational group */ + bysort year country urtgrp : egen taga = max(tag3) + drop tag3 + + /* check if the rotational group with duplicates is the one covering most years in current release. If yes ignore */ + tab urtgrp year if taga == 1 & lgstgrp == 0 & _merge == 1 + drop if taga == 1 & lgstgrp == 0 & _merge == 1 + drop test2 taga + + /* check for unbalances */ + bysort country : egen test3 = total(nrtgrp2009) + bysort country : egen test4 = total(nrtgrp2010) + bysort country : egen test5 = total(nrtgrp2011) + replace test3 = 0 if test3 == . + replace test4 = 0 if test4 == . + replace test5 = 0 if test5 == . + tab urtgrp year if ( test3 != 0 & lgstgrp == 0 & _merge == 1 ) & ( test4 != 0 & lgstgrp == 0 & _merge == 1 ) &( test5 != 0 & lgstgrp == 0 & _merge == 1 ) + drop if ( test3 != 0 & lgstgrp == 0 & _merge == 1 ) & ( test4 != 0 & lgstgrp == 0 & _merge == 1 ) &( test5 != 0 & lgstgrp == 0 & _merge == 1 ) + drop test3 test4 test5 + + /* this is the updated masterfile */ + gen merge2008 = _merge + drop _merge + save "${datapath}/EU-SILC/19 L-2023/masterD.dta", replace + + /* this is 2008D file for later control */ + keep if merge2008 == 1 + drop merge2008 + save "${datapath}/EU-SILC/4 L-2008/2008D.dta", replace + + +/* 2007 */ + /* open the 2007 Household register to get data from rotational groups inactive in more recent releases and their uhid */ + clear + local filen : dir "${datapath}/EU-SILC/3 L-2007/" files "udb_l07d_ver*" + display `filen' + local filel "${datapath}/EU-SILC/3 L-2007/" + display `"`filel'"' + local fileln : display `"`filel'"' `filen' + use `"`fileln'"', clear + + local new_var = lower("`var'") + + foreach var of varlist _all { + local new_var = lower("`var'") + cap rename `var' `new_var' + } + + */ block Stata from displaying IDs in exponential format */ + tostring db030, replace + gen year = db010 + gen country = db020 + gen hid = db030 + gen rotation_group = db075 + sort country hid year + replace country = "EL" if country == "GR" + + /* count the number of rotational groups for each country in this release */ + bysort country rotation_group: gen nvals = _n == 1 + bysort country : egen nrtgrp2007 = total(nvals) + drop nvals + + /* get the rotation group(s) that cover most years */ + bysort country rotation_group : egen maxyear = max(year) + bysort country rotation_group : egen minyear = min(year) + gen years_cov = maxyear - minyear + 1 + bysort country : egen maxgrp = max(years_cov) + gen slctd_rtgrp = 0 + bysort country rotation_group: replace slctd_rtgrp = rotation_group if years_cov == maxgrp + gen lgstgrp = 0 + bysort country rotation_group: replace lgstgrp = 1 if years_cov == maxgrp + + /* build household and rotationgroup IDs that are unique across releases */ + tostring rotation_group, generate(rotation_groupstr) + gen drpout_year = 0 + replace drpout_year = maxyear + 4 - years_cov + + /* check for rotation groups that dropped out before planned */ + bysort country rotation_group : egen maxyear_grp = max(year) + replace drpout_year = maxyear_grp if maxyear_grp < maxyear + + tostring drpout_year, replace + gen uhid = country + rotation_groupstr + drpout_year + hid + gen urtgrp = country + rotation_groupstr + drpout_year + destring drpout_year, replace + + /* check if any selected groups have been selected in the more recent 2008 release ("prolonged rotation groups") */ + sort year country rotation_group hid + merge m:m year country rotation_group using "${datapath}/EU-SILC/4 L-2008/2008D.dta" + replace slctd_rtgrp = rotation_group if _merge == 1 + + /* tag rotation groups that have already been selected in the more recent release in some years (overlapping) */ + gen atag = 0 + replace atag = 1 if _merge == 3 + bysort country rotation_group : egen btag = max(atag) + bysort country rotation_group : egen ndrpoy = max(drpout_year2008) + replace drpout_year = ndrpoy if btag == 1 + drop atag btag ndrpoy + + /* update urtgrp and uhid so that "prolonged rotation groups" maintain their urtgrp across different releases */ + tostring drpout_year, replace + replace uhid = country + rotation_groupstr + drpout_year + hid + replace urtgrp = country + rotation_groupstr + drpout_year + destring drpout_year, replace + drop rotation_groupstr maxyear_grp minyear years_cov maxgrp maxyear_grp maxyear + + /* drop overlapping rotation group years and data from the more recent release */ + keep if _merge == 1 + drop _merge + + /* check if any selected groups have been selected in the more recent 2009 release ("prolonged rotation groups") */ + merge m:m year country urtgrp using "${datapath}/EU-SILC/5 L-2009/2009D.dta" + replace slctd_rtgrp = rotation_group if _merge == 1 + keep if _merge == 1 + drop _merge + + /* check if any selected groups have been selected in the more recent 2010 release ("prolonged rotation groups") */ + merge m:m year country urtgrp using "${datapath}/EU-SILC/6 L-2010/2010D.dta" + replace slctd_rtgrp = rotation_group if _merge == 1 + keep if _merge == 1 + drop _merge + destring drpout_year, replace + drop if slctd_rtgrp == 0 + + /* for later checks */ + gen drpout_year2007 = drpout_year + gen slctd_urtgrp2007 = urtgrp + gen slctd_uhid2007 = uhid + + /* merge to masterfile */ + merge 1:1 year uhid using "${datapath}/EU-SILC/19 L-2023/masterD.dta" + + /* check for duplicates caused by same rotational groups in terms of same hid, different rotation_group */ + duplicates report year country hid + duplicates tag year country hid, generate(test2) + tab country year if test2 != 0 + tab urtgrp year if test2 != 0 + + /* check if the problem regards a large portion of the rotational group, if the portion is small ignore */ + gen tag1 = 0 + replace tag1 = 1 if test2 != 0 + bysort year country urtgrp : egen ndups = total(tag1) + gen tag2 = 1 + bysort year country urtgrp : egen tot = total(tag2) + gen rdups = ndups / tot + gen tag3 = 0 + replace tag3 = 1 if rdups > 0.5 + drop tag1 tag2 ndups tot rdups + + /* extend selection to whole rotational group */ + bysort year country urtgrp : egen taga = max(tag3) + drop tag3 + + /* check if the rotational group with duplicates is the one covering most years in current release. If yes ignore */ + tab urtgrp year if taga == 1 & lgstgrp == 0 & _merge == 1 + drop if taga == 1 & lgstgrp == 0 & _merge == 1 + drop test2 taga + + /* check for unbalances */ + bysort country : egen test3 = total(nrtgrp2008) + bysort country : egen test4 = total(nrtgrp2009) + bysort country : egen test5 = total(nrtgrp2010) + replace test3 = 0 if test3 == . + replace test4 = 0 if test4 == . + replace test5 = 0 if test5 == . + tab urtgrp year if ( test3 != 0 & lgstgrp == 0 & _merge == 1 ) & ( test4 != 0 & lgstgrp == 0 & _merge == 1 ) & ( test5 != 0 & lgstgrp == 0 & _merge == 1 ) + drop if ( test3 != 0 & lgstgrp == 0 & _merge == 1 ) & ( test4 != 0 & lgstgrp == 0 & _merge == 1 ) & ( test5 != 0 & lgstgrp == 0 & _merge == 1 ) + drop test3 test4 test5 + + /* this is the updated masterfile */ + gen merge2007 = _merge + drop _merge + save "${datapath}/EU-SILC/19 L-2023/masterD.dta", replace + + /* this is 2007D file for later control */ + keep if merge2007 == 1 + drop merge2007 + save "${datapath}/EU-SILC/3 L-2007/2007D.dta", replace + +/* 2006 */ + /* open the 2006 Household register to get data from rotational groups inactive in more recent releases and their uhid */ + clear + local filen : dir "${datapath}/EU-SILC/2 L-2006/" files "udb_l06d_ver*" + display `filen' + local filel "${datapath}/EU-SILC/2 L-2006/" + display `"`filel'"' + local fileln : display `"`filel'"' `filen' + use `"`fileln'"', clear + + local new_var = lower("`var'") + + foreach var of varlist _all { + local new_var = lower("`var'") + cap rename `var' `new_var' + } + + */ block Stata from displaying IDs in exponential format */ + tostring db030, replace + gen year = db010 + gen country = db020 + gen hid = db030 + gen rotation_group = db075 + sort country hid year + replace country = "EL" if country == "GR" + + /* count the number of rotational groups for each country in this release */ + bysort country rotation_group: gen nvals = _n == 1 + bysort country : egen nrtgrp2006 = total(nvals) + drop nvals + + /* get the rotation group(s) that cover most years */ + bysort country rotation_group : egen maxyear = max(year) + bysort country rotation_group : egen minyear = min(year) + gen years_cov = maxyear - minyear + 1 + bysort country : egen maxgrp = max(years_cov) + gen slctd_rtgrp = 0 + bysort country rotation_group: replace slctd_rtgrp = rotation_group if years_cov == maxgrp + gen lgstgrp = 0 + bysort country rotation_group: replace lgstgrp = 1 if years_cov == maxgrp + + /* build household and rotationgroup IDs that are unique across releases */ + tostring rotation_group, generate(rotation_groupstr) + gen drpout_year = 0 + replace drpout_year = maxyear + 4 - years_cov + + /* check for rotation groups that dropped out before planned */ + bysort country rotation_group : egen maxyear_grp = max(year) + replace drpout_year = maxyear_grp if maxyear_grp < maxyear + tostring drpout_year, replace + gen uhid = country + rotation_groupstr + drpout_year + hid + gen urtgrp = country + rotation_groupstr + drpout_year + destring drpout_year, replace + + /* check if any selected groups have been selected in the more recent 2007 release ("prolonged rotation groups") */ + sort year country rotation_group hid + merge m:m year country rotation_group using "${datapath}/EU-SILC/3 L-2007/2007D.dta" + replace slctd_rtgrp = rotation_group if _merge == 1 + + /* tag rotation groups that have already been selected in the more recent release in some years (overlapping) */ + gen atag = 0 + replace atag = 1 if _merge == 3 + bysort country rotation_group : egen btag = max(atag) + bysort country rotation_group : egen ndrpoy = max(drpout_year2007) + replace drpout_year = ndrpoy if btag == 1 + drop atag btag ndrpoy + + /* update urtgrp and uhid so that "prolonged rotation groups" maintain their urtgrp across different releases */ + tostring drpout_year, replace + replace uhid = country + rotation_groupstr + drpout_year + hid + replace urtgrp = country + rotation_groupstr + drpout_year + destring drpout_year, replace + drop rotation_groupstr maxyear_grp minyear years_cov maxgrp maxyear_grp maxyear + + /* drop overlapping rotation group years and data from the more recent release */ + keep if _merge == 1 + drop _merge + + /* check if any selected groups have been selected in the more recent 2008 release ("prolonged rotation groups") */ + merge m:m year country urtgrp using "${datapath}/EU-SILC/4 L-2008/2008D.dta" + replace slctd_rtgrp = rotation_group if _merge == 1 + keep if _merge == 1 + drop _merge + + /* check if any selected groups have been selected in the more recent 2009 release ("prolonged rotation groups") */ + merge m:m year country urtgrp using "${datapath}/EU-SILC/5 L-2009/2009D.dta" + replace slctd_rtgrp = rotation_group if _merge == 1 + keep if _merge == 1 + drop _merge + + destring drpout_year, replace + drop if slctd_rtgrp == 0 + + /* for later checks */ + gen drpout_year2006 = drpout_year + gen slctd_urtgrp2006 = urtgrp + gen slctd_uhid2006 = uhid + + /* merge to masterfile */ + merge 1:1 year uhid using "${datapath}/EU-SILC/19 L-2023/masterD.dta" + + /* check for duplicates caused by same rotational groups in terms of same hid, different rotation_group */ + duplicates report year country hid + duplicates tag year country hid, generate(test2) + tab country year if test2 != 0 + tab urtgrp year if test2 != 0 + + /* check if the problem regards a large portion of the rotational group, if the portion is small ignore */ + gen tag1 = 0 + replace tag1 = 1 if test2 != 0 + bysort year country urtgrp : egen ndups = total(tag1) + gen tag2 = 1 + bysort year country urtgrp : egen tot = total(tag2) + gen rdups = ndups / tot + gen tag3 = 0 + replace tag3 = 1 if rdups > 0.5 + drop tag1 tag2 ndups tot rdups + + /* extend selection to whole rotational group */ + bysort year country urtgrp : egen taga = max(tag3) + drop tag3 + + /* check if the rotational group with duplicates is the one covering most years in current release. If yes ignore */ + tab urtgrp year if taga == 1 & lgstgrp == 0 & _merge == 1 + drop if taga == 1 & lgstgrp == 0 & _merge == 1 + drop test2 taga + + /* check for unbalances */ + bysort country : egen test3 = total(nrtgrp2007) + bysort country : egen test4 = total(nrtgrp2008) + bysort country : egen test5 = total(nrtgrp2009) + replace test3 = 0 if test3 == . + replace test4 = 0 if test4 == . + replace test5 = 0 if test5 == . + tab urtgrp year if ( test3 != 0 & lgstgrp == 0 & _merge == 1 ) & ( test4 != 0 & lgstgrp == 0 & _merge == 1 ) &( test5 != 0 & lgstgrp == 0 & _merge == 1 ) + drop if ( test3 != 0 & lgstgrp == 0 & _merge == 1 ) & ( test4 != 0 & lgstgrp == 0 & _merge == 1 ) &( test5 != 0 & lgstgrp == 0 & _merge == 1 ) + drop test3 test4 test5 + + /* this is the updated masterfile */ + gen merge2006 = _merge + drop _merge + save "${datapath}/EU-SILC/19 L-2023/masterD.dta", replace + + /* this is 2006D file for later control */ + keep if merge2006 == 1 + drop merge2006 + save "${datapath}/EU-SILC/2 L-2006/2006D.dta", replace + + +/* 2005 */ + /* open the 2005 Household register to get data from rotational groups inactive in more recent releases and their uhid */ + clear + local filen : dir "${datapath}/EU-SILC/1 L-2005/" files "udb_l05d_ver*" + display `filen' + local filel "${datapath}/EU-SILC/1 L-2005/" + display `"`filel'"' + local fileln : display `"`filel'"' `filen' + use `"`fileln'"', clear + + + local new_var = lower("`var'") + + foreach var of varlist _all { + local new_var = lower("`var'") + cap rename `var' `new_var' + } + + */ block Stata from displaying IDs in exponential format */ + tostring db030, replace + gen year = db010 + gen country = db020 + gen hid = db030 + gen rotation_group = db075 + sort country hid year + replace country = "EL" if country == "GR" + + /* count the number of rotational groups for each country in this release */ + bysort country rotation_group: gen nvals = _n == 1 + bysort country : egen nrtgrp2005 = total(nvals) + drop nvals + + /* get the rotation group(s) that cover most years */ + bysort country rotation_group : egen maxyear = max(year) + bysort country rotation_group : egen minyear = min(year) + gen years_cov = maxyear - minyear + 1 + bysort country : egen maxgrp = max(years_cov) + gen slctd_rtgrp = 0 + bysort country rotation_group: replace slctd_rtgrp = rotation_group if years_cov == maxgrp + gen lgstgrp = 0 + bysort country rotation_group: replace lgstgrp = 1 if years_cov == maxgrp + + /* build household and rotationgroup IDs that are unique across releases */ + tostring rotation_group, generate(rotation_groupstr) + gen drpout_year = 0 + replace drpout_year = maxyear + 4 - years_cov + + /* check for rotation groups that dropped out before planned */ + bysort country rotation_group : egen maxyear_grp = max(year) + replace drpout_year = maxyear_grp if maxyear_grp < maxyear + + tostring drpout_year, replace + gen uhid = country + rotation_groupstr + drpout_year + hid + gen urtgrp = country + rotation_groupstr + drpout_year + destring drpout_year, replace + + /* check if any selected groups have been selected in the more recent 2006 release ("prolonged rotation groups") */ + sort year country rotation_group hid + merge m:m year country rotation_group using "${datapath}/EU-SILC/2 L-2006/2006D.dta" + replace slctd_rtgrp = rotation_group if _merge == 1 + + /* tag rotation groups that have already been selected in the more recent release in some years (overlapping) */ + gen atag = 0 + replace atag = 1 if _merge == 3 + bysort country rotation_group : egen btag = max(atag) + bysort country rotation_group : egen ndrpoy = max(drpout_year2006) + replace drpout_year = ndrpoy if btag == 1 + drop atag btag ndrpoy + + /* update urtgrp and uhid so that "prolonged rotation groups" maintain their urtgrp across different releases */ + tostring drpout_year, replace + replace uhid = country + rotation_groupstr + drpout_year + hid + replace urtgrp = country + rotation_groupstr + drpout_year + destring drpout_year, replace + drop rotation_groupstr maxyear_grp minyear years_cov maxgrp maxyear_grp maxyear + + /* drop overlapping rotation group years and data from the more recent release */ + keep if _merge == 1 + drop _merge + + /* check if any selected groups have been selected in the more recent 2007 release ("prolonged rotation groups") */ + merge m:m year country urtgrp using "${datapath}/EU-SILC/3 L-2007/2007D.dta" + replace slctd_rtgrp = rotation_group if _merge == 1 + keep if _merge == 1 + drop _merge + + /* check if any selected groups have been selected in the more recent 2008 release ("prolonged rotation groups") */ + merge m:m year country urtgrp using "${datapath}/EU-SILC/4 L-2008/2008D.dta" + replace slctd_rtgrp = rotation_group if _merge == 1 + keep if _merge == 1 + drop _merge + + destring drpout_year, replace + drop if slctd_rtgrp == 0 + + /* for later checks */ + gen drpout_year2005 = drpout_year + gen slctd_urtgrp2005 = urtgrp + gen slctd_uhid2005 = uhid + + /* merge to masterfile */ + merge 1:1 year uhid using "${datapath}/EU-SILC/19 L-2023/masterD.dta" + + /* check for duplicates caused by same rotational groups in terms of same hid, different rotation_group */ + duplicates report year country hid + duplicates tag year country hid, generate(test2) + tab country year if test2 != 0 + tab urtgrp year if test2 != 0 + + /* check if the problem regards a large portion of the rotational group, if the portion is small ignore */ + gen tag1 = 0 + replace tag1 = 1 if test2 != 0 + bysort year country urtgrp : egen ndups = total(tag1) + gen tag2 = 1 + bysort year country urtgrp : egen tot = total(tag2) + gen rdups = ndups / tot + gen tag3 = 0 + replace tag3 = 1 if rdups > 0.5 + drop tag1 tag2 ndups tot rdups + + /* extend selection to whole rotational group */ + bysort year country urtgrp : egen taga = max(tag3) + drop tag3 + + /* check if the rotational group with duplicates is the one covering most years in current release. If yes ignore */ + tab urtgrp year if taga == 1 & lgstgrp == 0 & _merge == 1 + drop if taga == 1 & lgstgrp == 0 & _merge == 1 + drop test2 taga + + /* check for unbalances */ + bysort country : egen test3 = total(nrtgrp2006) + bysort country : egen test4 = total(nrtgrp2007) + bysort country : egen test5 = total(nrtgrp2008) + replace test3 = 0 if test3 == . + replace test4 = 0 if test4 == . + replace test5 = 0 if test5 == . + tab urtgrp year if ( test3 != 0 & lgstgrp == 0 & _merge == 1 ) & ( test4 != 0 & lgstgrp == 0 & _merge == 1 ) &( test5 != 0 & lgstgrp == 0 & _merge == 1 ) + drop if ( test3 != 0 & lgstgrp == 0 & _merge == 1 ) & ( test4 != 0 & lgstgrp == 0 & _merge == 1 ) &( test5 != 0 & lgstgrp == 0 & _merge == 1 ) + drop test3 test4 test5 + + /* this is the updated masterfile */ + gen merge2005 = _merge + drop _merge + save "${datapath}/EU-SILC/19 L-2023/masterD.dta", replace + + /* this is 2005D file for later control */ + keep if merge2005 == 1 + drop merge2005 + save "${datapath}/EU-SILC/1 L-2005/2005D.dta", replace + + /* clean up and generate yrelease */ + use "${datapath}/EU-SILC/19 L-2023/masterD.dta", clear + gen yrelease = 0 + replace yrelease = 2020 if merge2020 == 1 + replace yrelease = 2019 if merge2019 == 1 + replace yrelease = 2018 if merge2018 == 1 + replace yrelease = 2017 if merge2017 == 1 + replace yrelease = 2016 if merge2016 == 1 + replace yrelease = 2015 if merge2015 == 1 + replace yrelease = 2014 if merge2014 == 1 + replace yrelease = 2013 if merge2013 == 1 + replace yrelease = 2012 if merge2012 == 1 + replace yrelease = 2011 if merge2011 == 1 + replace yrelease = 2010 if merge2010 == 1 + replace yrelease = 2009 if merge2009 == 1 + replace yrelease = 2008 if merge2008 == 1 + replace yrelease = 2007 if merge2007 == 1 + replace yrelease = 2006 if merge2006 == 1 + replace yrelease = 2005 if merge2005 == 1 + + drop slctd_rtgrp lgstgrp drpout_year maxgrp /// + nrtgrp* drpout_year* slctd_urtgrp* slctd_uhi* merge* + + + save "${datapath}/EU-SILC/19 L-2023/masterD.dta", replace diff --git a/input_processing/data_construction/SILC_panel_construction/2005_2023_panel/do_files/02_create_masterH.do b/input_processing/data_construction/SILC_panel_construction/2005_2023_panel/do_files/02_create_masterH.do new file mode 100644 index 0000000..7f50f31 --- /dev/null +++ b/input_processing/data_construction/SILC_panel_construction/2005_2023_panel/do_files/02_create_masterH.do @@ -0,0 +1,711 @@ +/******************************************************************************* +SUMMARY — 02_create_masterH.do + +This do-file builds the master household (H) panel dataset for EU-SILC 2005-2023 +It runs in two steps: + +STEP 1 + Create clean annual H files (one block per release year, 2023 down to 2005) + For each year, the raw H data file is read and merged 1:1 with the + corresponding YYYYD.dta produced by 01_create_masterD.do. + Keeping only _merge == 3 selects the correct sample: the D file has already + handled all cross-release deduplication and rotation group selection, so the + H file inherits a clean sample through this merge. + Each annual block saves a YYYYH.dta file containing both H and D variables + for the selected households in that release year. + The 2023 block is the master year and also saves masterH.dta. + +STEP 2 + Assemble the full panel + The 2023 masterH.dta is loaded and the annual YYYYH.dta files from 2022 back + to 2005 are merged in sequentially using year and uhid. Because the D file + ensured uhid is unique across releases, the 1:1 merges produce no + duplicates. + Superfluous D-file variables are dropped and the combined dataset is saved + as the final masterH.dta in the 2024 release folder. + +NOTE: This file must be run after 01_create_masterD.do. All cross-release +deduplication logic resides in the D file; this file trusts and relies on its +output. +*******************************************************************************/ + +/* 2023 */ + clear + local filen : dir "${datapath}/EU-SILC/19 L-2023/" files "udb_l23h_ver*" + display `filen' + local filel "${datapath}/EU-SILC/19 L-2023/" + display `"`filel'"' + local fileln : display `"`filel'"' `filen' + use `"`fileln'"', clear + + local new_var = lower("`var'") + + foreach var of varlist _all { + local new_var = lower("`var'") + cap rename `var' `new_var' + } + + * block Stata from displaying IDs in exponential format* + tostring hb030, replace + gen year = hb010 + gen hid = hb030 + gen country = hb020 + sort year country hid + + *checking for duplicates or errors in hid* + duplicates report country year hid + + /* select observations/households by merging with selected rotational groups from the D file */ + merge 1:1 year country hid using "${datapath}/EU-SILC/19 L-2023/2023D.dta" + keep if _merge == 3 + drop _merge + destring hb100 , replace force + save "${datapath}/EU-SILC/19 L-2023/masterH.dta", replace + +/* 2022 */ + clear + local filen : dir "${datapath}/EU-SILC/18 L-2022/" files "udb_l22h_ver*" + display `filen' + local filel "${datapath}/EU-SILC/18 L-2022/" + display `"`filel'"' + local fileln : display `"`filel'"' `filen' + use `"`fileln'"', clear + + local new_var = lower("`var'") + + foreach var of varlist _all { + local new_var = lower("`var'") + cap rename `var' `new_var' + } + + * block Stata from displaying IDs in exponential format* + tostring hb030, replace + gen year = hb010 + gen hid = hb030 + gen country = hb020 + sort year country hid + + *checking for duplicates or errors in hid* + duplicates report country year hid + + /* select observations/households by merging with selected rotational groups from the D file */ + merge 1:1 year country hid using "${datapath}/EU-SILC/18 L-2022/2022D.dta" + keep if _merge == 3 + drop _merge + destring hb100 , replace force + save "${datapath}/EU-SILC/18 L-2022/2022H.dta", replace + +/* 2021 */ + clear + local filen : dir "${datapath}/EU-SILC/17 L-2021/" files "udb_l21h_ver*" + display `filen' + local filel "${datapath}/EU-SILC/17 L-2021/" + display `"`filel'"' + local fileln : display `"`filel'"' `filen' + use `"`fileln'"', clear + + local new_var = lower("`var'") + + foreach var of varlist _all { + local new_var = lower("`var'") + cap rename `var' `new_var' + } + + + * block Stata from displaying IDs in exponential format* + tostring hb030, replace + gen year = hb010 + gen hid = hb030 + gen country = hb020 + sort year country hid + *checking for duplicates or errors in hid* + duplicates report country year hid + /* select observations/households by merging with selected rotational groups from the D file */ + merge 1:1 year country hid using "${datapath}/EU-SILC/17 L-2021/2021D.dta" + keep if _merge == 3 + drop _merge + destring hb100 , replace force + save "${datapath}/EU-SILC/17 L-2021/2021H.dta", replace + +/* 2020 */ + clear + local filen : dir "${datapath}/EU-SILC/16 L-2020/" files "udb_l20h_ver*" + display `filen' + local filel "${datapath}/EU-SILC/16 L-2020/" + display `"`filel'"' + local fileln : display `"`filel'"' `filen' + use `"`fileln'"', clear + + local new_var = lower("`var'") + + foreach var of varlist _all { + local new_var = lower("`var'") + cap rename `var' `new_var' + } + + * block Stata from displaying IDs in exponential format* + tostring hb030, replace + gen year = hb010 + gen hid = hb030 + gen country = hb020 + sort year country hid + + *checking for duplicates or errors in hid* + duplicates report country year hid + + /* select observations/households by merging with selected rotational groups from the D file */ + merge 1:1 year country hid using "${datapath}/EU-SILC/16 L-2020/2020D.dta" + keep if _merge == 3 + drop _merge + destring hb100 , replace force + save "${datapath}/EU-SILC/16 L-2020/2020H.dta", replace + +/* 2019 */ + clear + local filen : dir "${datapath}/EU-SILC/15 L-2019/" files "udb_l19h_ver*" + display `filen' + local filel "${datapath}/EU-SILC/15 L-2019/" + display `"`filel'"' + local fileln : display `"`filel'"' `filen' + use `"`fileln'"', clear + + + local new_var = lower("`var'") + + foreach var of varlist _all { + local new_var = lower("`var'") + cap rename `var' `new_var' + } + + + * block Stata from displaying IDs in exponential format* + tostring hb030, replace + gen year = hb010 + gen hid = hb030 + gen country = hb020 + sort year country hid + + *checking for duplicates or errors in hid* + duplicates report country year hid + + /* select observations/households by merging with selected rotational groups from the D file */ + merge 1:1 year country hid using "${datapath}/EU-SILC/15 L-2019/2019D.dta" + keep if _merge == 3 + drop _merge + destring hb100 , replace force + save "${datapath}/EU-SILC/15 L-2019/2019H.dta", replace + +/* 2018 */ + clear + local filen : dir "${datapath}/EU-SILC/14 L-2018/" files "udb_l18h_ver*" + display `filen' + local filel "${datapath}/EU-SILC/14 L-2018/" + display `"`filel'"' + local fileln : display `"`filel'"' `filen' + use `"`fileln'"', clear + + + local new_var = lower("`var'") + + foreach var of varlist _all { + local new_var = lower("`var'") + cap rename `var' `new_var' + } + + + * block Stata from displaying IDs in exponential format* + tostring hb030, replace + gen year = hb010 + gen hid = hb030 + gen country = hb020 + sort year country hid + *checking for duplicates or errors in hid* + duplicates report country year hid + /* select observations/households by merging with selected rotational groups from the D file */ + merge 1:1 year country hid using "${datapath}/EU-SILC/14 L-2018/2018D.dta" + keep if _merge == 3 + drop _merge + destring hb100 , replace force + save "${datapath}/EU-SILC/14 L-2018/2018H.dta", replace + +/* 2017 */ + clear + local filen : dir "${datapath}/EU-SILC/13 L-2017/" files "udb_l17h_ver*" + display `filen' + local filel "${datapath}/EU-SILC/13 L-2017/" + display `"`filel'"' + local fileln : display `"`filel'"' `filen' + use `"`fileln'"', clear + + + local new_var = lower("`var'") + + foreach var of varlist _all { + local new_var = lower("`var'") + cap rename `var' `new_var' + } + + + * block Stata from displaying IDs in exponential format* + tostring hb030, replace + gen year = hb010 + gen hid = hb030 + gen country = hb020 + sort year country hid + *checking for duplicates or errors in hid* + duplicates report country year hid + /* select observations/households by merging with selected rotational groups from the D file */ + merge 1:1 year country hid using "${datapath}/EU-SILC/13 L-2017/2017D.dta" + keep if _merge == 3 + drop _merge + destring hb100 , replace force + save "${datapath}/EU-SILC/13 L-2017/2017H.dta", replace + +/* 2016 */ + clear + local filen : dir "${datapath}/EU-SILC/12 L-2016/" files "udb_l16h_ver*" + display `filen' + local filel "${datapath}/EU-SILC/12 L-2016/" + display `"`filel'"' + local fileln : display `"`filel'"' `filen' + use `"`fileln'"', clear + + + local new_var = lower("`var'") + + foreach var of varlist _all { + local new_var = lower("`var'") + cap rename `var' `new_var' + } + + + * block Stata from displaying IDs in exponential format* + tostring hb030, replace + gen year = hb010 + gen hid = hb030 + gen country = hb020 + sort year country hid + *checking for duplicates or errors in hid* + duplicates report country year hid + /* select observations/households by merging with selected rotational groups from the D file */ + merge 1:1 year country hid using "${datapath}/EU-SILC/12 L-2016/2016D.dta" + keep if _merge == 3 + drop _merge + destring hb100 , replace force + save "${datapath}/EU-SILC/12 L-2016/2016H.dta", replace + +/* 2015 */ + clear + local filen : dir "${datapath}/EU-SILC/11 L-2015/" files "udb_l15h_ver*" + display `filen' + local filel "${datapath}/EU-SILC/11 L-2015/" + display `"`filel'"' + local fileln : display `"`filel'"' `filen' + use `"`fileln'"', clear + + + local new_var = lower("`var'") + + foreach var of varlist _all { + local new_var = lower("`var'") + cap rename `var' `new_var' + } + + + * block Stata from displaying IDs in exponential format* + tostring hb030, replace + gen year = hb010 + gen hid = hb030 + gen country = hb020 + sort year country hid + *checking for duplicates or errors in hid* + duplicates report country year hid + /* select observations/households by merging with selected rotational groups from the D file */ + merge 1:1 year country hid using "${datapath}/EU-SILC/11 L-2015/2015D.dta" + keep if _merge == 3 + drop _merge + destring hb100 , replace force + save "${datapath}/EU-SILC/11 L-2015/2015H.dta", replace + +/* 2014 */ + clear + local filen : dir "${datapath}/EU-SILC/10 L-2014/" files "udb_l14h_ver*" + display `filen' + local filel "${datapath}/EU-SILC/10 L-2014/" + display `"`filel'"' + local fileln : display `"`filel'"' `filen' + use `"`fileln'"', clear + + + local new_var = lower("`var'") + + foreach var of varlist _all { + local new_var = lower("`var'") + cap rename `var' `new_var' + } + + /* block Stata from displaying IDs in exponential format */ + tostring hb030, replace + gen year = hb010 + gen hid = hb030 + tostring hid, replace + gen country = hb020 + sort year country hid + /* checking for duplicates or errors in hid */ + duplicates report country year hid + /* select observations/households by merging with selected rotational groups from the D file */ + merge 1:1 year country hid using "${datapath}/EU-SILC/10 L-2014/2014D.dta" + keep if _merge == 3 + drop _merge + destring hb100 , replace force + save "${datapath}/EU-SILC/10 L-2014/2014H.dta", replace + +/* 2013 */ + clear + local filen : dir "${datapath}/EU-SILC/9 L-2013/" files "udb_l13h_ver*" + display `filen' + local filel "${datapath}/EU-SILC/9 L-2013/" + display `"`filel'"' + local fileln : display `"`filel'"' `filen' +use `"`fileln'"', clear + + + local new_var = lower("`var'") + + foreach var of varlist _all { + local new_var = lower("`var'") + cap rename `var' `new_var' + } + + /* block Stata from displaying IDs in exponential format */ + tostring hb030, replace + gen year = hb010 + gen hid = hb030 + tostring hid, replace + gen country = hb020 + sort year country hid + /* checking for duplicates or errors in hid */ + duplicates report country year hid + /* select observations/households by merging with selected rotational groups from the D file */ + + merge 1:1 year country hid using "${datapath}/EU-SILC/9 L-2013/2013D.dta" + keep if _merge == 3 + drop _merge + destring hb100 , replace force + save "${datapath}/EU-SILC/9 L-2013/2013H.dta", replace + + clear + local filen : dir "${datapath}/EU-SILC/8 L-2012/" files "udb_l12h_ver*" + display `filen' + local filel "${datapath}/EU-SILC/8 L-2012/" + display `"`filel'"' + local fileln : display `"`filel'"' `filen' + use `"`fileln'"', clear + + + local new_var = lower("`var'") + + foreach var of varlist _all { + local new_var = lower("`var'") + cap rename `var' `new_var' + } + + + /* block Stata from displaying IDs in exponential format */ + tostring hb030, replace + gen year = hb010 + gen hid = hb030 + tostring hid, replace + gen country = hb020 + sort year country hid + /* checking for duplicates or errors in hid */ + duplicates report country year hid + /* select observations/households by merging with selected rotational groups from the D file */ + merge 1:1 year country hid using "${datapath}/EU-SILC/8 L-2012/2012D.dta" + keep if _merge == 3 + drop _merge + destring hb100 , replace force + save "${datapath}/EU-SILC/8 L-2012/2012H.dta", replace + +/* 2011 */ + clear + local filen : dir "${datapath}/EU-SILC/7 L-2011/" files "udb_l11h_ver*" + display `filen' + local filel "${datapath}/EU-SILC/7 L-2011/" + display `"`filel'"' + local fileln : display `"`filel'"' `filen' +use `"`fileln'"', clear + + + local new_var = lower("`var'") + + foreach var of varlist _all { + local new_var = lower("`var'") + cap rename `var' `new_var' + } + + /* block Stata from displaying IDs in exponential format */ + tostring hb030, replace + gen year = hb010 + gen hid = hb030 + tostring hid, replace + gen country = hb020 + sort year country hid + /* checking for duplicates or errors in hid */ + duplicates report country year hid + /* select observations/households by merging with selected rotational groups from the D file */ + merge 1:1 year country hid using "${datapath}/EU-SILC/7 L-2011/2011D.dta" + keep if _merge == 3 + drop _merge + destring hb100 , replace force + save "${datapath}/EU-SILC/7 L-2011/2011H.dta", replace + +/* 2010 */ + clear + local filen : dir "${datapath}/EU-SILC/6 L-2010/" files "udb_l10h_ver*" + display `filen' + local filel "${datapath}/EU-SILC/6 L-2010/" + display `"`filel'"' + local fileln : display `"`filel'"' `filen' +use `"`fileln'"', clear + + + local new_var = lower("`var'") + + foreach var of varlist _all { + local new_var = lower("`var'") + cap rename `var' `new_var' + } + + + /* block Stata from displaying IDs in exponential format */ + tostring hb030, replace + gen year = hb010 + gen hid = hb030 + tostring hid, replace + gen country = hb020 + sort year country hid + /* checking for duplicates or errors in hid */ + duplicates report country year hid + /* select observations/households by merging with selected rotational groups from the D file */ + + merge 1:1 year country hid using "${datapath}/EU-SILC/6 L-2010/2010D.dta" + keep if _merge == 3 + drop _merge + destring hb100 , replace force + save "${datapath}/EU-SILC/6 L-2010/2010H.dta", replace + +/* 2009 */ + clear + local filen : dir "${datapath}/EU-SILC/5 L-2009/" files "udb_l09h_ver*" + display `filen' + local filel "${datapath}/EU-SILC/5 L-2009/" + display `"`filel'"' + local fileln : display `"`filel'"' `filen' +use `"`fileln'"', clear + + + local new_var = lower("`var'") + + foreach var of varlist _all { + local new_var = lower("`var'") + cap rename `var' `new_var' + } + + /* block Stata from displaying IDs in exponential format */ + tostring hb030, replace + gen year = hb010 + gen hid = hb030 + tostring hid, replace + gen country = hb020 + sort year country hid + /* checking for duplicates or errors in hid */ + duplicates report country year hid + /* select observations/households by merging with selected rotational groups from the D file */ + merge 1:1 year country hid using "${datapath}/EU-SILC/5 L-2009/2009D.dta" + keep if _merge == 3 + drop _merge + destring hb100 , replace force + save "${datapath}/EU-SILC/5 L-2009/2009H.dta", replace + +/* 2008 */ + clear + local filen : dir "${datapath}/EU-SILC/4 L-2008/" files "udb_l08h_ver*" + display `filen' + local filel "${datapath}/EU-SILC/4 L-2008/" + display `"`filel'"' + local fileln : display `"`filel'"' `filen' +use `"`fileln'"', clear + + + local new_var = lower("`var'") + + foreach var of varlist _all { + local new_var = lower("`var'") + cap rename `var' `new_var' + } + + + /* block Stata from displaying IDs in exponential format */ + tostring hb030, replace + gen year = hb010 + gen hid = hb030 + tostring hid, replace + gen country = hb020 + sort year country hid + /* checking for duplicates or errors in hid */ + duplicates report country year hid + /* select observations/households by merging with selected rotational groups from the D file */ + merge 1:1 year country hid using "${datapath}/EU-SILC/4 L-2008/2008D.dta" + keep if _merge == 3 + drop _merge + destring hb100 , replace force + save "${datapath}/EU-SILC/4 L-2008/2008H.dta", replace + +/* 2007 */ + clear + local filen : dir "${datapath}/EU-SILC/3 L-2007/" files "udb_l07h_ver*" + display `filen' + local filel "${datapath}/EU-SILC/3 L-2007/" + display `"`filel'"' + local fileln : display `"`filel'"' `filen' +use `"`fileln'"', clear + + local new_var = lower("`var'") + + foreach var of varlist _all { + local new_var = lower("`var'") + cap rename `var' `new_var' + } + + /* block Stata from displaying IDs in exponential format */ + tostring hb030, replace + gen year = hb010 + gen hid = hb030 + tostring hid, replace + gen country = hb020 + sort year country hid + /* checking for duplicates or errors in hid */ + duplicates report country year hid + * select observations/households by merging with selected rotational groups from the D file */ + merge 1:1 year country hid using "${datapath}/EU-SILC/3 L-2007/2007D.dta" + keep if _merge == 3 + drop _merge + destring hb100 , replace force + save "${datapath}/EU-SILC/3 L-2007/2007H.dta", replace + +/* 2006 */ + clear + local filen : dir "${datapath}/EU-SILC/2 L-2006/" files "udb_l06h_ver*" + display `filen' + local filel "${datapath}/EU-SILC/2 L-2006/" + display `"`filel'"' + local fileln : display `"`filel'"' `filen' +use `"`fileln'"', clear + + + local new_var = lower("`var'") + + foreach var of varlist _all { + local new_var = lower("`var'") + cap rename `var' `new_var' + } + + + /* block Stata from displaying IDs in exponential format */ + tostring hb030, replace + gen year = hb010 + gen hid = hb030 + tostring hid, replace + gen country = hb020 + sort year country hid + /* checking for duplicates or errors in hid */ + duplicates report country year hid + + /* select observations/households by merging with selected rotational groups from the D file */ + merge 1:1 year country hid using "${datapath}/EU-SILC/2 L-2006/2006D.dta" + keep if _merge == 3 + drop _merge + destring hb100 , replace force + save "${datapath}/EU-SILC/2 L-2006/2006H.dta", replace + +/* 2005 */ + clear + local filen : dir "${datapath}/EU-SILC/1 L-2005/" files "udb_l05h_ver*" + display `filen' + local filel "${datapath}/EU-SILC/1 L-2005/" + display `"`filel'"' + local fileln : display `"`filel'"' `filen' +use `"`fileln'"', clear + + + local new_var = lower("`var'") + + foreach var of varlist _all { + local new_var = lower("`var'") + cap rename `var' `new_var' + } + + /* block Stata from displaying IDs in exponential format */ + tostring hb030, replace + gen year = hb010 + gen hid = hb030 + tostring hid, replace + gen country = hb020 + sort year country hid + /* checking for duplicates or errors in hid */ + duplicates report country year hid + /* select observations/households by merging with selected rotational groups from the D file */ + merge 1:1 year country hid using "${datapath}/EU-SILC/1 L-2005/2005D.dta" + keep if _merge == 3 + drop _merge + destring hb100 , replace force + save "${datapath}/EU-SILC/1 L-2005/2005H.dta", replace + + + /* merge masterH with the 20XXH files from previous releases */ + use "${datapath}/EU-SILC/19 L-2023/masterH.dta", clear + merge 1:1 year uhid using "${datapath}/EU-SILC/18 L-2022/2022H.dta" + drop _merge + merge 1:1 year uhid using "${datapath}/EU-SILC/17 L-2021/2021H.dta" + drop _merge + merge 1:1 year uhid using "${datapath}/EU-SILC/16 L-2020/2020H.dta" + drop _merge + merge 1:1 year uhid using "${datapath}/EU-SILC/15 L-2019/2019H.dta" + drop _merge + merge 1:1 year uhid using "${datapath}/EU-SILC/14 L-2018/2018H.dta" + drop _merge + merge 1:1 year uhid using "${datapath}/EU-SILC/13 L-2017/2017H.dta" + drop _merge + merge 1:1 year uhid using "${datapath}/EU-SILC/12 L-2016/2016H.dta" + drop _merge + merge 1:1 year uhid using "${datapath}/EU-SILC/11 L-2015/2015H.dta" + drop _merge + merge 1:1 year uhid using "${datapath}/EU-SILC/10 L-2014/2014H.dta" + drop _merge + merge 1:1 year uhid using "${datapath}/EU-SILC/9 L-2013/2013H.dta" + drop _merge + merge 1:1 year uhid using "${datapath}/EU-SILC/8 L-2012/2012H.dta" + drop _merge + merge 1:1 year uhid using "${datapath}/EU-SILC/7 L-2011/2011H.dta" + drop _merge + merge 1:1 year uhid using "${datapath}/EU-SILC/6 L-2010/2010H.dta" + drop _merge + merge 1:1 year uhid using "${datapath}/EU-SILC/5 L-2009/2009H.dta" + drop _merge + merge 1:1 year uhid using "${datapath}/EU-SILC/4 L-2008/2008H.dta" + drop _merge + merge 1:1 year uhid using "${datapath}/EU-SILC/3 L-2007/2007H.dta" + drop _merge + merge 1:1 year uhid using "${datapath}/EU-SILC/2 L-2006/2006H.dta" + drop _merge + merge 1:1 year uhid using "${datapath}/EU-SILC/1 L-2005/2005H.dta" + drop _merge + + /* drop superflous variables */ + drop db010 db020 db030 db040 db040_f db060 db060_f db062 db062_f db070 db070_f db075 db075_f db095 db095_f db100 db100_f db110 db110_f /// + nrtgrp* slctd_rtgrp drpout_year drpout_year* slctd_uhid* slctd_urtgrp* /// + maxgrp lgstgrp db090 db090_f merge* + + save "${datapath}/EU-SILC/19 L-2023/masterH.dta", replace + diff --git a/input_processing/data_construction/SILC_panel_construction/2005_2023_panel/do_files/03_create_masterR.do b/input_processing/data_construction/SILC_panel_construction/2005_2023_panel/do_files/03_create_masterR.do new file mode 100644 index 0000000..4ae079a --- /dev/null +++ b/input_processing/data_construction/SILC_panel_construction/2005_2023_panel/do_files/03_create_masterR.do @@ -0,0 +1,869 @@ +/******************************************************************************* +SUMMARY — 03_create_masterR.do + +This file builds the master personal register (R) panel for EU-SILC 2005–2023. +It follows the same two-step structure as 02_create_masterH.do, with two key +differences reflecting that R is a person-level (not household-level) file: + +STEP 1 + Create clean annual R files (one block per release year, 2023 down to 2005) + For each year, the raw R data file is read and merged m:1 with the + corresponding YYYYD.dta (many persons per household, hence m:1 not 1:1). + Keeping only _merge == 3 selects the correct sample — the D file has already + handled all cross-release deduplication, so the R file inherits a clean s + ample through this merge. + A unique personal ID (upid) is then constructed by concatenating the first 7 + characters of the household uhid with the individual pid. This ensures upid + is unique across releases. Each block saves a YYYYR.dta file. + +STEP 2 + Assemble the full panel + The 2023R.dta is loaded and the annual YYYYR files from 2022 back to 2005 + are merged in sequentially using year, upid, and uhid. + The merge is done in batches with intermediate saves due to memory + requirements. + Superfluous D-file variables are dropped and a numeric group identifier + (upidnum) is added before the final masterR.dta is saved to the 2024 release + folder. + +NOTE: This file must be run after 01_create_masterD.do. All cross-release +deduplication logic resides in the D file; this file trusts and relies on its +output. + +*******************************************************************************/ + + +/* 2023 */ + + clear + local filen : dir "${datapath}/EU-SILC/19 L-2023/" files "udb_l23r_ver*" + display `filen' + local filel "${datapath}/EU-SILC/19 L-2023/" + display `"`filel'"' + local fileln : display `"`filel'"' `filen' + use `"`fileln'"', clear + + local new_var=lower("`var'") + + foreach var of varlist _all { + local new_var = lower("`var'") + cap rename `var' `new_var' + } + + tostring rb030, replace format("%15.0f") + tostring rb040, replace + gen year = rb010 + gen hid = rb040 + gen pid = rb030 + gen country = rb020 + sort year country hid + + /*checking for duplicates or errors in id*/ + duplicates report country year hid pid + duplicates report country year pid + + /*merge with IDs from house hold register (D file) and drop households that are only in the D file, but not in the R file */ + merge m:1 year country hid using "${datapath}/EU-SILC/19 L-2023/2023D.dta" + keep if _merge == 3 + drop _merge + + /*generate personal IDs that are unique across all releases by taking the last 2 digits of pid (personal number) and merging it to uhid*/ + /*keep in mind that there are some individuals present in more than one family, so there are duplicates in pid alone but not in uhid and upid */ + gen suhid = substr(uhid,1,7) + gen upid = suhid + pid + drop suhid + + /*this is the masterfileR where to add data from previous releases to */ + save "${datapath}/EU-SILC/19 L-2023/2023R", replace + +/* 2022 */ + clear + local filen : dir "${datapath}/EU-SILC/18 L-2022/" files "udb_l22r_ver*" + display `filen' + local filel "${datapath}/EU-SILC/18 L-2022/" + display `"`filel'"' + local fileln : display `"`filel'"' `filen' + use `"`fileln'"', clear + + local new_var=lower("`var'") + + foreach var of varlist _all { + local new_var = lower("`var'") + cap rename `var' `new_var' + } + + tostring rb030, replace format("%15.0f") + tostring rb040, replace + gen year = rb010 + gen hid = rb040 + gen pid = rb030 + gen country = rb020 + sort year country hid + + /*checking for duplicates or errors in id*/ + duplicates report country year hid pid + duplicates report country year pid + + /*merge with IDs from house hold register (D file) and drop households that are only in the D file, but not in the R file */ + merge m:1 year country hid using "${datapath}/EU-SILC/18 L-2022/2022D.dta" + keep if _merge == 3 + drop _merge + + /*genereate personal IDs that are unique across all releases by taking the last 2 digits of pid (personal number) and merging it to uhid*/ + /*keep in mind that there are some individuals present in more than one family, so there are duplicates in pid alone but not in uhid and upid */ + gen suhid = substr(uhid,1,7) + gen upid = suhid + pid + drop suhid + + /*this is the masterfileR where to add data from previous releases to */ + save "${datapath}/EU-SILC/18 L-2022/2022R", replace + +/* 2021 */ + clear + local filen : dir "${datapath}/EU-SILC/17 L-2021/" files "udb_l21r_ver*" + display `filen' + local filel "${datapath}/EU-SILC/17 L-2021/" + display `"`filel'"' + local fileln : display `"`filel'"' `filen' + use `"`fileln'"', clear + + local new_var=lower("`var'") + + foreach var of varlist _all { + local new_var = lower("`var'") + cap rename `var' `new_var' + } + + tostring rb030, replace format("%15.0f") + tostring rb040, replace + gen year = rb010 + gen hid = rb040 + gen pid = rb030 + gen country = rb020 + sort year country hid + + /*checking for duplicates or errors in id*/ + duplicates report country year hid pid + duplicates report country year pid + + /*merge with IDs from house hold register (D file) and drop households that are only in the D file, but not in the R file */ + merge m:1 year country hid using "${datapath}/EU-SILC/17 L-2021/2021D.dta" + keep if _merge == 3 + drop _merge + + /*genereate personal IDs that are unique across all releases by taking the last 2 digits of pid (personal number) and merging it to uhid*/ + /*keep in mind that there are some individuals present in more than one family, so there are duplicates in pid alone but not in uhid and upid */ + gen suhid = substr(uhid,1,7) + gen upid = suhid + pid + drop suhid + + /*this is the masterfileR where to add data from previous releases to */ + save "${datapath}/EU-SILC/17 L-2021/2021R", replace + +/* 2020 */ + clear + local filen : dir "${datapath}/EU-SILC/16 L-2020/" files "udb_l20r_ver*" + display `filen' + local filel "${datapath}/EU-SILC/16 L-2020/" + display `"`filel'"' + local fileln : display `"`filel'"' `filen' + use `"`fileln'"', clear + + local new_var=lower("`var'") + + foreach var of varlist _all { + local new_var = lower("`var'") + cap rename `var' `new_var' + } + + tostring rb030, replace format("%15.0f") + tostring rb040, replace + gen year = rb010 + gen hid = rb040 + gen pid = rb030 + gen country = rb020 + sort year country hid + + /*checking for duplicates or errors in id*/ + duplicates report country year hid pid + duplicates report country year pid + + /*merge with IDs from house hold register (D file) and drop households that are only in the D file, but not in the R file */ + merge m:1 year country hid using "${datapath}/EU-SILC/16 L-2020/2020D.dta" + keep if _merge == 3 + drop _merge + + /*generate personal IDs that are unique across all releases by taking the last 2 digits of pid (personal number) and merging it to uhid*/ + /*keep in mind that there are some individuals present in more than one family, so there are duplicates in pid alone but not in uhid and upid */ + gen suhid = substr(uhid,1,7) + gen upid = suhid + pid + drop suhid + + /*this is the masterfileR where to add data from previous releases to */ + save "${datapath}/EU-SILC/16 L-2020/2020R", replace + +/* 2019 */ + clear + local filen : dir "${datapath}/EU-SILC/15 L-2019/" files "udb_l19r_ver*" + display `filen' + local filel "${datapath}/EU-SILC/15 L-2019/" + display `"`filel'"' + local fileln : display `"`filel'"' `filen' + use `"`fileln'"', clear + + local new_var=lower("`var'") + + foreach var of varlist _all { + local new_var = lower("`var'") + cap rename `var' `new_var' + } + + tostring rb030, replace format("%15.0f") + tostring rb040, replace + gen year = rb010 + gen hid = rb040 + gen pid = rb030 + gen country = rb020 + sort year country hid + + /*checking for duplicates or errors in id*/ + duplicates report country year hid pid + duplicates report country year pid + + /*merge with IDs from house hold register (D file) and drop households that are only in the D file, but not in the R file */ + merge m:1 year country hid using "${datapath}/EU-SILC/15 L-2019/2019D.dta" + keep if _merge == 3 + drop _merge + + /*genereate personal IDs that are unique across all releases by taking the last 2 digits of pid (personal number) and merging it to uhid*/ + /*keep in mind that there are some individuals present in more than one family, so there are duplicates in pid alone but not in uhid and upid */ + gen suhid = substr(uhid,1,7) + gen upid = suhid + pid + drop suhid + + /*this is the masterfileR where to add data from previous releases to */ + save "${datapath}/EU-SILC/15 L-2019/2019R", replace + +/* 2018 */ + clear + local filen : dir "${datapath}/EU-SILC/14 L-2018/" files "udb_l18r_ver*" + display `filen' + local filel "${datapath}/EU-SILC/14 L-2018/" + display `"`filel'"' + local fileln : display `"`filel'"' `filen' + use `"`fileln'"', clear + + local new_var=lower("`var'") + + foreach var of varlist _all { + local new_var = lower("`var'") + cap rename `var' `new_var' + } + + tostring rb030, replace format("%15.0f") + tostring rb040, replace + gen year = rb010 + gen hid = rb040 + gen pid = rb030 + gen country = rb020 + sort year country hid + + /*checking for duplicates or errors in id*/ + duplicates report country year hid pid + duplicates report country year pid + + /*merge with IDs from house hold register (D file) and drop households that are only in the D file, but not in the R file */ + merge m:1 year country hid using "${datapath}/EU-SILC/14 L-2018/2018D.dta" + keep if _merge == 3 + drop _merge + + /*genereate personal IDs that are unique across all releases by taking the last 2 digits of pid (personal number) and merging it to uhid*/ + /*keep in mind that there are some individuals present in more than one family, so there are duplicates in pid alone but not in uhid and upid */ + gen suhid = substr(uhid,1,7) + gen upid = suhid + pid + drop suhid + + /*this is the masterfileR where to add data from previous releases to */ + save "${datapath}/EU-SILC/14 L-2018/2018R", replace + +/* 2017 */ + clear + local filen : dir "${datapath}/EU-SILC/13 L-2017/" files "udb_l17r_ver*" + display `filen' + local filel "${datapath}/EU-SILC/13 L-2017/" + display `"`filel'"' + local fileln : display `"`filel'"' `filen' + use `"`fileln'"', clear + + local new_var=lower("`var'") + + foreach var of varlist _all { + local new_var = lower("`var'") + cap rename `var' `new_var' + } + + tostring rb030, replace format("%15.0f") + tostring rb040, replace + gen year = rb010 + gen hid = rb040 + gen pid = rb030 + gen country = rb020 + sort year country hid + /*checking for duplicates or errors in id*/ + duplicates report country year hid pid + duplicates report country year pid + /*merge with IDs from house hold register (D file) and drop households that are only in the D file, but not in the R file */ + merge m:1 year country hid using "${datapath}/EU-SILC/13 L-2017/2017D.dta" + keep if _merge == 3 + drop _merge + /*genereate personal IDs that are unique across all releases by taking the last 2 digits of pid (personal number) and merging it to uhid*/ + /*keep in mind that there are some individuals present in more than one family, so there are duplicates in pid alone but not in uhid and upid */ + gen suhid = substr(uhid,1,7) + gen upid = suhid + pid + drop suhid + /*this is the masterfileR where to add data from previous releases to */ + save "${datapath}/EU-SILC/13 L-2017/2017R", replace + +/* 2016 */ + clear + local filen : dir "${datapath}/EU-SILC/12 L-2016/" files "udb_l16r_ver*" + display `filen' + local filel "${datapath}/EU-SILC/12 L-2016/" + display `"`filel'"' + local fileln : display `"`filel'"' `filen' + use `"`fileln'"', clear + + local new_var=lower("`var'") + + foreach var of varlist _all { + local new_var = lower("`var'") + cap rename `var' `new_var' + } + + tostring rb030, replace format("%15.0f") + tostring rb040, replace + gen year = rb010 + gen hid = rb040 + gen pid = rb030 + gen country = rb020 + sort year country hid + /*checking for duplicates or errors in id*/ + duplicates report country year hid pid + duplicates report country year pid + /*merge with IDs from house hold register (D file) and drop households that are only in the D file, but not in the R file */ + merge m:1 year country hid using "${datapath}/EU-SILC/12 L-2016/2016D.dta" + keep if _merge == 3 + drop _merge + /*genereate personal IDs that are unique across all releases by taking the last 2 digits of pid (personal number) and merging it to uhid*/ + /*keep in mind that there are some individuals present in more than one family, so there are duplicates in pid alone but not in uhid and upid */ + gen suhid = substr(uhid,1,7) + gen upid = suhid + pid + drop suhid + /*this is the masterfileR where to add data from previous releases to */ + save "${datapath}/EU-SILC/12 L-2016/2016R", replace + +/* 2015 */ + clear + local filen : dir "${datapath}/EU-SILC/11 L-2015/" files "udb_l15r_ver*" + display `filen' + local filel "${datapath}/EU-SILC/11 L-2015/" + display `"`filel'"' + local fileln : display `"`filel'"' `filen' + use `"`fileln'"', clear + + + local new_var=lower("`var'") + + foreach var of varlist _all { + local new_var = lower("`var'") + cap rename `var' `new_var' + } + + + tostring rb030, replace format("%15.0f") + tostring rb040, replace + gen year = rb010 + gen hid = rb040 + gen pid = rb030 + gen country = rb020 + sort year country hid + /*checking for duplicates or errors in id*/ + duplicates report country year hid pid + duplicates report country year pid + /*merge with IDs from house hold register (D file) and drop households that are only in the D file, but not in the R file */ + merge m:1 year country hid using "${datapath}/EU-SILC/11 L-2015/2015D.dta" + keep if _merge == 3 + drop _merge + /*genereate personal IDs that are unique across all releases by taking the last 2 digits of pid (personal number) and merging it to uhid*/ + /*keep in mind that there are some individuals present in more than one family, so there are duplicates in pid alone but not in uhid and upid */ + gen suhid = substr(uhid,1,7) + gen upid = suhid + pid + drop suhid + /*this is the masterfileR where to add data from previous releases to */ + save "${datapath}/EU-SILC/11 L-2015/2015R", replace + + +/* 2014 */ + clear + local filen : dir "${datapath}/EU-SILC/10 L-2014/" files "udb_l14r_ver*" + display `filen' + local filel "${datapath}/EU-SILC/10 L-2014/" + display `"`filel'"' + local fileln : display `"`filel'"' `filen' +use `"`fileln'"', clear + + + local new_var=lower("`var'") + + foreach var of varlist _all { + local new_var = lower("`var'") + cap rename `var' `new_var' + } + + + tostring rb030, replace format("%15.0f") + tostring rb040, replace + gen year = rb010 + gen hid = rb040 + gen pid = rb030 + gen country = rb020 + sort year country hid + /*checking for duplicates or errors in id*/ + duplicates report country year hid pid + duplicates report country year pid + /*merge with IDs from house hold register (D file) and drop households that are only in the D file, but not in the R file */ + merge m:1 year country hid using "${datapath}/EU-SILC/10 L-2014/2014D.dta" + keep if _merge == 3 + drop _merge + /*genereate personal IDs that are unique across all releases by taking the first 7 numbers of uhid and adding them to pid*/ + /*keep in mind that there are some individuals present in more than one family, so there are duplicates in upid alone but not in uhid and upid */ + gen suhid = substr(uhid, 1, 7) + gen upid = suhid + pid + drop suhid + /*this is the masterfileR where to add data from previous releases to */ + save "${datapath}/EU-SILC/10 L-2014/2014R", replace + +/* 2013 */ + clear + local filen : dir "${datapath}/EU-SILC/9 L-2013/" files "udb_l13r_ver*" + display `filen' + local filel "${datapath}/EU-SILC/9 L-2013/" + display `"`filel'"' + local fileln : display `"`filel'"' `filen' +use `"`fileln'"', clear + + + local new_var=lower("`var'") + + foreach var of varlist _all { + local new_var = lower("`var'") + cap rename `var' `new_var' + } + + + tostring rb030, replace format("%15.0f") + tostring rb040, replace + gen year = rb010 + gen hid = rb040 + gen pid = rb030 + gen country = rb020 + sort year country hid + /*checking for duplicates or errors in id*/ + duplicates report country year hid pid + duplicates report country year pid + /*merge with IDs from house hold register (D file) and drop households that are only in the D file, but not in the R file */ + merge m:1 year country hid using "${datapath}/EU-SILC/9 L-2013/2013D.dta" + keep if _merge == 3 + drop _merge + /*genereate personal IDs that are unique across all releases by taking the first 7 numbers of uhid and adding them to pid*/ + /*keep in mind that there are some individuals present in more than one family, so there are duplicates in upid alone but not in uhid and upid */ + gen suhid = substr(uhid, 1, 7) + gen upid = suhid + pid + drop suhid + /*this is the masterfileR where to add data from previous releases to */ + save "${datapath}/EU-SILC/9 L-2013/2013R", replace + +/* 2012 */ + clear + local filen : dir "${datapath}/EU-SILC/8 L-2012/" files "udb_l12r_ver*" + display `filen' + local filel "${datapath}/EU-SILC/8 L-2012/" + display `"`filel'"' + local fileln : display `"`filel'"' `filen' +use `"`fileln'"', clear + + + local new_var=lower("`var'") + + foreach var of varlist _all { + local new_var = lower("`var'") + cap rename `var' `new_var' + } + + tostring rb030, replace format("%15.0f") + tostring rb040, replace + gen year = rb010 + gen hid = rb040 + gen pid = rb030 + gen country = rb020 + sort year country hid + /*checking for duplicates or errors in id*/ + duplicates report country year hid pid + duplicates report country year pid + /*merge with IDs from house hold register (D file) and drop households that are only in the D file, but not in the R file */ + merge m:1 year country hid using "${datapath}/EU-SILC/8 L-2012/2012D.dta" + keep if _merge == 3 + drop _merge + /*genereate personal IDs that are unique across all releases by taking the first 7 numbers of uhid and adding them to pid*/ + /*keep in mind that there are some individuals present in more than one family, so there are duplicates in upid alone but not in uhid and upid */ + gen suhid = substr(uhid, 1, 7) + gen upid = suhid + pid + drop suhid + /*this is the masterfileR where to add data from previous releases to */ + save "${datapath}/EU-SILC/8 L-2012/2012R", replace + +/* 2011 */ + clear + local filen : dir "${datapath}/EU-SILC/7 L-2011/" files "udb_l11r_ver*" + display `filen' + local filel "${datapath}/EU-SILC/7 L-2011/" + display `"`filel'"' + local fileln : display `"`filel'"' `filen' +use `"`fileln'"', clear + + + local new_var=lower("`var'") + + foreach var of varlist _all { + local new_var = lower("`var'") + cap rename `var' `new_var' + } + + tostring rb030, replace format("%15.0f") + tostring rb040, replace + gen year = rb010 + gen hid = rb040 + gen pid = rb030 + gen country = rb020 + sort year country hid + /*checking for duplicates or errors in id*/ + duplicates report country year hid pid + duplicates report country year pid + /*merge with IDs from house hold register (D file) and drop households that are only in the D file, but not in the R file */ + merge m:1 year country hid using "${datapath}/EU-SILC/7 L-2011/2011D.dta" + keep if _merge == 3 + drop _merge + /*genereate personal IDs that are unique across all releases by taking the first 7 numbers of uhid and adding them to pid*/ + /*keep in mind that there are some individuals present in more than one family, so there are duplicates in upid alone but not in uhid and upid */ + gen suhid = substr(uhid, 1, 7) + gen upid = suhid + pid + drop suhid + /*this is the masterfileR where to add data from previous releases to */ + save "${datapath}/EU-SILC/7 L-2011/2011R", replace + +/* 2010 */ + clear + local filen : dir "${datapath}/EU-SILC/6 L-2010/" files "udb_l10r_ver*" + display `filen' + local filel "${datapath}/EU-SILC/6 L-2010/" + display `"`filel'"' + local fileln : display `"`filel'"' `filen' +use `"`fileln'"', clear + + + local new_var=lower("`var'") + + foreach var of varlist _all { + local new_var = lower("`var'") + cap rename `var' `new_var' + } + + tostring rb030, replace format("%15.0f") + tostring rb040, replace + gen year = rb010 + gen hid = rb040 + gen pid = rb030 + gen country = rb020 + sort year country hid + /*checking for duplicates or errors in id*/ + duplicates report country year hid pid + duplicates report country year pid + /*merge with IDs from house hold register (D file) and drop households that are only in the D file, but not in the R file */ + merge m:1 year country hid using "${datapath}/EU-SILC/6 L-2010/2010D.dta" + keep if _merge == 3 + drop _merge + /*genereate personal IDs that are unique across all releases by taking the first 7 numbers of uhid and adding them to pid*/ + /*keep in mind that there are some individuals present in more than one family, so there are duplicates in upid alone but not in uhid and upid */ + gen suhid = substr(uhid, 1, 7) + gen upid = suhid + pid + drop suhid + /*this is the masterfileR where to add data from previous releases to */ + save "${datapath}/EU-SILC/6 L-2010/2010R", replace + +/* 2009 */ + clear + local filen : dir "${datapath}/EU-SILC/5 L-2009/" files "udb_l09r_ver*" + display `filen' + local filel "${datapath}/EU-SILC/5 L-2009/" + display `"`filel'"' + local fileln : display `"`filel'"' `filen' +use `"`fileln'"', clear + + + local new_var=lower("`var'") + + foreach var of varlist _all { + local new_var = lower("`var'") + cap rename `var' `new_var' + } + + tostring rb030, replace format("%15.0f") + tostring rb040, replace + gen year = rb010 + gen hid = rb040 + gen pid = rb030 + gen country = rb020 + sort year country hid + /*checking for duplicates or errors in id*/ + duplicates report country year hid pid + duplicates report country year pid + /*merge with IDs from house hold register (D file) and drop households that are only in the D file, but not in the R file */ + merge m:1 year country hid using "${datapath}/EU-SILC/5 L-2009/2009D.dta" + keep if _merge == 3 + drop _merge + /*genereate personal IDs that are unique across all releases by taking the first 7 numbers of uhid and adding them to pid*/ + /*keep in mind that there are some individuals present in more than one family, so there are duplicates in upid alone but not in uhid and upid */ + gen suhid = substr(uhid, 1, 7) + gen upid = suhid + pid + drop suhid + /*this is the masterfileR where to add data from previous releases to */ + save "${datapath}/EU-SILC/5 L-2009/2009R", replace + +/* 2008 */ + clear + local filen : dir "${datapath}/EU-SILC/4 L-2008/" files "udb_l08r_ver*" + display `filen' + local filel "${datapath}/EU-SILC/4 L-2008/" + display `"`filel'"' + local fileln : display `"`filel'"' `filen' +use `"`fileln'"', clear + + + local new_var=lower("`var'") + + foreach var of varlist _all { + local new_var = lower("`var'") + cap rename `var' `new_var' + } + + tostring rb030, replace format("%15.0f") + tostring rb040, replace + gen year = rb010 + gen hid = rb040 + gen pid = rb030 + gen country = rb020 + sort year country hid + /*checking for duplicates or errors in id*/ + duplicates report country year hid pid + duplicates report country year pid + /*merge with IDs from house hold register (D file) and drop households that are only in the D file, but not in the R file */ + merge m:1 year country hid using "${datapath}/EU-SILC/4 L-2008/2008D.dta" + keep if _merge == 3 + drop _merge + /*genereate personal IDs that are unique across all releases by taking the first 7 numbers of uhid and adding them to pid*/ + /*keep in mind that there are some individuals present in more than one family, so there are duplicates in upid alone but not in uhid and upid */ + gen suhid = substr(uhid, 1, 7) + gen upid = suhid + pid + drop suhid + /*this is the masterfileR where to add data from previous releases to */ + save "${datapath}/EU-SILC/4 L-2008/2008R", replace + +/* 2008 */ + clear + local filen : dir "${datapath}/EU-SILC/3 L-2007/" files "udb_l07r_ver*" + display `filen' + local filel "${datapath}/EU-SILC/3 L-2007/" + display `"`filel'"' + local fileln : display `"`filel'"' `filen' + use `"`fileln'"', clear + + + local new_var=lower("`var'") + + foreach var of varlist _all { + local new_var = lower("`var'") + cap rename `var' `new_var' + } + + tostring rb030, replace format("%15.0f") + tostring rb040, replace + gen year = rb010 + gen hid = rb040 + gen pid = rb030 + gen country = rb020 + sort year country hid + /*checking for duplicates or errors in id*/ + duplicates report country year hid pid + duplicates report country year pid + /*merge with IDs from house hold register (D file) and drop households that are only in the D file, but not in the R file */ + merge m:1 year country hid using "${datapath}/EU-SILC/3 L-2007/2007D.dta" + keep if _merge == 3 + drop _merge + /*genereate personal IDs that are unique across all releases by taking the first 7 numbers of uhid and adding them to pid*/ + /*keep in mind that there are some individuals present in more than one family, so there are duplicates in upid alone but not in uhid and upid */ + gen suhid = substr(uhid, 1, 7) + gen upid = suhid + pid + drop suhid + /*this is the masterfileR where to add data from previous releases to */ + save "${datapath}/EU-SILC/3 L-2007/2007R", replace + +/* 2006 */ + clear + local filen : dir "${datapath}/EU-SILC/2 L-2006/" files "udb_l06r_ver*" + display `filen' + local filel "${datapath}/EU-SILC/2 L-2006/" + display `"`filel'"' + local fileln : display `"`filel'"' `filen' + use `"`fileln'"', clear + + + local new_var=lower("`var'") + + foreach var of varlist _all { + local new_var = lower("`var'") + cap rename `var' `new_var' + } + + tostring rb030, replace format("%15.0f") + tostring rb040, replace + gen year = rb010 + gen hid = rb040 + gen pid = rb030 + gen country = rb020 + sort year country hid + /*checking for duplicates or errors in id*/ + duplicates report country year hid pid + duplicates report country year pid + /*merge with IDs from house hold register (D file) and drop households that are only in the D file, but not in the R file */ + merge m:1 year country hid using "${datapath}/EU-SILC/2 L-2006/2006D.dta" + keep if _merge == 3 + drop _merge + /*genereate personal IDs that are unique across all releases by taking the first 7 numbers of uhid and adding them to pid*/ + /*keep in mind that there are some individuals present in more than one family, so there are duplicates in upid alone but not in uhid and upid */ + gen suhid = substr(uhid, 1, 7) + gen upid = suhid + pid + drop suhid + /*this is the masterfileR where to add data from previous releases to */ + save "${datapath}/EU-SILC/2 L-2006/2006R", replace + +/* 2005 */ + clear + local filen : dir "${datapath}/EU-SILC/1 L-2005/" files "udb_l05r_ver*" + display `filen' + local filel "${datapath}/EU-SILC/1 L-2005/" + display `"`filel'"' + local fileln : display `"`filel'"' `filen' +use `"`fileln'"', clear + + + local new_var=lower("`var'") + + foreach var of varlist _all { + local new_var = lower("`var'") + cap rename `var' `new_var' + } + + tostring rb030, replace format("%15.0f") + tostring rb040, replace + gen year = rb010 + gen hid = rb040 + gen pid = rb030 + gen country = rb020 + sort year country hid + /*checking for duplicates or errors in id*/ + duplicates report country year hid pid + duplicates report country year pid + /*merge with IDs from house hold register (D file) and drop households that are only in the D file, but not in the R file */ + merge m:1 year country hid using "${datapath}/EU-SILC/1 L-2005/2005D.dta" + keep if _merge == 3 + drop _merge + /*genereate personal IDs that are unique across all releases by taking the first 7 numbers of uhid and adding them to pid*/ + /*keep in mind that there are some individuals present in more than one family, so there are duplicates in upid alone but not in uhid and upid */ + gen suhid = substr(uhid, 1, 7) + gen upid = suhid + pid + drop suhid + /*this is the masterfileR where to add data from previous releases to */ + save "${datapath}/EU-SILC/1 L-2005/2005R", replace + + /*merge data from all releases into one masterfile. + !! this process is memory intensive. */ + + use "${datapath}/EU-SILC/19 L-2023/2023R.dta", clear + + merge 1:1 year upid uhid using "${datapath}/EU-SILC/18 L-2022/2022R.dta" + drop _merge + merge 1:1 year upid uhid using "${datapath}/EU-SILC/17 L-2021/2021R.dta" + drop _merge + merge 1:1 year upid uhid using "${datapath}/EU-SILC/16 L-2020/2020R.dta" + drop _merge + save "${datapath}/EU-SILC/19 L-2023/masterR.dta", replace + clear + + use "${datapath}/EU-SILC/19 L-2023/masterR.dta", clear + + merge 1:1 year upid uhid using "${datapath}/EU-SILC/15 L-2019/2019R.dta" + drop _merge + merge 1:1 year upid uhid using "${datapath}/EU-SILC/14 L-2018/2018R.dta" + drop _merge + merge 1:1 year upid uhid using "${datapath}/EU-SILC/13 L-2017/2017R.dta" + drop _merge + merge 1:1 year upid uhid using "${datapath}/EU-SILC/12 L-2016/2016R.dta" + drop _merge + merge 1:1 year upid uhid using "${datapath}/EU-SILC/11 L-2015/2015R.dta" + drop _merge + save "${datapath}/EU-SILC/19 L-2023/masterR.dta", replace + clear + + use "${datapath}/EU-SILC/19 L-2023/masterR.dta", clear + + merge 1:1 year upid uhid using "${datapath}/EU-SILC/10 L-2014/2014R.dta" + drop _merge + merge 1:1 year upid uhid using "${datapath}/EU-SILC/9 L-2013/2013R.dta" + drop _merge + merge 1:1 year upid uhid using "${datapath}/EU-SILC/8 L-2012/2012R.dta" + drop _merge + merge 1:1 year upid uhid using "${datapath}/EU-SILC/7 L-2011/2011R.dta" + drop _merge + merge 1:1 year upid uhid using "${datapath}/EU-SILC/6 L-2010/2010R.dta" + drop _merge + merge 1:1 year upid uhid using "${datapath}/EU-SILC/5 L-2009/2009R.dta" + drop _merge + save "${datapath}/EU-SILC/19 L-2023/masterR.dta", replace + clear + + use "${datapath}/EU-SILC/19 L-2023/masterR.dta", clear + merge 1:1 year upid uhid using "${datapath}/EU-SILC/4 L-2008/2008R.dta" + drop _merge + merge 1:1 year upid uhid using "${datapath}/EU-SILC/3 L-2007/2007R.dta" + drop _merge + save "${datapath}/EU-SILC/19 L-2023/masterR.dta", replace + clear + + use "${datapath}/EU-SILC/19 L-2023/masterR.dta", clear + merge 1:1 year upid uhid using "${datapath}/EU-SILC/2 L-2006/2006R.dta" + drop _merge + save "${datapath}/EU-SILC/19 L-2023/masterR.dta", replace + clear + + use "${datapath}/EU-SILC/19 L-2023/masterR.dta", clear + merge 1:1 year upid uhid using "${datapath}/EU-SILC/1 L-2005/2005R.dta" + drop _merge + + /* drop superflous variables*/ + drop db010 db020 db030 db040 db040_f db060 db060_f db062 db062_f db070 db070_f db075 db075_f db095 db095_f db100 db100_f db110 db110_f /// + nrtgrp* slctd_rtgrp drpout_year drpout_year* slctd_urtgrp* slctd_uhid* maxgrp lgstgrp drpout_year* /// + db090 db090_f nrtgrp2013 merge* + + /*adding upidnum*/ + egen upidnum = group(upid) + save "${datapath}/EU-SILC/19 L-2023/masterR.dta", replace diff --git a/input_processing/data_construction/SILC_panel_construction/2005_2023_panel/do_files/04_create_masterP.do b/input_processing/data_construction/SILC_panel_construction/2005_2023_panel/do_files/04_create_masterP.do new file mode 100644 index 0000000..834e056 --- /dev/null +++ b/input_processing/data_construction/SILC_panel_construction/2005_2023_panel/do_files/04_create_masterP.do @@ -0,0 +1,758 @@ +/******************************************************************************* +SUMMARY — 03_create_masterP.do + +This file builds the master personal data (P) panel for EU-SILC 2005–2023. +It follows the same two-step structure as 03_create_masterR.do, but P is +linked to the R file (not directly to D) because person IDs in P are defined +at the person level while household membership is captured in R. + +STEP 1 + Create clean annual P files (one block per release year, 2023 down to 2005) + For each year, the raw P data file is read and merged 1:m with the + corresponding YYYYR.dta on year, country, and pid. Because R can contain a + person in multiple households (pid not unique in R), the merge is 1:m. + After keeping _merge == 3, duplicates on year, country, pid are dropped + (force) since P contains one row per person. + The clean sample is thus inherited from the R file, which in turn inherited + it from the D file. + Each block saves a YYYYP.dta file. + +STEP 2 + Assemble the full panel + The 2023P.dta is loaded and annual YYYYP files from 2022 back to 2005 are + merged in one at a time using year, upid, country, and uhid, with an + intermediate save after each step. + The one-year-at-a-time approach manages memory. + R-file and D-file variables carried over from the merge in Step 1 are + dropped at the end before the final masterP.dta is saved . + +NOTE: This file must be run after 03_create_masterR.do. Sample selection flows +D → R → P; this file trusts the output of the upstream files. + +*******************************************************************************/ + +/* 2023 */ + clear + local filen : dir "${datapath}/EU-SILC/19 L-2023/" files "udb_l23p_ver*" + display `filen' + local filel "${datapath}/EU-SILC/19 L-2023/" + display `"`filel'"' + local fileln : display `"`filel'"' `filen' + use `"`fileln'"', clear + + local new_var=lower("`var'") + + foreach var of varlist _all { + local new_var = lower("`var'") + cap rename `var' `new_var' + } + + tostring pb030, replace format("%15.0f") + gen year = pb010 + gen pid = pb030 + gen country = pb020 + sort year country pid + + /*checking for duplicates/errors in pid (should be unique)*/ + duplicates report year country pid + + /*merge with R files to keep only obs from selcted rotational groups*/ + merge 1:m year country pid using "${datapath}/EU-SILC/19 L-2023/2023R.dta" + keep if _merge == 3 + drop _merge + + /* R file contains combinations in pid and hid, so pids are not unique. In P pids are unique, so we can drop the duplicates */ + duplicates drop year country pid, force + + /*this is the masterfileR where to add data from previous releases to */ + save "${datapath}/EU-SILC/19 L-2023/2023P", replace + +/* 2022 */ + clear + local filen : dir "${datapath}/EU-SILC/18 L-2022/" files "udb_l22p_ver*" + display `filen' + local filel "${datapath}/EU-SILC/18 L-2022/" + display `"`filel'"' + local fileln : display `"`filel'"' `filen' + use `"`fileln'"', clear + + local new_var=lower("`var'") + + foreach var of varlist _all { + local new_var = lower("`var'") + cap rename `var' `new_var' + } + + tostring pb030, replace format("%15.0f") + gen year = pb010 + gen pid = pb030 + gen country = pb020 + sort year country pid + /*checking for duplicates/errors in pid (should be unique)*/ + duplicates report year country pid + /*merge with R files to keep only obs from selcted rotational groups*/ + merge 1:m year country pid using "${datapath}/EU-SILC/18 L-2022/2022R.dta" + keep if _merge == 3 + drop _merge + /* R file contains combinations in pid and hid, so pids are not unique. In P pids are unique, so we can drop the duplicates */ + duplicates drop year country pid, force + /*this is the masterfileR where to add data from previous releases to */ + save "${datapath}/EU-SILC/18 L-2022/2022P", replace + +/* 2021 */ + clear + local filen : dir "${datapath}/EU-SILC/17 L-2021/" files "udb_l21p_ver*" + display `filen' + local filel "${datapath}/EU-SILC/17 L-2021/" + display `"`filel'"' + local fileln : display `"`filel'"' `filen' + use `"`fileln'"', clear + + local new_var=lower("`var'") + + foreach var of varlist _all { + local new_var = lower("`var'") + cap rename `var' `new_var' + } + + tostring pb030, replace format("%15.0f") + gen year = pb010 + gen pid = pb030 + gen country = pb020 + sort year country pid + /*checking for duplicates/errors in pid (should be unique)*/ + duplicates report year country pid + /*merge with R files to keep only obs from selcted rotational groups*/ + merge 1:m year country pid using "${datapath}/EU-SILC/17 L-2021/2021R.dta" + keep if _merge == 3 + drop _merge + /* R file contains combinations in pid and hid, so pids are not unique. In P pids are unique, so we can drop the duplicates */ + duplicates drop year country pid, force + /*this is the masterfileR where to add data from previous releases to */ + save "${datapath}/EU-SILC/17 L-2021/2021P", replace + +/* 2020 */ + clear + local filen : dir "${datapath}/EU-SILC/16 L-2020/" files "udb_l20p_ver*" + display `filen' + local filel "${datapath}/EU-SILC/16 L-2020/" + display `"`filel'"' + local fileln : display `"`filel'"' `filen' + use `"`fileln'"', clear + + local new_var=lower("`var'") + + foreach var of varlist _all { + local new_var = lower("`var'") + cap rename `var' `new_var' + } + + tostring pb030, replace format("%15.0f") + gen year = pb010 + gen pid = pb030 + gen country = pb020 + sort year country pid + /*checking for duplicates/errors in pid (should be unique)*/ + duplicates report year country pid + /*merge with R files to keep only obs from selcted rotational groups*/ + merge 1:m year country pid using "${datapath}/EU-SILC/16 L-2020/2020R.dta" + keep if _merge == 3 + drop _merge + /* R file contains combinations in pid and hid, so pids are not unique. In P pids are unique, so we can drop the duplicates */ + duplicates drop year country pid, force + /*this is the masterfileR where to add data from previous releases to */ + save "${datapath}/EU-SILC/16 L-2020/2020P", replace + +/* 2019 */ + clear + local filen : dir "${datapath}/EU-SILC/15 L-2019/" files "udb_l19p_ver*" + display `filen' + local filel "${datapath}/EU-SILC/15 L-2019/" + display `"`filel'"' + local fileln : display `"`filel'"' `filen' + use `"`fileln'"', clear + + local new_var=lower("`var'") + + foreach var of varlist _all { + local new_var = lower("`var'") + cap rename `var' `new_var' + } + + tostring pb030, replace format("%15.0f") + gen year = pb010 + gen pid = pb030 + gen country = pb020 + sort year country pid + /*checking for duplicates/errors in pid (should be unique)*/ + duplicates report year country pid + /*merge with R files to keep only obs from selcted rotational groups*/ + merge 1:m year country pid using "${datapath}/EU-SILC/15 L-2019/2019R.dta" + keep if _merge == 3 + drop _merge + /* R file contains combinations in pid and hid, so pids are not unique. In P pids are unique, so we can drop the duplicates */ + duplicates drop year country pid, force + /*this is the masterfileR where to add data from previous releases to */ + save "${datapath}/EU-SILC/15 L-2019/2019P", replace + + +/* 2018 */ + clear + local filen : dir "${datapath}/EU-SILC/14 L-2018/" files "udb_l18p_ver*" + display `filen' + local filel "${datapath}/EU-SILC/14 L-2018/" + display `"`filel'"' + local fileln : display `"`filel'"' `filen' + use `"`fileln'"', clear + + local new_var=lower("`var'") + + foreach var of varlist _all { + local new_var = lower("`var'") + cap rename `var' `new_var' + } + + tostring pb030, replace format("%15.0f") + gen year = pb010 + gen pid = pb030 + gen country = pb020 + sort year country pid + /*checking for duplicates/errors in pid (should be unique)*/ + duplicates report year country pid + /*merge with R files to keep only obs from selcted rotational groups*/ + merge 1:m year country pid using "${datapath}/EU-SILC/14 L-2018/2018R.dta" + keep if _merge == 3 + drop _merge + /* R file contains combinations in pid and hid, so pids are not unique. In P pids are unique, so we can drop the duplicates */ + duplicates drop year country pid, force + /*this is the masterfileR where to add data from previous releases to */ + save "${datapath}/EU-SILC/14 L-2018/2018P", replace + + +/* 2017 */ + clear + local filen : dir "${datapath}/EU-SILC/13 L-2017/" files "udb_l17p_ver*" + display `filen' + local filel "${datapath}/EU-SILC/13 L-2017/" + display `"`filel'"' + local fileln : display `"`filel'"' `filen' + use `"`fileln'"', clear + + local new_var=lower("`var'") + + foreach var of varlist _all { + local new_var = lower("`var'") + cap rename `var' `new_var' + } + + tostring pb030, replace format("%15.0f") + gen year = pb010 + gen pid = pb030 + gen country = pb020 + sort year country pid + /*checking for duplicates/errors in pid (should be unique)*/ + duplicates report year country pid + /*merge with R files to keep only obs from selcted rotational groups*/ + merge 1:m year country pid using "${datapath}/EU-SILC/13 L-2017/2017R.dta" + keep if _merge == 3 + drop _merge + /* R file contains combinations in pid and hid, so pids are not unique. In P pids are unique, so we can drop the duplicates */ + duplicates drop year country pid, force + /*this is the masterfileR where to add data from previous releases to */ + save "${datapath}/EU-SILC/13 L-2017/2017P", replace + +/* 2016 */ + clear + local filen : dir "${datapath}/EU-SILC/12 L-2016/" files "udb_l16p_ver*" + display `filen' + local filel "${datapath}/EU-SILC/12 L-2016/" + display `"`filel'"' + local fileln : display `"`filel'"' `filen' + use `"`fileln'"', clear + + local new_var=lower("`var'") + + foreach var of varlist _all { + local new_var = lower("`var'") + cap rename `var' `new_var' + } + + tostring pb030, replace format("%15.0f") + gen year = pb010 + gen pid = pb030 + gen country = pb020 + sort year country pid + /*checking for duplicates/errors in pid (should be unique)*/ + duplicates report year country pid + /*merge with R files to keep only obs from selcted rotational groups*/ + merge 1:m year country pid using "${datapath}/EU-SILC/12 L-2016/2016R.dta" + keep if _merge == 3 + drop _merge + /* R file contains combinations in pid and hid, so pids are not unique. In P pids are unique, so we can drop the duplicates */ + duplicates drop year country pid, force + /*this is the masterfileR where to add data from previous releases to */ + save "${datapath}/EU-SILC/12 L-2016/2016P", replace + + +/* 2015 */ + clear + local filen : dir "${datapath}/EU-SILC/11 L-2015/" files "udb_l15p_ver*" + display `filen' + local filel "${datapath}/EU-SILC/11 L-2015/" + display `"`filel'"' + local fileln : display `"`filel'"' `filen' + use `"`fileln'"', clear + + + local new_var=lower("`var'") + + foreach var of varlist _all { + local new_var = lower("`var'") + cap rename `var' `new_var' + } + + tostring pb030, replace format("%15.0f") + gen year = pb010 + gen pid = pb030 + gen country = pb020 + sort year country pid + /*checking for duplicates/errors in pid (should be unique)*/ + duplicates report year country pid + /*merge with R files to keep only obs from selcted rotational groups*/ + merge 1:m year country pid using "${datapath}/EU-SILC/11 L-2015/2015R.dta" + keep if _merge == 3 + drop _merge + /* R file contains combinations in pid and hid, so pids are not unique. In P pids are unique, so we can drop the duplicates */ + duplicates drop year country pid, force + /*this is the masterfileR where to add data from previous releases to */ + save "${datapath}/EU-SILC/11 L-2015/2015P", replace + + /*selecting 2014 release data*/ + clear + local filen : dir "${datapath}/EU-SILC/10 L-2014/" files "udb_l14p_ver*" + display `filen' + local filel "${datapath}/EU-SILC/10 L-2014/" + display `"`filel'"' + local fileln : display `"`filel'"' `filen' + use `"`fileln'"', clear + + + local new_var=lower("`var'") + + foreach var of varlist _all { + local new_var = lower("`var'") + cap rename `var' `new_var' + } + + tostring pb030, replace format("%15.0f") + gen year = pb010 + gen pid = pb030 + gen country = pb020 + sort year country pid + /*checking for duplicates/errors in hid*/ + duplicates report year country pid + /*merge with IDs from house hold register (R file), keep matches */ + merge 1:m year country pid using "${datapath}/EU-SILC/10 L-2014/2014R.dta" + keep if _merge == 3 + drop _merge + /* R file contains combinations in pid and hid, so pids are not unique. In P pids are unique, so we can drop the duplicates */ + duplicates drop year country pid, force + save "${datapath}/EU-SILC/10 L-2014/2014P.dta", replace + + /*selecting 2013 release data*/ + clear + local filen : dir "${datapath}/EU-SILC/9 L-2013/" files "udb_l13p_ver*" + display `filen' + local filel "${datapath}/EU-SILC/9 L-2013/" + display `"`filel'"' + local fileln : display `"`filel'"' `filen' + use `"`fileln'"', clear + + + local new_var=lower("`var'") + + foreach var of varlist _all { + local new_var = lower("`var'") + cap rename `var' `new_var' + } + + tostring pb030, replace format("%15.0f") + gen year = pb010 + gen pid = pb030 + gen country = pb020 + sort year country pid + /*checking for duplicates/errors in pid*/ + duplicates report year country pid + /*merge with IDs from household register (R file), keep the matches */ + merge 1:m year country pid using "${datapath}/EU-SILC/9 L-2013/2013R.dta" + keep if _merge == 3 + drop _merge + /* R file contains combinations in pid and hid, so pids are not unique. In P pids are unique, so drop duplicates from R*/ + duplicates drop year country pid, force + save "${datapath}/EU-SILC/9 L-2013/2013P.dta", replace + + /*selecting 2012 release data*/ + clear + local filen : dir "${datapath}/EU-SILC/8 L-2012/" files "udb_l12p_ver*" + display `filen' + local filel "${datapath}/EU-SILC/8 L-2012/" + display `"`filel'"' + local fileln : display `"`filel'"' `filen' + use `"`fileln'"', clear + + + local new_var=lower("`var'") + + foreach var of varlist _all { + local new_var = lower("`var'") + cap rename `var' `new_var' + } + + tostring pb030, replace format("%15.0f") + gen year = pb010 + gen pid = pb030 + gen country = pb020 + sort year country pid + /*checking for duplicates/errors in hid*/ + duplicates report year country pid + /* R file contains combinations in pid and hid, so pids are not unique. In P pids are unique, so we can drop the duplicates */ + merge 1:m year country pid using "${datapath}/EU-SILC/8 L-2012/2012R.dta" + keep if _merge == 3 + drop _merge + duplicates drop year country pid, force + save "${datapath}/EU-SILC/8 L-2012/2012P.dta", replace + + /*selecting 2011 release data*/ + clear + local filen : dir "${datapath}/EU-SILC/7 L-2011/" files "udb_l11p_ver*" + display `filen' + local filel "${datapath}/EU-SILC/7 L-2011/" + display `"`filel'"' + local fileln : display `"`filel'"' `filen' + use `"`fileln'"', clear + + + local new_var=lower("`var'") + + foreach var of varlist _all { + local new_var = lower("`var'") + cap rename `var' `new_var' + } + + tostring pb030, replace format("%15.0f") + gen year = pb010 + gen pid = pb030 + gen country = pb020 + sort year country pid + /*checking for duplicates/errors in hid*/ + duplicates report year country pid + /* R file contains combinations in pid and hid, so pids are not unique. In P pids are unique, so we can drop the duplicates */ + merge 1:m year country pid using "${datapath}/EU-SILC/7 L-2011/2011R.dta" + keep if _merge == 3 + drop _merge + duplicates drop year country pid, force + save "${datapath}/EU-SILC/7 L-2011/2011P.dta", replace + + /*selecting 2010 release data*/ + clear + local filen : dir "${datapath}/EU-SILC/6 L-2010/" files "udb_l10p_ver*" + display `filen' + local filel "${datapath}/EU-SILC/6 L-2010/" + display `"`filel'"' + local fileln : display `"`filel'"' `filen' + use `"`fileln'"', clear + + + local new_var=lower("`var'") + + foreach var of varlist _all { + local new_var = lower("`var'") + cap rename `var' `new_var' + } + + tostring pb030, replace format("%15.0f") + gen year = pb010 + gen pid = pb030 + gen country = pb020 + sort year country pid + /*checking for duplicates/errors in hid*/ + duplicates report year country pid + /* R file contains combinations in pid and hid, so pids are not unique. In P pids are unique, so we can drop the duplicates */ + merge 1:m year country pid using "${datapath}/EU-SILC/6 L-2010/2010R.dta" + keep if _merge == 3 + drop _merge + duplicates drop year country pid, force + save "${datapath}/EU-SILC/6 L-2010/2010P.dta", replace + + /*selecting 2009 release data*/ + clear + local filen : dir "${datapath}/EU-SILC/5 L-2009/" files "udb_l09p_ver*" + display `filen' + local filel "${datapath}/EU-SILC/5 L-2009/" + display `"`filel'"' + local fileln : display `"`filel'"' `filen' + use `"`fileln'"', clear + + + local new_var=lower("`var'") + + foreach var of varlist _all { + local new_var = lower("`var'") + cap rename `var' `new_var' + } + + tostring pb030, replace format("%15.0f") + gen year = pb010 + gen pid = pb030 + gen country = pb020 + sort year country pid + /*checking for duplicates/errors in hid*/ + duplicates report year country pid + /* R file contains combinations in pid and hid, so pids are not unique. In P pids are unique, so we can drop the duplicates */ + merge 1:m year country pid using "${datapath}/EU-SILC/5 L-2009/2009R.dta" + keep if _merge == 3 + drop _merge + duplicates drop year country pid, force + save "${datapath}/EU-SILC/5 L-2009/2009P.dta", replace + + /*selecting 2008 release data*/ + clear + local filen : dir "${datapath}/EU-SILC/4 L-2008/" files "udb_l08p_ver*" + display `filen' + local filel "${datapath}/EU-SILC/4 L-2008/" + display `"`filel'"' + local fileln : display `"`filel'"' `filen' + use `"`fileln'"', clear + + + local new_var=lower("`var'") + + foreach var of varlist _all { + local new_var = lower("`var'") + cap rename `var' `new_var' + } + + tostring pb030, replace format("%15.0f") + rename pb010 year + rename pb030 pid + rename pb020 country + sort year country pid + /*checking for duplicates/errors in hid*/ + duplicates report year country pid + /* R file contains combinations in pid and hid, so pids are not unique. In P pids are unique, so we can drop the duplicates */ + merge 1:m year country pid using "${datapath}/EU-SILC/4 L-2008/2008R.dta" + keep if _merge == 3 + drop _merge + duplicates drop year country pid, force + save "${datapath}/EU-SILC/4 L-2008/2008P.dta", replace + +/* 2007 */ + clear + local filen : dir "${datapath}/EU-SILC/3 L-2007/" files "udb_l07p_ver*" + display `filen' + local filel "${datapath}/EU-SILC/3 L-2007/" + display `"`filel'"' + local fileln : display `"`filel'"' `filen' + use `"`fileln'"', clear + + + local new_var=lower("`var'") + + foreach var of varlist _all { + local new_var = lower("`var'") + cap rename `var' `new_var' + } + + tostring pb030, replace format("%15.0f") + gen year = pb010 + gen pid = pb030 + gen country = pb020 + sort year country pid + /*checking for duplicates/errors in pid*/ + duplicates report year country pid + /* R file contains combinations in pid and hid, so pids are not unique. In P pids are unique, so we can drop the duplicates */ + merge 1:m year country pid using "${datapath}/EU-SILC/3 L-2007/2007R.dta" + keep if _merge == 3 + drop _merge + duplicates drop year country pid, force + save "${datapath}/EU-SILC/3 L-2007/2007P.dta", replace + +/* 2006 */ + clear + local filen : dir "${datapath}/EU-SILC/2 L-2006/" files "udb_l06p_ver*" + display `filen' + local filel "${datapath}/EU-SILC/2 L-2006/" + display `"`filel'"' + local fileln : display `"`filel'"' `filen' + use `"`fileln'"', clear + + + local new_var=lower("`var'") + + foreach var of varlist _all { + local new_var = lower("`var'") + cap rename `var' `new_var' + } + + tostring pb030, replace format("%15.0f") + gen year = pb010 + gen pid = pb030 + gen country = pb020 + sort year country pid + /*checking for duplicates/errors in pid*/ + duplicates report year country pid + /* R file contains combinations in pid and hid, so pids are not unique. In P pids are unique, so we can drop the duplicates */ + merge 1:m year country pid using "${datapath}/EU-SILC/2 L-2006/2006R.dta" + keep if _merge == 3 + drop _merge + duplicates drop year country pid, force + save "${datapath}/EU-SILC/2 L-2006/2006P.dta", replace + +/* 2005 */ + clear + local filen : dir "${datapath}/EU-SILC/1 L-2005/" files "udb_l05p_ver*" + display `filen' + local filel "${datapath}/EU-SILC/1 L-2005/" + display `"`filel'"' + local fileln : display `"`filel'"' `filen' + use `"`fileln'"', clear + + + local new_var=lower("`var'") + + foreach var of varlist _all { + local new_var = lower("`var'") + cap rename `var' `new_var' + } + + tostring pb030, replace format("%15.0f") + gen year = pb010 + gen pid = pb030 + gen country = pb020 + sort year country pid + /*checking for duplicates/errors in pid*/ + duplicates report year country pid + /* R file contains combinations in pid and hid, so pids are not unique. In P pids are unique, so we can drop the duplicates */ + merge 1:m year country pid using "${datapath}/EU-SILC/1 L-2005/2005R.dta" + keep if _merge == 3 + drop _merge + duplicates drop year country pid, force + save "${datapath}/EU-SILC/1 L-2005/2005P.dta", replace + + /*merge data from all releases with masterfile. + !! this process is memory intensive. */ + +use "${datapath}/EU-SILC/19 L-2023/2023P.dta", clear + merge 1:1 year upid country uhid using "${datapath}/EU-SILC/18 L-2022/2022P.dta" + drop _merge + merge 1:1 year upid country uhid using "${datapath}/EU-SILC/17 L-2021/2021P.dta" + drop _merge + merge 1:1 year upid country uhid using "${datapath}/EU-SILC/16 L-2020/2020P.dta" + drop _merge + + save "${datapath}/EU-SILC/19 L-2023/masterP.dta", replace + clear + use "${datapath}/EU-SILC/19 L-2023/masterP.dta" + + merge 1:1 year upid country uhid using "${datapath}/EU-SILC/15 L-2019/2019P.dta" + drop _merge + merge 1:1 year upid country uhid using "${datapath}/EU-SILC/14 L-2018/2018P.dta" + drop _merge + merge 1:1 year upid country uhid using "${datapath}/EU-SILC/13 L-2017/2017P.dta" + drop _merge + + save "${datapath}/EU-SILC/19 L-2023/masterP.dta", replace + clear + use "${datapath}/EU-SILC/19 L-2023/masterP.dta" + + merge 1:1 year upid country uhid using "${datapath}/EU-SILC/12 L-2016/2016P.dta" + drop _merge + + save "${datapath}/EU-SILC/19 L-2023/masterP.dta", replace + clear + use "${datapath}/EU-SILC/19 L-2023/masterP.dta" + + merge 1:1 year upid country uhid using "${datapath}/EU-SILC/11 L-2015/2015P.dta" + drop _merge + + save "${datapath}/EU-SILC/19 L-2023/masterP.dta", replace + clear + use "${datapath}/EU-SILC/19 L-2023/masterP.dta" + + merge 1:1 year upid country uhid using "${datapath}/EU-SILC/10 L-2014/2014P.dta" + drop _merge + + save "${datapath}/EU-SILC/19 L-2023/masterP.dta", replace + clear + use "${datapath}/EU-SILC/19 L-2023/masterP.dta" + + merge 1:1 year upid country uhid using "${datapath}/EU-SILC/9 L-2013/2013P.dta" + drop _merge + + save "${datapath}/EU-SILC/19 L-2023/masterP.dta", replace + clear + use "${datapath}/EU-SILC/19 L-2023/masterP.dta" + + merge 1:1 year upid country uhid using "${datapath}/EU-SILC/8 L-2012/2012P.dta" + drop _merge + + save "${datapath}/EU-SILC/19 L-2023/masterP.dta", replace + clear + use "${datapath}/EU-SILC/19 L-2023/masterP.dta" + + merge 1:1 year upid country uhid using "${datapath}/EU-SILC/7 L-2011/2011P.dta" + drop _merge + + save "${datapath}/EU-SILC/19 L-2023/masterP.dta", replace + clear + use "${datapath}/EU-SILC/19 L-2023/masterP.dta" + + merge 1:1 year upid country uhid using "${datapath}/EU-SILC/6 L-2010/2010P.dta" + drop _merge + + save "${datapath}/EU-SILC/19 L-2023/masterP.dta", replace + clear + use "${datapath}/EU-SILC/19 L-2023/masterP.dta" + + merge 1:1 year upid country uhid using "${datapath}/EU-SILC/5 L-2009/2009P.dta" + drop _merge + + save "${datapath}/EU-SILC/19 L-2023/masterP.dta", replace + clear + use "${datapath}/EU-SILC/19 L-2023/masterP.dta" + + merge 1:1 year upid country uhid using "${datapath}/EU-SILC/4 L-2008/2008P.dta" + drop _merge + + save "${datapath}/EU-SILC/19 L-2023/masterP.dta", replace + clear + use "${datapath}/EU-SILC/19 L-2023/masterP.dta" + + merge 1:1 year upid country uhid using "${datapath}/EU-SILC/3 L-2007/2007P.dta" + drop _merge + + save "${datapath}/EU-SILC/19 L-2023/masterP.dta", replace + clear + use "${datapath}/EU-SILC/19 L-2023/masterP.dta" + + merge 1:1 year upid country uhid using "${datapath}/EU-SILC/2 L-2006/2006P.dta" + drop _merge + + save "${datapath}/EU-SILC/19 L-2023/masterP.dta", replace + clear + use "${datapath}/EU-SILC/19 L-2023/masterP.dta" + + merge 1:1 year upid country uhid using "${datapath}/EU-SILC/1 L-2005/2005P.dta" + drop _merge + + /*eliminate variables from the R file*/ + drop rb010 rb010 rb020 rb030 rb040 rb060 rb060_f rb062 rb062_f rb063 rb063_f rb064 rb064_f rb070 rb070_f rb080 rb080_f rb090 rb090_f rb100 /// + rb100_f rb110 rb110_f rb120 rb120_f rb140 rb140_f rb150 rb150_f rb160 rb160_f rb170 rb170_f rb180 rb180_f rb190 rb190_f rb200 rb200_f /// + rb210 rb210_f rb220 rb220_f rb230 rb230_f rb240 rb240_f rb245 rb245_f rb250 rb250_f rb260 rb260_f rb270 rb270_f rx010 rx020 hid db010 /// + db020 db030 db040 db040_f db060 db060_f db062 db062_f db070 db070_f db075 db075_f db095 db095_f db100 db100_f db110 db110_f + + /*drop superflous variables */ + drop drpout_year* slctd_urtgrp* slctd_uhid* merge* nrtgrp* lgstgrp maxgrp drpout_year slctd_rtgrp + + /* This is the personal data file (P) file 2005 - 2023 */ + save "${datapath}/EU-SILC/19 L-2023/masterP.dta", replace diff --git a/input_processing/data_construction/SILC_panel_construction/2005_2023_panel/do_files/05_weights.do b/input_processing/data_construction/SILC_panel_construction/2005_2023_panel/do_files/05_weights.do new file mode 100644 index 0000000..78d7d19 --- /dev/null +++ b/input_processing/data_construction/SILC_panel_construction/2005_2023_panel/do_files/05_weights.do @@ -0,0 +1,270 @@ +/******************************************************************************* +SUMMARY — 05_weights.do (2025 release, extended to 2023) + +This file rescales the survey weights in masterR.dta and masterP.dta so that +weighted totals align with official Eurostat country population figures. + +STEP 1 — Rescale R-file weights + The population dataset is loaded, reshaped to long format, + and merged with masterR.dta on country and year. Two scaled weights are + produced: + - rb060s: scaled version of the base personal weight (rb060), rescaled so + that the weighted total for each country-year matches the + population total. + - rb064s: scaled version of the longitudinal weight (rb064), extended to all + observations for a given individual (upid) before rescaling. + Diagnostic checks are run to verify the scaling rates by country and year. + +STEP 2 — Rescale P-file weights + The same population dataset is merged with masterP.dta on country and year. + A scaled weight pb080s is produced from the personal base weight pb080, + rescaled to match population totals by country-year and rotation group. + +NOTE: Requires population_21_23 — a country-by-year population file covering +2021–2023, constructed from Eurostat population totals. This file must be +prepared before running this do-file. See the population construction section +below for instructions. + +NOTE: The erase section below can be used to delete intermediate annual +YYYYD/H/P/R files after the panel has been successfully built, to free +up disk space. +*/ + +*******************************************************************************/ + + /* Erasing superflous files from the disc */ + + erase "${datapath}/EU-SILC/19 L-2023/2023D.dta" +// erase "${datapath}/EU-SILC/19 L-2023/2023H.dta" + erase "${datapath}/EU-SILC/19 L-2023/2023P.dta" + erase "${datapath}/EU-SILC/19 L-2023/2023R.dta" + + erase "${datapath}/EU-SILC/18 L-2022/2022D.dta" + erase "${datapath}/EU-SILC/18 L-2022/2022H.dta" + erase "${datapath}/EU-SILC/18 L-2022/2022P.dta" + erase "${datapath}/EU-SILC/18 L-2022/2022R.dta" + + erase "${datapath}/EU-SILC/17 L-2021/2021D.dta" + erase "${datapath}/EU-SILC/17 L-2021/2021H.dta" + erase "${datapath}/EU-SILC/17 L-2021/2021P.dta" + erase "${datapath}/EU-SILC/17 L-2021/2021R.dta" + + erase "${datapath}/EU-SILC/15 L-2019/2019D.dta" + erase "${datapath}/EU-SILC/15 L-2019/2019H.dta" + erase "${datapath}/EU-SILC/15 L-2019/2019P.dta" + erase "${datapath}/EU-SILC/15 L-2019/2019R.dta" + + erase "${datapath}/EU-SILC/13 L-2017/2017D.dta" + erase "${datapath}/EU-SILC/13 L-2017/2017H.dta" + erase "${datapath}/EU-SILC/13 L-2017/2017P.dta" + erase "${datapath}/EU-SILC/13 L-2017/2017R.dta" + + erase "${datapath}/EU-SILC/12 L-2016/2016D.dta" + erase "${datapath}/EU-SILC/12 L-2016/2016H.dta" + erase "${datapath}/EU-SILC/12 L-2016/2016P.dta" + erase "${datapath}/EU-SILC/12 L-2016/2016R.dta" + + erase "${datapath}/EU-SILC/11 L-2015/2015D.dta" + erase "${datapath}/EU-SILC/11 L-2015/2015H.dta" + erase "${datapath}/EU-SILC/11 L-2015/2015P.dta" + erase "${datapath}/EU-SILC/11 L-2015/2015R.dta" + + erase "${datapath}/EU-SILC/10 L-2014/2014D.dta" + erase "${datapath}/EU-SILC/10 L-2014/2014H.dta" + erase "${datapath}/EU-SILC/10 L-2014/2014P.dta" + erase "${datapath}/EU-SILC/10 L-2014/2014R.dta" + + erase "${datapath}/EU-SILC/9 L-2013/2013D.dta" + erase "${datapath}/EU-SILC/9 L-2013/2013H.dta" + erase "${datapath}/EU-SILC/9 L-2013/2013P.dta" + erase "${datapath}/EU-SILC/9 L-2013/2013R.dta" + + erase "${datapath}/EU-SILC/8 L-2012/2012D.dta" + erase "${datapath}/EU-SILC/8 L-2012/2012H.dta" + erase "${datapath}/EU-SILC/8 L-2012/2012P.dta" + erase "${datapath}/EU-SILC/8 L-2012/2012R.dta" + + erase "${datapath}/EU-SILC/7 L-2011/2011D.dta" + erase "${datapath}/EU-SILC/7 L-2011/2011H.dta" + erase "${datapath}/EU-SILC/7 L-2011/2011P.dta" + erase "${datapath}/EU-SILC/7 L-2011/2011R.dta" + + erase "${datapath}/EU-SILC/6 L-2010/2010D.dta" + erase "${datapath}/EU-SILC/6 L-2010/2010H.dta" + erase "${datapath}/EU-SILC/6 L-2010/2010P.dta" + erase "${datapath}/EU-SILC/6 L-2010/2010R.dta" + + erase "${datapath}/EU-SILC/5 L-2009/2009D.dta" + erase "${datapath}/EU-SILC/5 L-2009/2009H.dta" + erase "${datapath}/EU-SILC/5 L-2009/2009P.dta" + erase "${datapath}/EU-SILC/5 L-2009/2009R.dta" + + erase "${datapath}/EU-SILC/4 L-2008/2008D.dta" + erase "${datapath}/EU-SILC/4 L-2008/2008H.dta" + erase "${datapath}/EU-SILC/4 L-2008/2008P.dta" + erase "${datapath}/EU-SILC/4 L-2008/2008R.dta" + + erase "${datapath}/EU-SILC/3 L-2007/2007D.dta" + erase "${datapath}/EU-SILC/3 L-2007/2007H.dta" + erase "${datapath}/EU-SILC/3 L-2007/2007P.dta" + erase "${datapath}/EU-SILC/3 L-2007/2007R.dta" + + erase "${datapath}/EU-SILC/2 L-2006/2006D.dta" + erase "${datapath}/EU-SILC/2 L-2006/2006H.dta" + erase "${datapath}/EU-SILC/2 L-2006/2006P.dta" + erase "${datapath}/EU-SILC/2 L-2006/2006R.dta" + + erase "${datapath}/EU-SILC/1 L-2005/2005D.dta" + erase "${datapath}/EU-SILC/1 L-2005/2005H.dta" + erase "${datapath}/EU-SILC/1 L-2005/2005P.dta" + erase "${datapath}/EU-SILC/1 L-2005/2005R.dta" + + display "Rescaling weights in R file." + + /* import the total population for each country from eurostat */ + /* add latest years*/ +/* clear + import excel "$csv_path/populations/country_population_update.xlsx", sheet("to_load") firstrow + + rename B a2023 + rename C a2022 + rename D a2021 + rename GEO country + + drop if country == "" + + save "$data_dir/population_21_23", replace */ + + /* earlier year provided by GESIS */ + /*clear + findfile totalpopulation_2003_2020.dta, path(BASE;SITE;.;PERSONAL;PLUS) + return list + use`"`r(fn)'"'*/ + + use "$input_data_dir/totalpopulation_2003_2020", clear + + rename B a2003 + rename C a2004 + rename D a2005 + rename E a2006 + rename F a2007 + rename G a2008 + rename H a2009 + rename I a2010 + rename J a2011 + rename K a2012 + rename L a2013 + rename M a2014 + rename N a2015 + rename O a2016 + rename P a2017 + rename Q a2018 + rename R a2019 + rename S a2020 + + /* merge all years*/ + merge 1:1 country using "$input_data_dir/population_21_23" + drop if _m == 2 + drop _m + + save "$data_dir/population_03_23", replace + + reshape long a, i(country) j(year) + rename a pop + destring pop, replace + drop if pop == . + merge 1:m country year using "${datapath}/EU-SILC/19 L-2023/masterR.dta" + drop if _merge == 1 + drop _merge + + /* now calculate scales for the base weights for individuals in R file rscale */ + bysort year country: egen wcountry = total (rb060) + bysort year country urtgrp: egen wgroup = total( rb060 ) + gen rscale = ( wgroup / wcountry ) + gen rb060s = rscale * rb060 + + /* check scaled weight */ + gen test1 = ( wcountry - pop ) / pop + bysort year country: egen test2 = total (rb060s) + gen smwrate60 = ( test2 - pop ) / pop + + /* what does this do? */ + tab country , summarize (smwrate60) + tab year, summarize (rscale) + drop test1 test2 wgroup wcountry + + /* extend presence of rb064 to all observations pertaining to a certain unit */ + bysort upid : egen lrb064 = max(rb064) + bysort year country: egen lwcountry = total (lrb064) + bysort year country urtgrp: egen lwgroup = total( lrb064 ) + gen lrscale = ( lwgroup / lwcountry ) + gen rb064s = lrscale * lrb064 + + /* check scaled weight */ + gen test1 = ( lwcountry - pop ) / pop + bysort year country: egen test2 = total (rb064s) + gen smwrate64 = ( test2 - pop ) / pop + + /* what does this do? */ + tab country , summarize (smwrate64) + tab year, summarize (lrscale) + drop test1 test2 lwgroup lwcountry + + save "${datapath}/EU-SILC/19 L-2023/masterR.dta", replace + +/**/ + display "Rescaling weights in P file." + + /*import the total population for each country from eurostat*/ + /*total population*/ + /*clear + findfile totalpopulation_2003_2020.dta, path(BASE;SITE;.;PERSONAL;PLUS) + return list + use`"`r(fn)'"' + rename B a2003 + rename C a2004 + rename D a2005 + rename E a2006 + rename F a2007 + rename G a2008 + rename H a2009 + rename I a2010 + rename J a2011 + rename K a2012 + rename L a2013 + rename M a2014 + rename N a2015 + rename O a2016 + rename P a2017 + rename Q a2018 + rename R a2019 + rename S a2020 */ + + use "$data_dir/population_03_23", clear + + reshape long a, i(country) j(year) + rename a pop + destring pop, replace + drop if pop == . + merge 1:m country year using "${datapath}/EU-SILC/19 L-2023/masterP.dta" + drop if _merge == 1 + drop _merge + + /*now calculate scales for the base weights for individuals in R file rscale*/ + bysort year country: egen wcountry = total (pb080) + bysort year country urtgrp: egen wgroup = total( pb080 ) + gen pscale = ( wgroup / wcountry ) + gen pb080s = pscale * pb080 + + /*check scaled weight*/ + gen test1 = ( wcountry - pop ) / pop + bysort year country: egen test2 = total (pb080s) + gen smwrate80 = ( test2 - pop ) / pop + + /*what does this do?*/ + tab country , summarize (smwrate80) + tab year, summarize (pscale) + drop test1 test2 wgroup wcountry + + save "${datapath}/EU-SILC/19 L-2023/masterP.dta", replace + //display "The files masterP.dta, masterR.dta, masterH.dta and masterD.dta are ready for use and can be found in ${datapath}/EU-SILC/19 L-2023/ " as result diff --git a/input_processing/data_construction/SILC_panel_construction/README_SILC_panel_construction.txt b/input_processing/data_construction/SILC_panel_construction/README_SILC_panel_construction.txt new file mode 100644 index 0000000..2e8e015 --- /dev/null +++ b/input_processing/data_construction/SILC_panel_construction/README_SILC_panel_construction.txt @@ -0,0 +1,33 @@ +Construction of the EU-SILC Longitudinal Panel + +Date: 6/6/26 +Author: A Burdett + +Many SimPaths inputs are based on longitudinal EU-SILC data. A formatted version of these data is used to construct the unique input dataset (UID), which is used for estimating processes and initialising the simulation. These data are processed using the GESIS compilation program (eusilcpanel_2020), which combines the raw longitudinal EU-SILC files into harmonised panel datasets (D, H, R and P). + +The downloadable version of the GESIS program constructs harmonised longitudinal datasets (D, H, P and R) containing all available panel observations for survey years 2005–2020. For SimPaths, we use an extended version of the compilation program that has been updated to process the 2024 EU-SILC longitudinal release, producing equivalent datasets covering 2005–2023. + +One important complication is that some variables used in the estimation (most notably the economic activity and education variables) change names and coding from 2021 onwards. Because the GESIS longitudinal compilation is constructed by appending complete panels backwards through time, observations prior to 2021 for individuals whose panel extends into 2021 or later are compiled using the post-2021 variable definitions. As a result, the pre-2021 values of these variables are missing for those individuals. + +To overcome this issue, the SimPaths data construction combines information from two compiled GESIS panels: + + • 2005–2020 panel: constructed using the original GESIS compilation program and the earlier EU-SILC longitudinal release. + • 2005–2023 panel: constructed using the extended GESIS compilation program and the 2024 EU-SILC longitudinal release. + +The variables affected by the post-2020 coding changes are taken from the 2005–2020 panel and merged into the extended 2005–2023 panel. This preserves complete historical information for these variables while allowing the panel to be extended through to 2023. + +The resulting datasets (D, H, R and P) are then combined to construct the UID using the do-files contained in the “data_construction” folder. The UID forms the basis for simulation initialisation, process estimation and validation of the simulated output. + +Workflow summary: + +1. Obtain the required longitudinal EU-SILC releases. + +2. Run the GESIS compilation program using the earlier longitudinal release to produce the harmonised 2005-2020 D, H, R, and P datasets. + +3. Run the extended version of the GESIS compilation program using the 2024 longitudinal release to produce the harmonised 2005-2023 D, H, R, and P datasets. + +4. Run the SimPaths UID construction files contained in the ”data_construction” folder to create the final UID. During this process, the variables affected by the post-2020 coding changes are merged from the 2005-2020 dataset into the 2005-2023 dataset. + +Links: +GESIS EU-SILC panel compilation program (eusilcpanel_2020) - https://www.gesis.org/en/missy/materials/EU-SILC/tools/datahandling + diff --git a/input_processing/data_construction/UID_construction/.DS_Store b/input_processing/data_construction/UID_construction/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..5e2ad7ea07dc0df94cffb71de7c1a91b715eae0b GIT binary patch literal 6148 zcmeHKyKciU45ZQw2D)U-sHGp^KTrba3p`{k(g5uc!&$m}%ZGL7XSK&iil#y`c4&_P zbr!{w>>ET=MAG_o--;|mq=75SNr&2WUA?kdMCQPxdpwQ9vC6s~Q0`1N(jT7N{s-UU z!`OAZ!?;6U^VUABwjb}SGiMn0SJ#d&)qc0%gsW753Qz$mKn4DX0;t()&AlT>DnJFO zz?TAgKV-OK37j15>cC(v08oG9&G2l!1Tbp=SOO~|9?s1j0#YJKczsLcYU|UCq-?o yK2B?GhCjm3hFnX>V5Jynr5F<}#mjGbS)Z|90w+hKBW`pce*{z)8Ws2r1wH{rQzD}P literal 0 HcmV?d00001 diff --git a/input_processing/data_construction/PL/00_master_data_set_construction_PL.do b/input_processing/data_construction/UID_construction/PL/00_master_data_set_construction_PL.do similarity index 95% rename from input_processing/data_construction/PL/00_master_data_set_construction_PL.do rename to input_processing/data_construction/UID_construction/PL/00_master_data_set_construction_PL.do index 6c9c24f..7767654 100644 --- a/input_processing/data_construction/PL/00_master_data_set_construction_PL.do +++ b/input_processing/data_construction/UID_construction/PL/00_master_data_set_construction_PL.do @@ -13,7 +13,8 @@ * ******************************************************************************** * NOTES: -* ENSURE HAVE ALREADY RUN 00_master_conditions.do FILE. +* ENSURE HAVE ALREADY RUN 00_master_conditions_PL.do FILE. +* This can be found in the main "input_processing" folder. Called below. * * The input data for this process comes from longitudinal EU-SILC. We * follow the procedure set out in the *GESIS Papers 2022/10* to construct diff --git a/input_processing/data_construction/PL/01_prepare_pooled_data_PL.do b/input_processing/data_construction/UID_construction/PL/01_prepare_pooled_data_PL.do similarity index 100% rename from input_processing/data_construction/PL/01_prepare_pooled_data_PL.do rename to input_processing/data_construction/UID_construction/PL/01_prepare_pooled_data_PL.do diff --git a/input_processing/data_construction/PL/02_create_variables_PL.do b/input_processing/data_construction/UID_construction/PL/02_create_variables_PL.do similarity index 100% rename from input_processing/data_construction/PL/02_create_variables_PL.do rename to input_processing/data_construction/UID_construction/PL/02_create_variables_PL.do diff --git a/input_processing/data_construction/PL/03_create_benefit_units_PL.do b/input_processing/data_construction/UID_construction/PL/03_create_benefit_units_PL.do similarity index 100% rename from input_processing/data_construction/PL/03_create_benefit_units_PL.do rename to input_processing/data_construction/UID_construction/PL/03_create_benefit_units_PL.do diff --git a/input_processing/data_construction/PL/04_reweight_PL.do b/input_processing/data_construction/UID_construction/PL/04_reweight_PL.do similarity index 100% rename from input_processing/data_construction/PL/04_reweight_PL.do rename to input_processing/data_construction/UID_construction/PL/04_reweight_PL.do diff --git a/input_processing/data_construction/PL/05_drop_hholds_slice_and_refactoring_PL.do b/input_processing/data_construction/UID_construction/PL/05_drop_hholds_slice_and_refactoring_PL.do similarity index 100% rename from input_processing/data_construction/PL/05_drop_hholds_slice_and_refactoring_PL.do rename to input_processing/data_construction/UID_construction/PL/05_drop_hholds_slice_and_refactoring_PL.do diff --git a/input_processing/data_construction/PL/06_check_yearly_data_PL.do b/input_processing/data_construction/UID_construction/PL/06_check_yearly_data_PL.do similarity index 100% rename from input_processing/data_construction/PL/06_check_yearly_data_PL.do rename to input_processing/data_construction/UID_construction/PL/06_check_yearly_data_PL.do diff --git a/input_processing/data_construction/PL/07_version_comparison_PL.do b/input_processing/data_construction/UID_construction/PL/07_version_comparison_PL.do similarity index 100% rename from input_processing/data_construction/PL/07_version_comparison_PL.do rename to input_processing/data_construction/UID_construction/PL/07_version_comparison_PL.do diff --git a/input_processing/data_construction/PL/extra_var_info/prepare_pooled_data _05_20.do b/input_processing/data_construction/UID_construction/PL/extra_var_info/prepare_pooled_data _05_20.do similarity index 100% rename from input_processing/data_construction/PL/extra_var_info/prepare_pooled_data _05_20.do rename to input_processing/data_construction/UID_construction/PL/extra_var_info/prepare_pooled_data _05_20.do diff --git a/input_processing/data_construction/PL/extra_var_info/vars_05_20_PL2.do b/input_processing/data_construction/UID_construction/PL/extra_var_info/vars_05_20_PL2.do similarity index 100% rename from input_processing/data_construction/PL/extra_var_info/vars_05_20_PL2.do rename to input_processing/data_construction/UID_construction/PL/extra_var_info/vars_05_20_PL2.do diff --git a/input_processing/regression_estimates/PL/00_master_regression_estimates_PL.do b/input_processing/regression_estimates/PL/00_master_regression_estimates_PL.do index 550dbb3..2e4ced6 100644 --- a/input_processing/regression_estimates/PL/00_master_regression_estimates_PL.do +++ b/input_processing/regression_estimates/PL/00_master_regression_estimates_PL.do @@ -9,7 +9,9 @@ * Aleksandra Kolndrekaj * LAST UPDATE: March 2026 (AB) ******************************************************************************** -* NOTES: +* NOTES: +* ENSURE HAVE ALREADY RUN 00_master_conditions_PL.do FILE. +* This can be found in the main "input_processing" folder. Called below. * *******************************************************************************/ From f172e49cc516d8ee4a087614825c34634ee04222 Mon Sep 17 00:00:00 2001 From: Ashley Burdett <76621383+ANBurdett@users.noreply.github.com> Date: Wed, 8 Jul 2026 18:08:13 +0100 Subject: [PATCH 2/5] Delete input_processing/data_construction/.DS_Store --- input_processing/data_construction/.DS_Store | Bin 6148 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 input_processing/data_construction/.DS_Store diff --git a/input_processing/data_construction/.DS_Store b/input_processing/data_construction/.DS_Store deleted file mode 100644 index 47fd5903a1c169446bc93b17c382f20bf1e840bd..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHK!AiqG5Pe&FhG=6(M z>;4{PHT#BB7Wb`e1{jl%lx*zIz?U)2oafJnr<(H)aUWTBiQAG_X9doB`Hs zk?0_xjmCg6U<|AoknclG6-*TCfd13L!M6a!KFwY@mtI0*V#P$U4#*vf3!%gi>a@jh zA)NMD{Sw7GUR!G6sx+ zKgEFSr&&5;N?Kc6$w{q^sn=8y@v8%_LpbrJn7&eqPpDpKk7Ytk6zhPrQ0zy*(_n)! I@S_a81L9wE5C8xG From 78ec5c2adb52146758525a65e81b13cc28e84b53 Mon Sep 17 00:00:00 2001 From: Ashley Burdett <76621383+ANBurdett@users.noreply.github.com> Date: Wed, 8 Jul 2026 18:08:24 +0100 Subject: [PATCH 3/5] Delete input_processing/data_construction/UID_construction/.DS_Store --- .../data_construction/UID_construction/.DS_Store | Bin 6148 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 input_processing/data_construction/UID_construction/.DS_Store diff --git a/input_processing/data_construction/UID_construction/.DS_Store b/input_processing/data_construction/UID_construction/.DS_Store deleted file mode 100644 index 5e2ad7ea07dc0df94cffb71de7c1a91b715eae0b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHKyKciU45ZQw2D)U-sHGp^KTrba3p`{k(g5uc!&$m}%ZGL7XSK&iil#y`c4&_P zbr!{w>>ET=MAG_o--;|mq=75SNr&2WUA?kdMCQPxdpwQ9vC6s~Q0`1N(jT7N{s-UU z!`OAZ!?;6U^VUABwjb}SGiMn0SJ#d&)qc0%gsW753Qz$mKn4DX0;t()&AlT>DnJFO zz?TAgKV-OK37j15>cC(v08oG9&G2l!1Tbp=SOO~|9?s1j0#YJKczsLcYU|UCq-?o yK2B?GhCjm3hFnX>V5Jynr5F<}#mjGbS)Z|90w+hKBW`pce*{z)8Ws2r1wH{rQzD}P From ccba9f63010ff90aca0f6c37f119b7fc238ab6eb Mon Sep 17 00:00:00 2001 From: Ashley Burdett <76621383+ANBurdett@users.noreply.github.com> Date: Wed, 8 Jul 2026 18:08:43 +0100 Subject: [PATCH 4/5] Delete input_processing/data_construction/SILC_panel_construction/2005_2023_panel/.DS_Store --- .../2005_2023_panel/.DS_Store | Bin 10244 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 input_processing/data_construction/SILC_panel_construction/2005_2023_panel/.DS_Store diff --git a/input_processing/data_construction/SILC_panel_construction/2005_2023_panel/.DS_Store b/input_processing/data_construction/SILC_panel_construction/2005_2023_panel/.DS_Store deleted file mode 100644 index ba5594226f8225c059fee837924cbd47a67d1021..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 10244 zcmeHMU2GIp6uxJ6u`>+NDFOv{k*$RYO@S5&Ey~aCwt!HerMu8=p<} zx%d3sd(XG$+}nFg2!Uu`X(l8@2;uRgRGNj=6oty^S5Y{ZBOwK}C;DJ>ENy186`CHy zL%>78L%>78L%>7edO-lsY+87cQLpw8@DT727)OBa46z5&@bnexeYW*hDVBzlUi`jmt7-dwWVZQ^@0!hOg4o!R; z=t5J>c!iiGNsbVW=tLz}YQ)OAbDU3Qb>pd^b6vf$rL~q~0L;Xof%3Er;s>`-kP}^l&wUSb`g<$MV zn?_eoEg%RzX;aT<43yNQWj&ooWi_JfG3_|hPUlQr?=mgT)=Z-}VXH@NDx(9Hi4uL5 znX}uR!4evd&$Q;!IXbh{*gBx*ETn?)(wy3*%T}yf+p;OLeb?UZOVv`%+#BbKVjtpc z*xC_I&Fvr26nm&ww$iGhXvSbqL9?`!uEq^Hqt^I+f^cc!qQ%vLVBn^bXtD_})$@sb zO1m?!*~gqYLUjT0kb(~GF%>$CBAw+0N3y&w5I9IBZ8;Tdh36gi%7%a#x6oZlb`@8z z5ykj028d+g66nI$trr9PbvdQ#k|Hp(){SDIr@*|GI$Y-FEn*;nuU;8+(W-$)mmO^j z1P18LyJgGn%&G>i>V0?m9VqLwhUA>uIf60jayyXg{Eo<{M%xt4HmT47q%N?J=X>dZ z5k*mzOVx`OFO0>hg`^n;4RZp20+t%mMV{yDO95FQZeY#&chW_tJ1B!*MtPV{j7gfirMFJOB^Eqwpj=1y92>@G`s( zZ^B#fHe7&@;WAu-tMDa!2j9aF@C!GS6S-O3d~N|(%hhoW+-7bo7v>^dlc4d2_%)dX%*aEhkIq}ybIpGb63aiJ=bmCU7f=+ZT^DV(DH^g>svQR z$0L%3a4Bjh=7N&|WohFifD;{)BA13>Ah<+adaaz?kC~$$ox{g^O&U)=XAy9;5q?wQSz~FW3WZwC19Of#Z3ZcrQ~Pbtztx=8-+xi zey}~boyW|Q{;`n%fLtV3$QR^W4EaC62Rzhbn6H4<&=hM7M_O};6*qGufZE|-c5jy zF#JD*uO@}`-tv%!pQj4xiH9{I#pk;IbPnr^(9f8EAKKIDl^x!`B3(koScO>5&`o0y zWi Date: Wed, 8 Jul 2026 18:08:55 +0100 Subject: [PATCH 5/5] Delete input_processing/data_construction/SILC_panel_construction/2005_2023_panel/do_files/.DS_Store --- .../2005_2023_panel/do_files/.DS_Store | Bin 6148 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 input_processing/data_construction/SILC_panel_construction/2005_2023_panel/do_files/.DS_Store diff --git a/input_processing/data_construction/SILC_panel_construction/2005_2023_panel/do_files/.DS_Store b/input_processing/data_construction/SILC_panel_construction/2005_2023_panel/do_files/.DS_Store deleted file mode 100644 index c6395d6ef5322d6cd9ca1360359904f143c24ad5..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHKu};H447E!oieTu-gboQYF;FD-P=zn(2Y}il7#bx~>3%=M!o)W)z$YL!R(3vs z=d*ji2Ji~kElyT9jIVy1H&AVbO{TDCv>*Ye(f!PYL$fHU9> ztPBIJnRYf;CSy<38E^)c#ejSt0#qgfg!a3z&6ZDFqd9JVuE327!|PsVGRXp zC|irc8V-9fzsxWyYB;eqA8aSHbts%p$NnL?6K9G(Is?u?oq;vI>`MJVUEKfIgZ#=F za0dPr1KiFBc^|iAwY70`QfmYB94aDyQSl=LlT?b~E2VfFngsSB6JTZ-6=8w+k3gcq K2WMcZ47>xBi&jVg