diff --git a/.gitignore b/.gitignore index 7ca9e7a8f..e79eea954 100644 --- a/.gitignore +++ b/.gitignore @@ -153,6 +153,36 @@ input/EUROMODoutput/baseline/* input/InitialPopulations/original/* input/InitialPopulations/base/* +input/lifetime_incomes/sample/1994.csv +input/lifetime_incomes/sample/1995.csv +input/lifetime_incomes/sample/1996.csv +input/lifetime_incomes/sample/1997.csv +input/lifetime_incomes/sample/1998.csv +input/lifetime_incomes/sample/1999.csv +input/lifetime_incomes/sample/2000.csv +input/lifetime_incomes/sample/2001.csv +input/lifetime_incomes/sample/2002.csv +input/lifetime_incomes/sample/2003.csv +input/lifetime_incomes/sample/2004.csv +input/lifetime_incomes/sample/2005.csv +input/lifetime_incomes/sample/2006.csv +input/lifetime_incomes/sample/2007.csv +input/lifetime_incomes/sample/2008.csv +input/lifetime_incomes/sample/2009.csv +input/lifetime_incomes/sample/2010.csv +input/lifetime_incomes/sample/2011.csv +input/lifetime_incomes/sample/2012.csv +input/lifetime_incomes/sample/2013.csv +input/lifetime_incomes/sample/2014.csv +input/lifetime_incomes/sample/2015.csv +input/lifetime_incomes/sample/2016.csv +input/lifetime_incomes/sample/2017.csv +input/lifetime_incomes/sample/2018.csv +input/lifetime_incomes/sample/2019.csv +input/lifetime_incomes/sample/2020.csv +input/lifetime_incomes/sample/2021.csv +input/lifetime_incomes/sample/2022.csv +input/lifetime_incomes/sample/2023.csv # Documentation build and browser-check output /site/ diff --git a/documentation/repository-guide.md b/documentation/repository-guide.md index e038e681f..82a95abc1 100644 --- a/documentation/repository-guide.md +++ b/documentation/repository-guide.md @@ -280,7 +280,7 @@ When IO is enabled, this package creates projected income paths for birth cohort | Class | Purpose | | --- | --- | -| `ManagerProjectLifetimeIncomes` | Generates the synthetic income trajectory database for all birth cohorts in the simulation horizon. | +| `ManagerLifetimeIncomes` | Generates the synthetic income trajectory database for all birth cohorts in the simulation horizon. | | `LifetimeIncomeImputation` | Matches each simulated person to a donor income trajectory via binary search on the income CDF. | | `AnnualIncome` | Implements the AR(2) income process with age-gender anchoring. | | `BirthCohort` | Groups individuals by birth year for cohort-level income projection. | diff --git a/documentation/wiki/developer-guide/repository-guide.md b/documentation/wiki/developer-guide/repository-guide.md index 25128db8a..3f0fb81dc 100644 --- a/documentation/wiki/developer-guide/repository-guide.md +++ b/documentation/wiki/developer-guide/repository-guide.md @@ -263,7 +263,7 @@ When IO is enabled, this package creates projected income paths for birth cohort | Class | Purpose | | --- | --- | -| `ManagerProjectLifetimeIncomes` | Generates the synthetic income trajectory database for all birth cohorts in the simulation horizon. | +| `ManagerLifetimeIncomes` | Generates the synthetic income trajectory database for all birth cohorts in the simulation horizon. | | `LifetimeIncomeImputation` | Matches each simulated person to a donor income trajectory via binary search on the income CDF. | | `AnnualIncome` | Implements the AR(2) income process with age-gender anchoring. | | `BirthCohort` | Groups individuals by birth year for cohort-level income projection. | diff --git a/input/DatabaseCountryYear.xlsx b/input/DatabaseCountryYear.xlsx index b9306fb4c..0279c60dc 100644 Binary files a/input/DatabaseCountryYear.xlsx and b/input/DatabaseCountryYear.xlsx differ diff --git a/input/InitialPopulations/compile/07_was_wealth_data.do b/input/InitialPopulations/compile/07_was_wealth_data.do index 1822a6c65..69fc0500a 100644 --- a/input/InitialPopulations/compile/07_was_wealth_data.do +++ b/input/InitialPopulations/compile/07_was_wealth_data.do @@ -265,7 +265,7 @@ foreach file in "${dir_was_data}/was_round_5_person_eul_oct_2020.dta" /// recode dvgiemp dvgise dvgippen dvgiinv dvoigrrannual_i dvoiggtannual_i dvoigegannual_i dvoigfrannual_i dvoigmaannual_i dvoigroannual_i dvoigopannual_i (missing=0) gen inci = dvgiemp + dvgise + dvgippen + dvgiinv + dvoigrrannual_i + dvoiggtannual_i + dvoigegannual_i + dvoigfrannual_i + dvoigmaannual_i + dvoigroannual_i + dvoigopannual_i gen pensioner = (dvgippen > 0.01) - + // fix some partner person numbers duplicates tag case partno, gen(dup) // Rows should be unique in case partno, assuming monogamy replace dup=0 if partno==17 // partno=17 if no partner in household @@ -774,7 +774,7 @@ foreach file in "${dir_was_data}/was_round_5_person_eul_oct_2020.dta" /// cc_debt cc_debt_trans cc_debt_pers hp_debt loan_debt loan_debt_exc_slc loan_debt_slc /// loan_doorstep loan_pawnbroker loan_payday informal_loan_debt highcost_loan_debt /// mail_debt unsec_debt_total unsec_debt_core highcost_debt_proxy lowcost_debt /// - op_memb pp_membu op_membu op_db op_dc pensioners /// + op_memb pp_membu op_membu op_db op_dc pensioners /// pp_core_annual pp_fallback_annual pp_contrib_annual pp_contrib_rate_income priv_total_contrib_rate_inc /// pp_contrib_source_core pp_contrib_source_fallback occ_emp_contrib_rate occ_emp_contrib_annual /// caser5 caser6 caser7 personw5 personw6 personr7 @@ -997,8 +997,6 @@ label var was_pid "Synthetic longitudinal person ID (WAS rounds 5-8 linkage)" sort was_pid year was_round save "${dir_data}/was_wealthdata.dta", replace - - /**************************************************************/ * * ANALYSIS OF WORKING VARIABLES diff --git a/input/regression_files/reg_lifetime_incomes.xlsx b/input/regression_files/reg_lifetime_incomes.xlsx index b0c0a919d..52d324d78 100644 Binary files a/input/regression_files/reg_lifetime_incomes.xlsx and b/input/regression_files/reg_lifetime_incomes.xlsx differ diff --git a/src/main/java/simpaths/data/Parameters.java b/src/main/java/simpaths/data/Parameters.java index 34d028c7f..7a6368ff2 100644 --- a/src/main/java/simpaths/data/Parameters.java +++ b/src/main/java/simpaths/data/Parameters.java @@ -23,7 +23,6 @@ import org.apache.commons.math3.util.Pair; import org.apache.poi.openxml4j.util.ZipSecureFile; import simpaths.model.decisions.Grids; -import simpaths.model.lifetime_incomes.EquivalisedIncomeCDF; import simpaths.model.taxes.DonorTaxUnit; import simpaths.model.taxes.MatchFeature; import simpaths.model.taxes.database.TaxDonorDataParser; @@ -2408,12 +2407,13 @@ public static double getMortalityProbability(Gender demSex, int age, int year) { public static Double getEquivalisedIncome(Gender demSex, int age, int year) { Double val = null; - if (year<=equivalisedIncomeMaxYear && year>=equivalisedIncomeMinYear && age<=equivalisedIncomeMaxAge) { + int ageHere = Math.min(age, equivalisedIncomeMaxAge); + if (year<=equivalisedIncomeMaxYear && year>=equivalisedIncomeMinYear) { // obtain value from observed matrix - Number nn = ((Number) equivalisedIncomeByGenderAgeYear.getValue(demSex.toString(), age, year)); + Number nn = ((Number) equivalisedIncomeByGenderAgeYear.getValue(demSex.toString(), ageHere, year)); if (nn==null) { - throw new IllegalAccessError("ERROR - problem evaluating mean equivalised income for year: " + year + ", age: " + age + " and demSex " + demSex.toString()); + throw new IllegalAccessError("ERROR - problem evaluating mean equivalised income for year: " + year + ", age: " + ageHere + " and demSex " + demSex.toString()); } val = nn.doubleValue(); } diff --git a/src/main/java/simpaths/model/BenefitUnit.java b/src/main/java/simpaths/model/BenefitUnit.java index 909bb1a42..3613cb8a0 100644 --- a/src/main/java/simpaths/model/BenefitUnit.java +++ b/src/main/java/simpaths/model/BenefitUnit.java @@ -4947,6 +4947,8 @@ public Double getDisposableIncomeMonthly() { return yDispMonth; } + public Double getYDispMonth() { return yDispMonth; } + public Double getGrossIncomeMonthly() { return yGrossMonth; } @@ -5459,7 +5461,7 @@ public double getXDiscConsumptionAnnual(boolean throwError) { } public double getEquivalisedWeight() { - //Equivalence scale gives a weight of 1.0 to the first adult; + //OECD (BHC) equivalence scale gives a weight of 1.0 to the first adult; //0.5 to the second and each subsequent person aged 14 and over; //0.3 to each child aged under 14. double weight = 0.0; diff --git a/src/main/java/simpaths/model/Person.java b/src/main/java/simpaths/model/Person.java index 5b2fc44ef..a691cf9af 100644 --- a/src/main/java/simpaths/model/Person.java +++ b/src/main/java/simpaths/model/Person.java @@ -25,8 +25,8 @@ import simpaths.model.decisions.Axis; import simpaths.model.decisions.DecisionParams; import simpaths.model.enums.*; -import simpaths.model.lifetime_incomes.AnnualIncome; -import simpaths.model.lifetime_incomes.Individual; +import simpaths.model.lifetime_incomes.RegressionProxy; +import simpaths.model.lifetime_incomes.TemplateIndividual; import simpaths.model.person.PrivatePension; import java.math.BigDecimal; @@ -51,9 +51,6 @@ public class Person implements EventListener, IDoubleSource, IIntSource, Weight, @JoinColumn(name = "burun", referencedColumnName = "simulation_run"), @JoinColumn(name = "prid", referencedColumnName = "working_id") }) private BenefitUnit benefitUnit; - @ManyToOne(fetch = FetchType.EAGER, cascade = CascadeType.REFRESH) - @JoinColumn(name = "income_id", referencedColumnName = "id") - private Individual ltIncomeDonor; // identifiers private Long idPersOriginal; @@ -151,6 +148,7 @@ public class Person implements EventListener, IDoubleSource, IIntSource, Weight, @Lag(field="yBenUCReceivedFlag") @Transient private Boolean yBenUCReceivedFlagL1; @Column(name="yBenNonUCReceivedFlag") private Boolean yBenNonUCReceivedFlag; // Person receives a benefit which is not UC @Lag(field="yBenNonUCReceivedFlag") @Transient private Boolean yBenNonUCReceivedFlagL1; + @Transient TemplateIndividual ltIncomeDonor; @NullInitialised @Column(name="yLifeTime") private Double yLifeTime; // mean annual equivalised household disposable income by age @Enumerated(EnumType.STRING) private Labour labHrsWorkEnumWeek; //Number of hours of labour supplied each week @@ -352,11 +350,11 @@ public Person (Person originalPerson, long statSeed, SampleEntry demEnterSample) this.demEnterSample = demEnterSample; - if (originalPerson.ltIncomeDonor!=null) { + if (demEnterSample != SampleEntry.InputData && demEnterSample != SampleEntry.ProcessedInputData && Parameters.lifetimeIncomeImpute) { + this.ltIncomeDonor = originalPerson.ltIncomeDonor; - } - if (Parameters.lifetimeIncomeImpute) this.yLifeTime = originalPerson.getYLifeTime(); + } demAge = originalPerson.demAge; demAgeSq = demAge * demAge; @@ -2613,6 +2611,95 @@ public enum Variables { labStatusC4Retired_MaleL1, labStatusPartnerC3NotEmployedL1, labStatusC4NotEmployed_MaleL1, + Age, + Age0, + Age1, + Age2, + Age3, + Age4, + Age5, + Age6, + Age7, + Age8, + Age9, + Age10, + Age11, + Age12, + Age13, + Age14, + Age15, + Age16, + Age17, + Age18, + Age19, + Age20, + Age21, + Age22, + Age23, + Age24, + Age25, + Age26, + Age27, + Age28, + Age29, + Age30, + Age31, + Age32, + Age33, + Age34, + Age35, + Age36, + Age37, + Age38, + Age39, + Age40, + Age41, + Age42, + Age43, + Age44, + Age45, + Age46, + Age47, + Age48, + Age49, + Age50, + Age51, + Age52, + Age53, + Age54, + Age55, + Age56, + Age57, + Age58, + Age59, + Age60, + Age61, + Age62, + Age63, + Age64, + Age65, + Age66, + Age67, + Age68, + Age69, + Age70, + Age71, + Age72, + Age73, + Age74, + Age75, + Age76, + Age77, + Age78, + Age79, + Age80, + Age81, + Age82, + Age83, + Age84, + Log_GDP_per_capita, + AgeSquared, + AgeCubed, Age20to24, Age25to29, Age30to34, @@ -3179,6 +3266,94 @@ public double getDoubleValue(Enum variableID) { case Dag_sq_L1 -> { return (double) (demAge - 1) * (demAge - 1); } + case Age0 -> {return (demAge==0) ? 1: 0;} + case Age1 -> {return (demAge==1) ? 1: 0;} + case Age2 -> {return (demAge==2) ? 1: 0;} + case Age3 -> {return (demAge==3) ? 1: 0;} + case Age4 -> {return (demAge==4) ? 1: 0;} + case Age5 -> {return (demAge==5) ? 1: 0;} + case Age6 -> {return (demAge==6) ? 1: 0;} + case Age7 -> {return (demAge==7) ? 1: 0;} + case Age8 -> {return (demAge==8) ? 1: 0;} + case Age9 -> {return (demAge==9) ? 1: 0;} + case Age10 -> {return (demAge==10) ? 1: 0;} + case Age11 -> {return (demAge==11) ? 1: 0;} + case Age12 -> {return (demAge==12) ? 1: 0;} + case Age13 -> {return (demAge==13) ? 1: 0;} + case Age14 -> {return (demAge==14) ? 1: 0;} + case Age15 -> {return (demAge==15) ? 1: 0;} + case Age16 -> {return (demAge==16) ? 1: 0;} + case Age17 -> {return (demAge==17) ? 1: 0;} + case Age18 -> {return (demAge==18) ? 1: 0;} + case Age19 -> {return (demAge==19) ? 1: 0;} + case Age20 -> {return (demAge==20) ? 1: 0;} + case Age21 -> {return (demAge==21) ? 1: 0;} + case Age22 -> {return (demAge==22) ? 1: 0;} + case Age23 -> {return (demAge==23) ? 1: 0;} + case Age24 -> {return (demAge==24) ? 1: 0;} + case Age25 -> {return (demAge==25) ? 1: 0;} + case Age26 -> {return (demAge==26) ? 1: 0;} + case Age27 -> {return (demAge==27) ? 1: 0;} + case Age28 -> {return (demAge==28) ? 1: 0;} + case Age29 -> {return (demAge==29) ? 1: 0;} + case Age30 -> {return (demAge==30) ? 1: 0;} + case Age31 -> {return (demAge==31) ? 1: 0;} + case Age32 -> {return (demAge==32) ? 1: 0;} + case Age33 -> {return (demAge==33) ? 1: 0;} + case Age34 -> {return (demAge==34) ? 1: 0;} + case Age35 -> {return (demAge==35) ? 1: 0;} + case Age36 -> {return (demAge==36) ? 1: 0;} + case Age37 -> {return (demAge==37) ? 1: 0;} + case Age38 -> {return (demAge==38) ? 1: 0;} + case Age39 -> {return (demAge==39) ? 1: 0;} + case Age40 -> {return (demAge==40) ? 1: 0;} + case Age41 -> {return (demAge==41) ? 1: 0;} + case Age42 -> {return (demAge==42) ? 1: 0;} + case Age43 -> {return (demAge==43) ? 1: 0;} + case Age44 -> {return (demAge==44) ? 1: 0;} + case Age45 -> {return (demAge==45) ? 1: 0;} + case Age46 -> {return (demAge==46) ? 1: 0;} + case Age47 -> {return (demAge==47) ? 1: 0;} + case Age48 -> {return (demAge==48) ? 1: 0;} + case Age49 -> {return (demAge==49) ? 1: 0;} + case Age50 -> {return (demAge==50) ? 1: 0;} + case Age51 -> {return (demAge==51) ? 1: 0;} + case Age52 -> {return (demAge==52) ? 1: 0;} + case Age53 -> {return (demAge==53) ? 1: 0;} + case Age54 -> {return (demAge==54) ? 1: 0;} + case Age55 -> {return (demAge==55) ? 1: 0;} + case Age56 -> {return (demAge==56) ? 1: 0;} + case Age57 -> {return (demAge==57) ? 1: 0;} + case Age58 -> {return (demAge==58) ? 1: 0;} + case Age59 -> {return (demAge==59) ? 1: 0;} + case Age60 -> {return (demAge==60) ? 1: 0;} + case Age61 -> {return (demAge==61) ? 1: 0;} + case Age62 -> {return (demAge==62) ? 1: 0;} + case Age63 -> {return (demAge==63) ? 1: 0;} + case Age64 -> {return (demAge==64) ? 1: 0;} + case Age65 -> {return (demAge==65) ? 1: 0;} + case Age66 -> {return (demAge==66) ? 1: 0;} + case Age67 -> {return (demAge==67) ? 1: 0;} + case Age68 -> {return (demAge==68) ? 1: 0;} + case Age69 -> {return (demAge==69) ? 1: 0;} + case Age70 -> {return (demAge==70) ? 1: 0;} + case Age71 -> {return (demAge==71) ? 1: 0;} + case Age72 -> {return (demAge==72) ? 1: 0;} + case Age73 -> {return (demAge==73) ? 1: 0;} + case Age74 -> {return (demAge==74) ? 1: 0;} + case Age75 -> {return (demAge==75) ? 1: 0;} + case Age76 -> {return (demAge==76) ? 1: 0;} + case Age77 -> {return (demAge==77) ? 1: 0;} + case Age78 -> {return (demAge==78) ? 1: 0;} + case Age79 -> {return (demAge==79) ? 1: 0;} + case Age80 -> {return (demAge==80) ? 1: 0;} + case Age81 -> {return (demAge==81) ? 1: 0;} + case Age82 -> {return (demAge==82) ? 1: 0;} + case Age83 -> {return (demAge==83) ? 1: 0;} + case Age84 -> {return (demAge==84) ? 1: 0;} + case Log_GDP_per_capita -> { + return Math.log(Parameters.getTimeSeriesValue(getYear(), TimeSeriesVariable.GDPperCapita)); + } case Age20to24 -> { return (demAge >= 20 && demAge <= 24) ? 1. : 0.; } @@ -5082,6 +5257,70 @@ public void populateSocialCareReceipt_lag1(SocialCareReceiptState state) { } } + public void matchLifetimeIncome(int age, boolean agePlus, final double[] sortedZ, List templateIndividuals) { + + boolean matchHere; + if (agePlus) + matchHere = (demAge >= age); + else + matchHere = (demAge == age); + if (matchHere) { + + // identify normalised income target + double eqIncomePerYear = benefitUnit.getHousehold().getEquivalisedDisposableIncomeYearly(); + if (eqIncomePerYear > 0.0) { + + double gmEqDispIncomePerYear = getGMEquivalisedIncome(demAge, getYear()); + double zTarget = Math.log(eqIncomePerYear / gmEqDispIncomePerYear); + int ii = Arrays.binarySearch(sortedZ, zTarget); + if (ii >= 0) { + // exact match found + setLtIncomeDonor(templateIndividuals.get(ii)); + } else { + // adopt nearest neighbour + int ins = -ii - 1; + if (ins == 0) { + // lower bound + setLtIncomeDonor(templateIndividuals.get(ins)); + } else if (ins == sortedZ.length) { + // upper bound + setLtIncomeDonor(templateIndividuals.get(ins - 1)); + } else if (zTarget - sortedZ[ins -1] < sortedZ[ins] - zTarget) { + // choose lower bound + setLtIncomeDonor(templateIndividuals.get(ins - 1)); + } else { + // choose upper bound + setLtIncomeDonor(templateIndividuals.get(ins)); + } + } + } + else { + + setLtIncomeDonor(templateIndividuals.get(0)); + } + + // set lifetime income + setLtIncome(age); + } + } + + private double getGMEquivalisedIncome(int age, int year) { + + Double val = Parameters.getEquivalisedIncome(demMaleFlag, age, year); + if (!Parameters.isFinite(val)) { + // outside observed range, use regression model + + RegressionProxy proxy = new RegressionProxy(age, year); + if (demMaleFlag.toBoolean()) { + val = Math.exp(Parameters.getRegLifetimeIncome1a().getScore(proxy, RegressionProxy.Variables.class)); + } + else { + val = Math.exp(Parameters.getRegLifetimeIncome1b().getScore(proxy, RegressionProxy.Variables.class)); + } + } + return val; + } + public void setCareFromInformalFlag(boolean val) { careFromInformalFlag = val; } @@ -6113,24 +6352,28 @@ public Double getHealthPhysicalPcsL1() { return healthPhysicalPcsL1; } - public void setLtIncomeDonor(Individual individual) { - ltIncomeDonor = individual; + public void setLtIncomeDonor(TemplateIndividual donor) { + ltIncomeDonor = donor; + } + + public double getImputedIncome(int age) { + + int year = age + getYear() - demAge; + double gmIncome = getGMEquivalisedIncome(age, year); + double annualIncome = Math.exp(ltIncomeDonor.getNormIncome(age).getZValue()) * gmIncome; + return annualIncome; } public void setLtIncome(int maxAge) { yLifeTime = 0.0; - if (demAge > 0) { + for (int age=0; age<=maxAge; age++) { - int birthYear = ltIncomeDonor.getBirthYear(); - int ageLimit = Math.min(maxAge, demAge -1); - for (int aa=0; aa<=ageLimit; aa++) { - AnnualIncome annualIncome = ltIncomeDonor.getAnnualIncome(birthYear+aa); - if (annualIncome == null) - throw new RuntimeException("Annual income for year " + (birthYear+aa) + " not found for donor " + ltIncomeDonor.getId()); - yLifeTime += annualIncome.getValue(); - } - yLifeTime /= (double)(ageLimit+1); + int year = age + getYear() - demAge; + double gmIncome = getGMEquivalisedIncome(age, year); + double annualIncome = Math.exp(ltIncomeDonor.getNormIncome(age).getZValue()) * gmIncome; + yLifeTime += annualIncome; } + yLifeTime /= (double)(maxAge+1); } public double getYLifeTime() { @@ -6154,7 +6397,7 @@ public void updateLtIncome() { } } - public Individual getLtIncomeDonor() {return ltIncomeDonor;} + public TemplateIndividual getLtIncomeDonor() {return ltIncomeDonor;} public Integer getLiwwh() { return labEmpNyear; diff --git a/src/main/java/simpaths/model/SimPathsModel.java b/src/main/java/simpaths/model/SimPathsModel.java index a618bd59b..81251166d 100644 --- a/src/main/java/simpaths/model/SimPathsModel.java +++ b/src/main/java/simpaths/model/SimPathsModel.java @@ -3,8 +3,6 @@ // import Java packages -import com.opencsv.exceptions.CsvDataTypeMismatchException; -import com.opencsv.exceptions.CsvRequiredFieldEmptyException; import jakarta.persistence.EntityManager; import jakarta.persistence.EntityManagerFactory; import jakarta.persistence.EntityTransaction; @@ -39,9 +37,8 @@ import simpaths.model.decisions.DecisionParams; import simpaths.model.decisions.ManagerPopulateGrids; import simpaths.model.enums.*; -import simpaths.model.lifetime_incomes.BirthCohort; -import simpaths.model.lifetime_incomes.LifetimeIncomeImputation; -import simpaths.model.lifetime_incomes.ManagerProjectLifetimeIncomes; +import simpaths.model.lifetime_incomes.ManagerLifetimeIncomes; +import simpaths.model.lifetime_incomes.TemplateIndividual; import simpaths.model.taxes.*; import simpaths.model.taxes.database.DatabaseExtension; import simpaths.model.taxes.database.TaxDonorDataParser; @@ -283,15 +280,13 @@ private record PendingParentLink(Person child, long sourceChildId, Long sourceMo @GUIparameter(description = "tick to enable intertemporal optimised consumption and labour decisions") private boolean enableIntertemporalOptimisations = false; - private boolean lifetimeIncomeGenerate = false; // request to generate new set of lifetime incomes - private Integer lifetimeIncomeStartBirthYear = 1934; - private Integer lifetimeIncomeEndBirthYear = 2019; - private Integer lifetimeIncomeEndAge = 80; - private Integer lifetimeIncomeCohortSize = 100000; - private boolean lifetimeIncomeWriteToCSV = false; + private boolean lifetimeIncomeImpute = false; // request to generate new set of lifetime incomes + private Integer lifetimeIncomeCapAge = 80; + private Integer lifetimeIncomeEndAge = 120; + private Integer lifetimeIncomeCohortSize = 500000; + private boolean lifetimeIncomeWriteToCSV = true; private long lifetimeIncomeRandomSeed = 505; - private double lifetimeIncomeAge0StdDev = 0.9; - private boolean lifetimeIncomeImpute = false; // request to impute lifetime income data for population at load + List lifetimeIncomeTemplates = new ArrayList<>(); @GUIparameter(description = "tick to use behavioural solutions saved by a previous simulation") private boolean useSavedBehaviour = false; @@ -404,10 +399,12 @@ public void buildObjects() { projectSocialCare, donorPoolAveraging, taxDonorUpratingByWage, fixTimeTrend, flagDefaultToTimeSeriesAverages, saveImperfectTaxDBMatches, timeTrendStopsIn, startYear, endYear, interestRateInnov, disposableIncomeFromLabourInnov, flagSuppressChildcareCosts, flagSuppressSocialCareCosts, lifetimeIncomeImpute); - if (lifetimeIncomeGenerate) { - ManagerProjectLifetimeIncomes.run(log, lifetimeIncomeStartBirthYear, - lifetimeIncomeEndBirthYear, lifetimeIncomeEndAge, lifetimeIncomeCohortSize, lifetimeIncomeWriteToCSV, - lifetimeIncomeRandomSeed, lifetimeIncomeAge0StdDev, outputPath); + if (lifetimeIncomeImpute) { + ManagerLifetimeIncomes.generateNormalisedProfiles(log, lifetimeIncomeCapAge, lifetimeIncomeEndAge, lifetimeIncomeCohortSize, lifetimeIncomeTemplates, + lifetimeIncomeWriteToCSV, lifetimeIncomeRandomSeed, outputPath); + boolean projectSample = true; + if (projectSample) + ManagerLifetimeIncomes.imputeProfiles2(log, lifetimeIncomeCapAge, lifetimeIncomeRandomSeed, lifetimeIncomeTemplates, lifetimeIncomeWriteToCSV, outputPath); } if (enableIntertemporalOptimisations) { @@ -2790,7 +2787,6 @@ private void createInitialPopulationDataStructures() { //TODO: Slight differences between otherwise identical simulations arise when loading "processed" vs "unprocessed" data (distinguished by the if statement below) Processed processed = PersistPopulation ? getProcessed() : null; - LifetimeIncomeImputation lifetimeIncomes = lifetimeIncomeImpute ? getLifetimeIncomes(startYear) : null; if (processed!=null) { List households = new ArrayList<>(processed.getHouseholds()); @@ -2800,10 +2796,6 @@ private void createInitialPopulationDataStructures() { inheritanceModule.restorePopulationLineage(processed.getInheritanceLineage(), processed.getPersons()); - // check if need to add income histories - if (lifetimeIncomes!=null) - lifetimeIncomes.matchDonorProfiles(households); - // update counters and augment attributes long householdIdCounter = 1L, benefitUnitIdCounter = 1L, personIdCounter = 1L; for ( Household originalHousehold : households) { @@ -2840,9 +2832,6 @@ private void createInitialPopulationDataStructures() { System.out.println("completed loading survey data for starting population"); log.info("completed loading survey data for starting population"); - // check if need to add income histories - if (lifetimeIncomes!=null) - lifetimeIncomes.matchDonorProfiles(inputHouseholdList); if (!useWeights) { // Expand population, sample, and remove weights @@ -2958,6 +2947,12 @@ else if (flagSearch1) { if (processed == null) resolveUnambiguousCrossHouseholdParents(); + if (lifetimeIncomeImpute) { + + String outputPath = "./output/" + getEngine().getCurrentExperiment().runId; + ManagerLifetimeIncomes.imputeProfiles(log, persons, lifetimeIncomeTemplates, lifetimeIncomeWriteToCSV, outputPath); + } + if (Parameters.projectNonPensionWealth) { // HW2c residual reconstruction uses the same common income ranks // that are assigned immediately before housing projection later. @@ -3777,42 +3772,6 @@ public void clearPersonsToMatch() { } } - private LifetimeIncomeImputation getLifetimeIncomes(int year) { - - System.out.println("Loading simulated income histories"); - LifetimeIncomeImputation lifetimeIncomes = null; - EntityTransaction txn = null; - try { - - // query database - String fileName = Parameters.getInputDirectory() + "input"; - var propertyMap = new HashMap(); - propertyMap.put("hibernate.connection.url", "jdbc:h2:file:" + fileName + ";TRACE_LEVEL_FILE=0;TRACE_LEVEL_SYSTEM_OUT=0;AUTO_SERVER=TRUE"); - EntityManager em = Persistence.createEntityManagerFactory("lifetime-incomes", propertyMap).createEntityManager(); - txn = em.getTransaction(); - txn.begin(); - String query = "SELECT DISTINCT cohort FROM BirthCohort cohort"; -// String query = "SELECT DISTINCT cohort FROM BirthCohort cohort LEFT JOIN FETCH cohort.individuals individuals LEFT JOIN FETCH individuals.incomes incomes"; - System.out.println("Submitting SQL query"); - log.info("Submitting SQL query: " + query); - List cohorts = em.createQuery(query).getResultList(); - lifetimeIncomes = new LifetimeIncomeImputation(year, cohorts); - - // close database connection - log.info("Query complete"); - System.out.println("Query complete"); - em.close(); - } catch (Exception e) { - if (txn != null) { - txn.rollback(); - } - e.printStackTrace(); - throw new RuntimeException("Problem sourcing data for starting population"); - } - - return lifetimeIncomes; - } - private Processed getProcessed() { return getProcessed(country, startYear, popSize, ignoreTargetsAtPopulationLoad); } diff --git a/src/main/java/simpaths/model/enums/Gender.java b/src/main/java/simpaths/model/enums/Gender.java index 4d7383d5d..bd04e1351 100644 --- a/src/main/java/simpaths/model/enums/Gender.java +++ b/src/main/java/simpaths/model/enums/Gender.java @@ -1,7 +1,12 @@ package simpaths.model.enums; public enum Gender { - Male, - Female, - + Male(true), + Female(false); + + private final boolean value; + Gender(boolean value) { this.value = value; } + public boolean toBoolean() { return value; } + + public static Gender of(boolean b) { return b ? Male : Female; } } diff --git a/src/main/java/simpaths/model/lifetime_incomes/AdjustmentFactors.java b/src/main/java/simpaths/model/lifetime_incomes/AdjustmentFactors.java index 9fcb48fca..e930449f8 100644 --- a/src/main/java/simpaths/model/lifetime_incomes/AdjustmentFactors.java +++ b/src/main/java/simpaths/model/lifetime_incomes/AdjustmentFactors.java @@ -4,91 +4,266 @@ public class AdjustmentFactors { public static double normIncomeZero = 1.0; + /* + public static double fixedEffect = 0.0; + public static double[] whiteNoise = { + 1 , // age0 + 0.87148293 , // age1 + 0.764571713 , // age2 + 0.772685847 , // age3 + 0.784034322 , // age4 + 0.788145336 , // age5 + 0.840927347 , // age6 + 0.804404853 , // age7 + 0.746862871 , // age8 + 0.805857937 , // age9 + 0.768046173 , // age10 + 0.768877138 , // age11 + 0.787305004 , // age12 + 0.807517712 , // age13 + 0.779309551 , // age14 + 0.797063961 , // age15 + 0.774348026 , // age16 + 0.765564658 , // age17 + 0.663181832 , // age18 + 0.59206669 , // age19 + 0.586870346 , // age20 + 0.611374431 , // age21 + 0.631364727 , // age22 + 0.624448431 , // age23 + 0.689266129 , // age24 + 0.697007655 , // age25 + 0.681058745 , // age26 + 0.716029031 , // age27 + 0.758303204 , // age28 + 0.755814235 , // age29 + 0.729608534 , // age30 + 0.731067955 , // age31 + 0.761946787 , // age32 + 0.75419121 , // age33 + 0.826190512 , // age34 + 0.809296211 , // age35 + 0.781894238 , // age36 + 0.785505776 , // age37 + 0.761463929 , // age38 + 0.760232732 , // age39 + 0.738854134 , // age40 + 0.778571519 , // age41 + 0.787172986 , // age42 + 0.768135586 , // age43 + 0.778594224 , // age44 + 0.747630779 , // age45 + 0.767889635 , // age46 + 0.762757537 , // age47 + 0.743691402 , // age48 + 0.740342881 , // age49 + 0.721592208 , // age50 + 0.759871531 , // age51 + 0.763245192 , // age52 + 0.747005532 , // age53 + 0.713803749 , // age54 + 0.679820227 , // age55 + 0.698457799 , // age56 + 0.700661299 , // age57 + 0.731558535 , // age58 + 0.727728572 , // age59 + 0.657778974 , // age60 + 0.616675498 , // age61 + 0.656335435 , // age62 + 0.687579782 , // age63 + 0.630234724 , // age64 + 0.593370842 , // age65 + 0.62863954 , // age66 + 0.656988208 , // age67 + 0.672553106 , // age68 + 0.775680501 , // age69 + 0.763509579 , // age70 + 0.739352231 , // age71 + 0.74964186 , // age72 + 0.737836806 , // age73 + 0.726926872 , // age74 + 0.735899241 , // age75 + 0.764102071 , // age76 + 0.76885391 , // age77 + 0.78931846 , // age78 + 0.739869501 , // age79 + 0.721907094 , // age80 + }; + */ + + /* public static double fixedEffect = 0.5; + public static double[] whiteNoise = { + 1 , // age0 + 0.810600425 , // age1 + 0.765659634 , // age2 + 0.754058618 , // age3 + 0.783828906 , // age4 + 0.768063746 , // age5 + 0.796604212 , // age6 + 0.789854452 , // age7 + 0.716078297 , // age8 + 0.736137043 , // age9 + 0.753398336 , // age10 + 0.716854528 , // age11 + 0.746125865 , // age12 + 0.76583905 , // age13 + 0.761874565 , // age14 + 0.751396932 , // age15 + 0.750993983 , // age16 + 0.720905381 , // age17 + 0.640968838 , // age18 + 0.558559436 , // age19 + 0.531750813 , // age20 + 0.572330549 , // age21 + 0.612800824 , // age22 + 0.601559337 , // age23 + 0.641238663 , // age24 + 0.676612772 , // age25 + 0.651634131 , // age26 + 0.680195328 , // age27 + 0.733788701 , // age28 + 0.732064056 , // age29 + 0.706656305 , // age30 + 0.683901466 , // age31 + 0.725622353 , // age32 + 0.731489093 , // age33 + 0.762805296 , // age34 + 0.798268366 , // age35 + 0.758836387 , // age36 + 0.742153661 , // age37 + 0.719845452 , // age38 + 0.725223646 , // age39 + 0.72121148 , // age40 + 0.734940966 , // age41 + 0.76595968 , // age42 + 0.742528405 , // age43 + 0.733591473 , // age44 + 0.729856594 , // age45 + 0.721115214 , // age46 + 0.737938141 , // age47 + 0.716538806 , // age48 + 0.704445303 , // age49 + 0.703614956 , // age50 + 0.718490275 , // age51 + 0.737896031 , // age52 + 0.715696588 , // age53 + 0.691953588 , // age54 + 0.653860122 , // age55 + 0.65851408 , // age56 + 0.688278184 , // age57 + 0.69938592 , // age58 + 0.710389374 , // age59 + 0.640564246 , // age60 + 0.577854441 , // age61 + 0.601254583 , // age62 + 0.661035049 , // age63 + 0.634446418 , // age64 + 0.539196673 , // age65 + 0.529101491 , // age66 + 0.586106269 , // age67 + 0.641271089 , // age68 + 0.746262214 , // age69 + 0.780353352 , // age70 + 0.714603171 , // age71 + 0.691786014 , // age72 + 0.678865725 , // age73 + 0.670399108 , // age74 + 0.670262638 , // age75 + 0.707404633 , // age76 + 0.722143763 , // age77 + 0.719761022 , // age78 + 0.685978702 , // age79 + 0.664111464 , // age80 + }; + */ + public static double fixedEffect = 1.0; public static double[] whiteNoise = { - 0.5000 , // age0 - 0.5000 , // age1 - 0.5000 , // age2 - 0.5000 , // age3 - 0.5000 , // age4 - 0.5000 , // age5 - 0.5000 , // age6 - 0.5000 , // age7 - 0.5000 , // age8 - 0.5000 , // age9 - 0.5000 , // age10 - 0.5000 , // age11 - 0.5000 , // age12 - 0.5000 , // age13 - 0.5000 , // age14 - 0.5000 , // age15 - 0.5000 , // age16 - 0.5000 , // age17 - 0.5000 , // age18 - 0.5000 , // age19 - 0.5000 , // age20 - 0.5000 , // age21 - 0.5000 , // age22 - 0.5000 , // age23 - 0.5000 , // age24 - 0.5000 , // age25 - 0.5000 , // age26 - 0.5000 , // age27 - 0.5000 , // age28 - 0.5000 , // age29 - 0.5000 , // age30 - 0.5000 , // age31 - 0.5000 , // age32 - 0.5000 , // age33 - 0.5000 , // age34 - 0.5000 , // age35 - 0.5000 , // age36 - 0.5000 , // age37 - 0.5000 , // age38 - 0.5000 , // age39 - 0.5000 , // age40 - 0.5000 , // age41 - 0.5000 , // age42 - 0.5000 , // age43 - 0.5000 , // age44 - 0.5000 , // age45 - 0.5000 , // age46 - 0.5000 , // age47 - 0.5000 , // age48 - 0.5000 , // age49 - 0.5000 , // age50 - 0.5000 , // age51 - 0.5000 , // age52 - 0.5000 , // age53 - 0.5000 , // age54 - 0.5000 , // age55 - 0.5000 , // age56 - 0.5000 , // age57 - 0.5000 , // age58 - 0.5000 , // age59 - 0.5000 , // age60 - 0.5000 , // age61 - 0.5000 , // age62 - 0.5000 , // age63 - 0.5000 , // age64 - 0.5000 , // age65 - 0.5000 , // age66 - 0.5000 , // age67 - 0.5000 , // age68 - 0.5000 , // age69 - 0.5000 , // age70 - 0.5000 , // age71 - 0.5000 , // age72 - 0.5000 , // age73 - 0.5000 , // age74 - 0.5000 , // age75 - 0.5000 , // age76 - 0.5000 , // age77 - 0.5000 , // age78 - 0.5000 , // age79 - 0.5000 , // age80 + 1 , // age0 + 0.816551806 , // age1 + 0.671776289 , // age2 + 0.688421489 , // age3 + 0.679900794 , // age4 + 0.667012955 , // age5 + 0.722852228 , // age6 + 0.671472692 , // age7 + 0.599534122 , // age8 + 0.685351441 , // age9 + 0.613059005 , // age10 + 0.62131478 , // age11 + 0.628947678 , // age12 + 0.661954246 , // age13 + 0.621189189 , // age14 + 0.656172352 , // age15 + 0.618058971 , // age16 + 0.627020309 , // age17 + 0.526110173 , // age18 + 0.457971187 , // age19 + 0.460786097 , // age20 + 0.49127186 , // age21 + 0.501715014 , // age22 + 0.483131471 , // age23 + 0.548024396 , // age24 + 0.542679997 , // age25 + 0.535023471 , // age26 + 0.58000527 , // age27 + 0.617527725 , // age28 + 0.616615978 , // age29 + 0.584899953 , // age30 + 0.599483402 , // age31 + 0.629539457 , // age32 + 0.594541677 , // age33 + 0.693864587 , // age34 + 0.667192975 , // age35 + 0.640435411 , // age36 + 0.6357391 , // age37 + 0.62033899 , // age38 + 0.630532822 , // age39 + 0.599621547 , // age40 + 0.651843724 , // age41 + 0.651229367 , // age42 + 0.621933044 , // age43 + 0.652963517 , // age44 + 0.603598254 , // age45 + 0.639341694 , // age46 + 0.628733613 , // age47 + 0.610794804 , // age48 + 0.625973302 , // age49 + 0.596545879 , // age50 + 0.640012106 , // age51 + 0.63102302 , // age52 + 0.629489893 , // age53 + 0.595867162 , // age54 + 0.56465231 , // age55 + 0.599367946 , // age56 + 0.591013298 , // age57 + 0.635381115 , // age58 + 0.620930877 , // age59 + 0.540859636 , // age60 + 0.494865152 , // age61 + 0.546741693 , // age62 + 0.580068222 , // age63 + 0.484895284 , // age64 + 0.423128383 , // age65 + 0.43296438 , // age66 + 0.446951354 , // age67 + 0.45968271 , // age68 + 0.553448357 , // age69 + 0.525122451 , // age70 + 0.497218134 , // age71 + 0.498566639 , // age72 + 0.491156781 , // age73 + 0.483444267 , // age74 + 0.499260188 , // age75 + 0.522373778 , // age76 + 0.515497121 , // age77 + 0.53025143 , // age78 + 0.493819264 , // age79 + 0.493819264 , // age80 }; + /* + */ public static double getWhiteNoise(int age) { return whiteNoise[age]; diff --git a/src/main/java/simpaths/model/lifetime_incomes/AnnualIncome.java b/src/main/java/simpaths/model/lifetime_incomes/AnnualIncome.java deleted file mode 100644 index 970c1e240..000000000 --- a/src/main/java/simpaths/model/lifetime_incomes/AnnualIncome.java +++ /dev/null @@ -1,144 +0,0 @@ -package simpaths.model.lifetime_incomes; - -import jakarta.persistence.*; -import microsim.statistics.IDoubleSource; -import simpaths.data.Parameters; -import simpaths.model.enums.Gender; - -@Entity -public class AnnualIncome implements IDoubleSource { - - @Id @GeneratedValue(strategy = GenerationType.IDENTITY) @Column(name = "id", unique = true, nullable = false) private Long id; - @ManyToOne(fetch = FetchType.LAZY, cascade=CascadeType.REFRESH) - @JoinColumns({ - @JoinColumn(name = "individual_id", referencedColumnName = "id") - }) - private Individual individual; - - @Column(name="calendar_year") private int year; - @Column(name="income_value") private Double value; - @Transient private Double z_m1 = null; - @Transient private Double z_m2 = null; - - @Transient private Double rnd = null; - @Transient private Double innov = null; - @Transient private Double gmIncome = null; - @Transient private Double gmIncome_m1 = null; - - - /** - * CONSTRUCTOR - */ - public AnnualIncome() {} - public AnnualIncome(int year, Individual individual, double rnd, double age0StdDev) { - this.individual = individual; - this.year = year; - this.rnd = rnd; - value = incomeEvaluation(rnd, age0StdDev); - - individual.addAnnualIncome(this); - } - - public double getValue() { - if (Parameters.isFinite(value)) { - return value; - } - else { - throw new RuntimeException("AnnualIncome.getValue() called before value has been set"); - } - } - public int getYear() {return year;} - public Individual getIndividual() {return individual;} - public void setIndividual(Individual individual) { - this.individual = individual; - } - - public double incomeEvaluation(double rnd, double age0StdDev) { - - Double val = null; - int age = individual.getAge(year); - if (age==0) { - // random draw of new income - - double gmIncome = getGMEquivalisedIncome(year); - this.gmIncome = gmIncome; - double gauss = Parameters.getStandardNormalDistribution().inverseCumulativeProbability(rnd); - innov = gauss; - val = Math.exp(age0StdDev*gauss) * gmIncome; - } - else if (age==1) { - // AR1 process - - double gmIncome = getGMEquivalisedIncome(year); - this.gmIncome = gmIncome; - - double gmIncome_m1 = getGMEquivalisedIncome(year-1); - this.gmIncome_m1 = gmIncome_m1; - double income_m1 = individual.getAnnualIncome(year-1).getValue(); - - z_m1 = Math.log(income_m1 / gmIncome_m1); - - double z_score = Parameters.getRegLifetimeIncome2b().getScore(this, AnnualIncome.DoublesVariables.class); - innov = 0.0; - val = Math.exp(z_score+innov) * gmIncome; - } - else { - // AR2 process - - double gmIncome = getGMEquivalisedIncome(year); - this.gmIncome = gmIncome; - - double gmIncome_m1 = getGMEquivalisedIncome(year-1); - this.gmIncome_m1 = gmIncome_m1; - double income_m1 = individual.getAnnualIncome(year-1).getValue(); - - double gmIncome_m2 = getGMEquivalisedIncome( year-2); - double income_m2 = individual.getAnnualIncome(year-2).getValue(); - - z_m1 = Math.log(income_m1 / gmIncome_m1); - z_m2 = Math.log(income_m2 / gmIncome_m2); - - double z_score = Parameters.getRegLifetimeIncome2a().getScore(this, AnnualIncome.DoublesVariables.class); - innov = 0.0; - val = Math.exp(z_score+innov) * gmIncome; - } - return val; - } - - private double getGMEquivalisedIncome(int year) { - - individual.setYear(year); - int age = individual.getAge(); - Gender gender = individual.getGender(); - Double val = Parameters.getEquivalisedIncome(gender, age, year); - if (!Parameters.isFinite(val)) { - // outside observed range, use regression model - - if (Gender.Male.equals(gender)) { - val = Math.exp(Parameters.getRegLifetimeIncome1a().getScore(individual, Individual.DoublesVariables.class)); - } - else { - val = Math.exp(Parameters.getRegLifetimeIncome1b().getScore(individual, Individual.DoublesVariables.class)); - } - } - return val; - } - - public enum DoublesVariables { - z_lag1, - z_lag2 - } - public double getDoubleValue(Enum variableID) { - - switch ((AnnualIncome.DoublesVariables) variableID) { - case z_lag1 -> {return z_m1;} - case z_lag2 -> {return z_m2;} - default -> { - throw new RuntimeException("request for unrecognised variable: individual"); - } - } - } - public String toString() { - return id.toString(); - } -} diff --git a/src/main/java/simpaths/model/lifetime_incomes/BirthCohort.java b/src/main/java/simpaths/model/lifetime_incomes/BirthCohort.java deleted file mode 100644 index 77b407b95..000000000 --- a/src/main/java/simpaths/model/lifetime_incomes/BirthCohort.java +++ /dev/null @@ -1,59 +0,0 @@ -package simpaths.model.lifetime_incomes; - -import jakarta.persistence.*; - -import java.util.*; - -import simpaths.model.enums.Gender; - -@Entity -public class BirthCohort { - - - /** - * ATTRIBUTES - */ - @Id @GeneratedValue(strategy = GenerationType.IDENTITY) @Column(name = "id", unique = true, nullable = false) private Long id; - @OneToMany(fetch = FetchType.LAZY, cascade = CascadeType.ALL, mappedBy = "cohort") - @OrderBy("id ASC") - private Set individuals = new LinkedHashSet<>(); - - @Column(name="birth_year") private Integer birthYear; - @Column(name="end_age") private Integer endAge; - @Enumerated(EnumType.STRING) private Gender demSex; - - @Transient List sortedIndividuals = null; - - - /** - * CONSTRUCTOR - */ - public BirthCohort() {} - public BirthCohort(Integer birthYear, Gender demSex, Integer endAge) { - this.birthYear = birthYear; - this.demSex = demSex; - this.endAge = endAge; - } - - public Set getIndividuals() { - return individuals; - } - public void setIndividuals(Set individuals) { - this.individuals = individuals; - } - public void setSortedIndividuals(List sortedIndividuals) {this.sortedIndividuals = sortedIndividuals;} - public List getSortedIndividuals() {return sortedIndividuals;} - public int getBirthYear() { - return birthYear; - } - public Gender getGender() { - return demSex; - } - public int getEndAge() {return endAge;} - public void addIndividual(Individual individual) { - individuals.add(individual); - } - public String toString() { - return id.toString(); - } -} diff --git a/src/main/java/simpaths/model/lifetime_incomes/EquivalisedIncomeCDF.java b/src/main/java/simpaths/model/lifetime_incomes/EquivalisedIncomeCDF.java deleted file mode 100644 index 7135afd43..000000000 --- a/src/main/java/simpaths/model/lifetime_incomes/EquivalisedIncomeCDF.java +++ /dev/null @@ -1,48 +0,0 @@ -package simpaths.model.lifetime_incomes; - -import microsim.data.MultiKeyCoefficientMap; -import org.apache.commons.collections4.keyvalue.MultiKey; - -import java.lang.reflect.Array; -import java.util.*; -import java.util.stream.Collectors; - -public class EquivalisedIncomeCDF { - TreeMap cdf = new TreeMap<>(); - Object[] keys; - - public EquivalisedIncomeCDF(MultiKeyCoefficientMap data) { - - for (Object key: data.keySet()) { - - double keyValue = ((Number) ((MultiKey) key).getKey(0)).doubleValue(); - double valHere = ((Number) data.getValue(key)).doubleValue(); - cdf.put(keyValue,valHere); - } - keys = cdf.keySet().toArray(); - } - - public double getValue(double rnd) { - - // find bracket keys - int lwr = 0; - int upr = keys.length-1; - while (lwr < upr-1) { - int tstIndex = (upr + lwr) / 2; - double tstValue = cdf.get(keys[tstIndex]); - if (tstValue < rnd) { - lwr = tstIndex; - } else { - upr = tstIndex; - } - } - - // interpolate value and return - double lwrValue = cdf.get(keys[lwr]); - double uprValue = cdf.get(keys[upr]); - double ratio = (rnd - lwrValue)/(uprValue - lwrValue); - double lwrKey = (Double)keys[lwr]; - double uprKey = (Double)keys[upr]; - return (lwrKey + ratio*(uprKey - lwrKey)); - } -} diff --git a/src/main/java/simpaths/model/lifetime_incomes/ExportCSV.java b/src/main/java/simpaths/model/lifetime_incomes/ExportCSV.java deleted file mode 100644 index 5edb40bf0..000000000 --- a/src/main/java/simpaths/model/lifetime_incomes/ExportCSV.java +++ /dev/null @@ -1,119 +0,0 @@ -package simpaths.model.lifetime_incomes; - -import jakarta.persistence.Transient; -import microsim.data.db.PanelEntityKey; -import microsim.engine.SimulationEngine; -import simpaths.data.Parameters; - -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileWriter; -import java.io.IOException; -import java.lang.reflect.Field; -import java.util.*; - -public class ExportCSV { - - //Fields for exporting tables to output .csv files - final static String newLine = "\n"; - final static String delimiter = ","; - final static String directory = Parameters.getInputDirectory() + "lifetime_incomes"; - - private Set fieldsForExport; - private BufferedWriter bufferWriter; - private String idFieldName; - - private Collection targetCollection; - - public ExportCSV(int birthYear, Object target) { - try { - - targetCollection = (Collection) target; - Object obj = targetCollection.iterator().next(); - - //Set up file and fileWriter - create new file and directory if required - String filename = obj.getClass().getSimpleName() + birthYear; - File ff = new File(directory + File.separator + filename + ".csv"); - boolean fAlreadyExists = ff.exists(); - if (!fAlreadyExists) { - File dir = new File(directory); - dir.mkdirs(); - ff.createNewFile(); - } - else { - ff.delete(); - ff.createNewFile(); - } - bufferWriter = new BufferedWriter(new FileWriter(ff, true)); - - //Create alphabetically sorted list of fields including private and inherited fields that belong to the target class. - List declaredFields = new ArrayList(); - List allFields = getAllFields(declaredFields, obj.getClass()); - TreeSet fieldNames = new TreeSet(); - for (Field field : allFields) { - - if (field.getType().isPrimitive() || Number.class.isAssignableFrom(field.getType()) || - field.getType().equals(String.class)|| field.getType().equals(Boolean.class) || - field.getType().isEnum() || field.getType().equals(Character.class) || - field.getType().equals(BirthCohort.class) || field.getType().equals(Individual.class) || - field.getType().equals(AnnualIncome.class) - ) { - - String name = field.getName(); - fieldNames.add(name); - } - } - fieldsForExport = new LinkedHashSet(); - for(String fieldName : fieldNames) { //Iterated in correct order - fieldsForExport.add(fieldName); - bufferWriter.append(fieldName + delimiter); - } - - for (Object oo : targetCollection) { - - bufferWriter.append(newLine); - for (String fieldName : fieldsForExport) { - Field thisField = findUnderlyingField(oo.getClass(), fieldName); - thisField.setAccessible(true); - Object value = thisField.get(oo); - if (value == null) { - bufferWriter.append("null"); - } - else { - bufferWriter.append(value.toString()); - } - bufferWriter.append(delimiter); - } - } - bufferWriter.flush(); - bufferWriter.close(); - } catch (IOException e) { - e.printStackTrace(); - } catch (IllegalArgumentException | IllegalAccessException e) { - e.printStackTrace(); - } catch (Throwable e) { - e.printStackTrace(); - throw e; - } - } - - - private static List getAllFields(List fields, Class type) { - fields.addAll(Arrays.asList(type.getDeclaredFields())); - if (type.getSuperclass() != null) { - fields = getAllFields(fields, type.getSuperclass()); - } - return fields; - } - - - private static Field findUnderlyingField(Class clazz, String fieldName) { - Class current = clazz; - do { - try { - return current.getDeclaredField(fieldName); - } catch(Exception e) {} - } while((current = current.getSuperclass()) != null); - return null; - } -} diff --git a/src/main/java/simpaths/model/lifetime_incomes/ImputedIncome.java b/src/main/java/simpaths/model/lifetime_incomes/ImputedIncome.java new file mode 100644 index 000000000..a3e5912c9 --- /dev/null +++ b/src/main/java/simpaths/model/lifetime_incomes/ImputedIncome.java @@ -0,0 +1,97 @@ +package simpaths.model.lifetime_incomes; + +import simpaths.model.Person; + +public class ImputedIncome { + + private long id; + private int age; + private int year; + private int female; + private int ageAtDeath; + private double income; + private Double observedIncome; + + public ImputedIncome() {} + public ImputedIncome(Person person, int age) { + this.id = person.getKey().getId(); + this.age = age; + this.year = person.getYear() - person.getDemAge() + age; + this.female = person.getDemMaleFlag().toBoolean() ? 0 : 1; + this.income = person.getImputedIncome(age); + if (person.getDemAge() == age) { + + this.observedIncome = person.getBenefitUnit().getHousehold().getEquivalisedDisposableIncomeYearly(); + } + else { + this.observedIncome = null; + } + } + public ImputedIncome(SampleIndividual individual, int age) { + this.id = individual.getId(); + this.age = age; + this.year = individual.getYear() - individual.getAge() + age; + this.female = individual.getGender().toBoolean() ? 0 : 1; + this.ageAtDeath = individual.getAgeAtDeath(); + this.income = individual.getImputedIncome(age); + if (individual.getAge() == age) { + + this.observedIncome = individual.getIncome(); + } + else { + this.observedIncome = null; + } + } + + public Double getObservedIncome() { + return observedIncome; + } + + public void setObservedIncome(Double observedIncome) { + this.observedIncome = observedIncome; + } + + public int getAge() { + return age; + } + + public void setAge(int age) { + this.age = age; + } + + public long getId() { + return id; + } + + public void setId(long id) { + this.id = id; + } + + public double getIncome() { + return income; + } + + public void setIncome(double income) { + this.income = income; + } + + public int getYear() { + return year; + } + + public void setYear(int year) { + this.year = year; + } + + public int getFemale() { + return female; + } + + public void setFemmale(int female) { + this.female = female; + } + + public int getAgeAtDeath() {return ageAtDeath;} + + public void setAgeAtDeath(int ageAtDeath) {this.ageAtDeath = ageAtDeath;} +} diff --git a/src/main/java/simpaths/model/lifetime_incomes/Individual.java b/src/main/java/simpaths/model/lifetime_incomes/Individual.java deleted file mode 100644 index b163b9c60..000000000 --- a/src/main/java/simpaths/model/lifetime_incomes/Individual.java +++ /dev/null @@ -1,268 +0,0 @@ -package simpaths.model.lifetime_incomes; - -import jakarta.persistence.*; -import microsim.event.EventListener; -import microsim.statistics.IDoubleSource; -import simpaths.data.Parameters; -import simpaths.model.Person; -import simpaths.model.enums.Gender; -import simpaths.model.enums.TimeSeriesVariable; - -import java.util.LinkedHashSet; - -import java.util.Set; - -@Entity -public class Individual implements IDoubleSource { - - - /** - * ATTRIBUTES - */ - @Id @GeneratedValue(strategy = GenerationType.IDENTITY) @Column(name = "id", unique = true, nullable = false) private Long id; - @ManyToOne(fetch = FetchType.LAZY, cascade=CascadeType.REFRESH) - @JoinColumns({ - @JoinColumn(name = "cohort_id", referencedColumnName = "id") - }) - private BirthCohort cohort; - @OneToMany(fetch = FetchType.LAZY, cascade = CascadeType.ALL, mappedBy = "individual") - @OrderBy("id ASC") - private Set incomes = new LinkedHashSet<>(); - @OneToMany(fetch = FetchType.LAZY, mappedBy = "ltIncomeDonor") - private Set persons = new LinkedHashSet<>(); - - @Transient int year; - - - /** - * CONSTRUCTOR - */ - public Individual() { - } - public Individual(BirthCohort cohort) { - this.cohort = cohort; - //cohort.addIndividual(this); - } - - public Long getId() {return id;} - public Integer getAge() { - return (cohort!=null) ? year - cohort.getBirthYear() : null; - } - public Integer getAge(int year) { - return (cohort!=null) ? year - cohort.getBirthYear() : null; - } - public Gender getGender() { - return (cohort != null) ? cohort.getGender() : null; - } - public Integer getBirthYear() { - return (cohort != null) ? cohort.getBirthYear() : null; - } - public AnnualIncome getAnnualIncome(int year) { - AnnualIncome income = null; - for (AnnualIncome inc : incomes) { - if (inc.getYear()==year) { - income = inc; - break; - } - } - return income; - } - public void setCohort(BirthCohort cohort) {this.cohort = cohort;} - public void setYear(int year) {this.year = year;} - public void addAnnualIncome(AnnualIncome income) { - incomes.add(income); - } - - public enum DoublesVariables { - - Age0, - Age1, - Age2, - Age3, - Age4, - Age5, - Age6, - Age7, - Age8, - Age9, - Age10, - Age11, - Age12, - Age13, - Age14, - Age15, - Age16, - Age17, - Age18, - Age19, - Age20, - Age21, - Age22, - Age23, - Age24, - Age25, - Age26, - Age27, - Age28, - Age29, - Age30, - Age31, - Age32, - Age33, - Age34, - Age35, - Age36, - Age37, - Age38, - Age39, - Age40, - Age41, - Age42, - Age43, - Age44, - Age45, - Age46, - Age47, - Age48, - Age49, - Age50, - Age51, - Age52, - Age53, - Age54, - Age55, - Age56, - Age57, - Age58, - Age59, - Age60, - Age61, - Age62, - Age63, - Age64, - Age65, - Age66, - Age67, - Age68, - Age69, - Age70, - Age71, - Age72, - Age73, - Age74, - Age75, - Age76, - Age77, - Age78, - Age79, - Age80, - Age81, - Age82, - Age83, - Age84, - Age80plus, - Age85plus, - Log_GDP_per_capita - } - public double getDoubleValue(Enum variableID) { - - switch ((Individual.DoublesVariables) variableID) { - case Age0 -> {return (year-getBirthYear()==0) ? 1: 0;} - case Age1 -> {return (year-getBirthYear()==1) ? 1: 0;} - case Age2 -> {return (year-getBirthYear()==2) ? 1: 0;} - case Age3 -> {return (year-getBirthYear()==3) ? 1: 0;} - case Age4 -> {return (year-getBirthYear()==4) ? 1: 0;} - case Age5 -> {return (year-getBirthYear()==5) ? 1: 0;} - case Age6 -> {return (year-getBirthYear()==6) ? 1: 0;} - case Age7 -> {return (year-getBirthYear()==7) ? 1: 0;} - case Age8 -> {return (year-getBirthYear()==8) ? 1: 0;} - case Age9 -> {return (year-getBirthYear()==9) ? 1: 0;} - case Age10 -> {return (year-getBirthYear()==10) ? 1: 0;} - case Age11 -> {return (year-getBirthYear()==11) ? 1: 0;} - case Age12 -> {return (year-getBirthYear()==12) ? 1: 0;} - case Age13 -> {return (year-getBirthYear()==13) ? 1: 0;} - case Age14 -> {return (year-getBirthYear()==14) ? 1: 0;} - case Age15 -> {return (year-getBirthYear()==15) ? 1: 0;} - case Age16 -> {return (year-getBirthYear()==16) ? 1: 0;} - case Age17 -> {return (year-getBirthYear()==17) ? 1: 0;} - case Age18 -> {return (year-getBirthYear()==18) ? 1: 0;} - case Age19 -> {return (year-getBirthYear()==19) ? 1: 0;} - case Age20 -> {return (year-getBirthYear()==20) ? 1: 0;} - case Age21 -> {return (year-getBirthYear()==21) ? 1: 0;} - case Age22 -> {return (year-getBirthYear()==22) ? 1: 0;} - case Age23 -> {return (year-getBirthYear()==23) ? 1: 0;} - case Age24 -> {return (year-getBirthYear()==24) ? 1: 0;} - case Age25 -> {return (year-getBirthYear()==25) ? 1: 0;} - case Age26 -> {return (year-getBirthYear()==26) ? 1: 0;} - case Age27 -> {return (year-getBirthYear()==27) ? 1: 0;} - case Age28 -> {return (year-getBirthYear()==28) ? 1: 0;} - case Age29 -> {return (year-getBirthYear()==29) ? 1: 0;} - case Age30 -> {return (year-getBirthYear()==30) ? 1: 0;} - case Age31 -> {return (year-getBirthYear()==31) ? 1: 0;} - case Age32 -> {return (year-getBirthYear()==32) ? 1: 0;} - case Age33 -> {return (year-getBirthYear()==33) ? 1: 0;} - case Age34 -> {return (year-getBirthYear()==34) ? 1: 0;} - case Age35 -> {return (year-getBirthYear()==35) ? 1: 0;} - case Age36 -> {return (year-getBirthYear()==36) ? 1: 0;} - case Age37 -> {return (year-getBirthYear()==37) ? 1: 0;} - case Age38 -> {return (year-getBirthYear()==38) ? 1: 0;} - case Age39 -> {return (year-getBirthYear()==39) ? 1: 0;} - case Age40 -> {return (year-getBirthYear()==40) ? 1: 0;} - case Age41 -> {return (year-getBirthYear()==41) ? 1: 0;} - case Age42 -> {return (year-getBirthYear()==42) ? 1: 0;} - case Age43 -> {return (year-getBirthYear()==43) ? 1: 0;} - case Age44 -> {return (year-getBirthYear()==44) ? 1: 0;} - case Age45 -> {return (year-getBirthYear()==45) ? 1: 0;} - case Age46 -> {return (year-getBirthYear()==46) ? 1: 0;} - case Age47 -> {return (year-getBirthYear()==47) ? 1: 0;} - case Age48 -> {return (year-getBirthYear()==48) ? 1: 0;} - case Age49 -> {return (year-getBirthYear()==49) ? 1: 0;} - case Age50 -> {return (year-getBirthYear()==50) ? 1: 0;} - case Age51 -> {return (year-getBirthYear()==51) ? 1: 0;} - case Age52 -> {return (year-getBirthYear()==52) ? 1: 0;} - case Age53 -> {return (year-getBirthYear()==53) ? 1: 0;} - case Age54 -> {return (year-getBirthYear()==54) ? 1: 0;} - case Age55 -> {return (year-getBirthYear()==55) ? 1: 0;} - case Age56 -> {return (year-getBirthYear()==56) ? 1: 0;} - case Age57 -> {return (year-getBirthYear()==57) ? 1: 0;} - case Age58 -> {return (year-getBirthYear()==58) ? 1: 0;} - case Age59 -> {return (year-getBirthYear()==59) ? 1: 0;} - case Age60 -> {return (year-getBirthYear()==60) ? 1: 0;} - case Age61 -> {return (year-getBirthYear()==61) ? 1: 0;} - case Age62 -> {return (year-getBirthYear()==62) ? 1: 0;} - case Age63 -> {return (year-getBirthYear()==63) ? 1: 0;} - case Age64 -> {return (year-getBirthYear()==64) ? 1: 0;} - case Age65 -> {return (year-getBirthYear()==65) ? 1: 0;} - case Age66 -> {return (year-getBirthYear()==66) ? 1: 0;} - case Age67 -> {return (year-getBirthYear()==67) ? 1: 0;} - case Age68 -> {return (year-getBirthYear()==68) ? 1: 0;} - case Age69 -> {return (year-getBirthYear()==69) ? 1: 0;} - case Age70 -> {return (year-getBirthYear()==70) ? 1: 0;} - case Age71 -> {return (year-getBirthYear()==71) ? 1: 0;} - case Age72 -> {return (year-getBirthYear()==72) ? 1: 0;} - case Age73 -> {return (year-getBirthYear()==73) ? 1: 0;} - case Age74 -> {return (year-getBirthYear()==74) ? 1: 0;} - case Age75 -> {return (year-getBirthYear()==75) ? 1: 0;} - case Age76 -> {return (year-getBirthYear()==76) ? 1: 0;} - case Age77 -> {return (year-getBirthYear()==77) ? 1: 0;} - case Age78 -> {return (year-getBirthYear()==78) ? 1: 0;} - case Age79 -> {return (year-getBirthYear()==79) ? 1: 0;} - case Age80 -> {return (year-getBirthYear()==80) ? 1: 0;} - case Age81 -> {return (year-getBirthYear()==81) ? 1: 0;} - case Age82 -> {return (year-getBirthYear()==82) ? 1: 0;} - case Age83 -> {return (year-getBirthYear()==83) ? 1: 0;} - case Age84 -> {return (year-getBirthYear()==84) ? 1: 0;} - case Age80plus -> {return (year-getBirthYear()>79) ? 1: 0;} - case Age85plus -> {return (year-getBirthYear()>84) ? 1: 0;} - case Log_GDP_per_capita -> { - return Math.log(Parameters.getTimeSeriesValue(year, TimeSeriesVariable.GDPperCapita)); - } - default -> { - throw new RuntimeException("request for unrecognised variable"); - } - } - } - public String toString() { - return id.toString(); - } -} diff --git a/src/main/java/simpaths/model/lifetime_incomes/IndividualComparator.java b/src/main/java/simpaths/model/lifetime_incomes/IndividualComparator.java deleted file mode 100644 index 6ff80f3da..000000000 --- a/src/main/java/simpaths/model/lifetime_incomes/IndividualComparator.java +++ /dev/null @@ -1,18 +0,0 @@ -package simpaths.model.lifetime_incomes; - -import java.util.Comparator; - -public class IndividualComparator implements Comparator { - - int year; - - @Override - public int compare(Individual firstIndividual, Individual secondIndividual) { - return Double.compare(firstIndividual.getAnnualIncome(year).getValue(), - secondIndividual.getAnnualIncome(year).getValue()); - } - - public IndividualComparator(int year) { - this.year = year; - } -} diff --git a/src/main/java/simpaths/model/lifetime_incomes/LifetimeIncomeImputation.java b/src/main/java/simpaths/model/lifetime_incomes/LifetimeIncomeImputation.java deleted file mode 100644 index b3cc02fbe..000000000 --- a/src/main/java/simpaths/model/lifetime_incomes/LifetimeIncomeImputation.java +++ /dev/null @@ -1,170 +0,0 @@ -package simpaths.model.lifetime_incomes; - -import simpaths.model.BenefitUnit; -import simpaths.model.Household; -import simpaths.model.Person; -import simpaths.model.enums.Gender; - -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.Set; -import java.util.stream.IntStream; - -public class LifetimeIncomeImputation { - - Integer year; - Integer endAge = null; - List cohorts; - - public LifetimeIncomeImputation(int year, List cohorts) { - this.year = year; - this.cohorts = cohorts; - IndividualComparator comparator = new IndividualComparator(year); - for (BirthCohort cohort : cohorts) { - - if (endAge==null) - endAge = cohort.getEndAge(); - - if (cohort.getBirthYear() <= year && cohort.getBirthYear() + endAge >= year) { - - List individuals = new ArrayList<>(cohort.getIndividuals()); - individuals.sort(comparator); - cohort.setSortedIndividuals(individuals); - } - } - } - - public void matchDonorProfiles(List households) { - - if (ltIncomeNeeded(households)) { - // require imputation of income histories - System.out.println("Imputing income histories for simulated population"); - IntStream.range(0, households.size()).sorted().parallel().forEach(ii -> { - //for (int ii=0; ii < households.size(); ii++) { - - // consider each household in initialisation set - Household household = households.get(ii); - - // identify target equivalised household income - double disposableIncomePerAnnum = 0.0; - double equivalenceScale = 0.0; - boolean firstAdult = true; - for (BenefitUnit benefitUnit : household.getBenefitUnits()) { - disposableIncomePerAnnum += benefitUnit.getDisposableIncomeMonthly() * 12.0; - for (Person pp : benefitUnit.getMembers()) { - if (pp.getDemAge() > 13) { - if (firstAdult) { - equivalenceScale += 1.0; - firstAdult = false; - } else { - equivalenceScale += 0.5; - } - } else { - equivalenceScale += 0.3; - } - } - } - double targetIncome = disposableIncomePerAnnum / equivalenceScale; - - for (BenefitUnit benefitUnit : household.getBenefitUnits()) { - // loop through each benefit unit - - for (Person person : benefitUnit.getMembers()) { - // loop through each person - - if (person.getDemAge()==0) { - person.setLtIncome(endAge); - } - else { - - // match birth cohort (birth year and gender) - List individuals = getLtIncomeDonors(person); - if (individuals.isEmpty()) - throw new IllegalArgumentException("No individuals found for person " + person.getId()); - - // match income - int lwr = 0, upr = individuals.size() - 1; - double lwrValue = individuals.get(lwr).getAnnualIncome(year).getValue() - targetIncome; - if (lwrValue > 0.0) { - // lower bound - - person.setLtIncomeDonor(individuals.get(lwr)); - } else { - double uprValue = individuals.get(upr).getAnnualIncome(year).getValue() - targetIncome; - if (uprValue < 0.0) { - // upper bound - - person.setLtIncomeDonor(individuals.get(upr)); - } else { - // find bounded value - - while (lwr < upr - 1) { - int tstIndex = (upr + lwr) / 2; - double tstValue = individuals.get(tstIndex).getAnnualIncome(year).getValue() - targetIncome; - if (tstValue < 0.0) { - lwr = tstIndex; - lwrValue = tstValue; - } else { - upr = tstIndex; - uprValue = tstValue; - } - } - if (uprValue < -lwrValue) { - person.setLtIncomeDonor(individuals.get(upr)); - } else { - person.setLtIncomeDonor(individuals.get(lwr)); - } - } - } - person.setLtIncome(endAge); - } - } - } - }); - //} - System.out.println("Completed imputing income histories for simulated population"); - } - } - - private List getLtIncomeDonors(Person person) { - int birthYear = year - Math.min(person.getDemAge(),endAge); - Gender gender = person.getDemMaleFlag(); - BirthCohort targetCohort = null; - for (BirthCohort cohort : cohorts) { - - if (cohort.getBirthYear() == birthYear && cohort.getGender() == gender) { - targetCohort = cohort; - break; - } - } - if (targetCohort == null) - throw new IllegalArgumentException("No cohort found for birth year " + birthYear + " and gender " + gender); - List individuals = targetCohort.getSortedIndividuals(); - return individuals; - } - - private boolean ltIncomeNeeded(List households) { - - boolean ltIncomeNeeded = false; - if (!households.isEmpty()) { - - Iterator iteratorHousehold = households.iterator(); - Household household = iteratorHousehold.next(); - Set benefitUnits = household.getBenefitUnits(); - if (!benefitUnits.isEmpty()) { - - Iterator iteratorBenefitUnit = benefitUnits.iterator(); - BenefitUnit benefitUnit = iteratorBenefitUnit.next(); - Set persons = benefitUnit.getMembers(); - if (!persons.isEmpty()) { - - Iterator iteratorPerson = persons.iterator(); - Person person = iteratorPerson.next(); - ltIncomeNeeded = (person.getLtIncomeDonor() == null); - } - } - } - return ltIncomeNeeded; - } -} diff --git a/src/main/java/simpaths/model/lifetime_incomes/ManagerLifetimeIncomes.java b/src/main/java/simpaths/model/lifetime_incomes/ManagerLifetimeIncomes.java new file mode 100644 index 000000000..68a2de678 --- /dev/null +++ b/src/main/java/simpaths/model/lifetime_incomes/ManagerLifetimeIncomes.java @@ -0,0 +1,268 @@ +package simpaths.model.lifetime_incomes; + +import com.opencsv.bean.CsvToBeanBuilder; +import com.opencsv.bean.StatefulBeanToCsv; +import com.opencsv.bean.StatefulBeanToCsvBuilder; +import com.opencsv.exceptions.CsvDataTypeMismatchException; +import com.opencsv.exceptions.CsvRequiredFieldEmptyException; +import simpaths.data.Parameters; +import java.io.*; +import java.util.*; +import java.util.random.RandomGenerator; +import org.apache.logging.log4j.Logger; +import simpaths.model.Person; +import simpaths.model.enums.Gender; + +public class ManagerLifetimeIncomes { + + + public static void generateNormalisedProfiles(Logger log, Integer capAge, Integer endAge, Integer simCohortSize, List templateIndividuals, boolean writeToCSV, long seed, String outputDir) { + + log.info("Initialising lifetime income projections"); + System.out.println("Initialising lifetime income projections"); + + + /* + SampleIndividual ind = new SampleIndividual(); + ind.setGender(Gender.Male); + double income = ind.getGMEquivalisedIncome(80, 2043); + income = ind.getGMEquivalisedIncome(85, 2050); + */ + + + // start projecting lifetime incomes + RandomGenerator generator = new Random(seed); + + // load initialisation values + List initialisationObservations; + try { + String path = Parameters.getInputDirectory() + "lifetime_incomes" + File.separator + "initialisation_observations.csv"; + FileReader fileReader = new FileReader(path); + initialisationObservations = new CsvToBeanBuilder(fileReader).withType(InitialisationObservation.class).build().parse(); + } catch (IOException e) { + e.printStackTrace(); + throw new RuntimeException("Problem reading initialisation observations"); + } catch (Throwable e) { + e.printStackTrace(); + throw e; + } + int initSampleSize = initialisationObservations.size(); + + // load white noise estimates + List whiteNoiseEstimatesAll; + try { + String path = Parameters.getInputDirectory() + "lifetime_incomes" + File.separator + "whitenoise_estimates.csv"; + FileReader fileReader = new FileReader(path); + whiteNoiseEstimatesAll = new CsvToBeanBuilder(fileReader).withType(WhiteNoiseEstimate.class).build().parse(); + } catch (IOException e) { + e.printStackTrace(); + throw new RuntimeException("Problem reading initialisation observations"); + } catch (Throwable e) { + e.printStackTrace(); + throw e; + } + + // initialise template donors for all age/gender cohorts + for (int ii=0; ii < simCohortSize; ii++) { + + InitialisationObservation obs = initialisationObservations.get(generator.nextInt(initSampleSize)); + TemplateIndividual individual = new TemplateIndividual(ii, obs, generator.nextLong()); + templateIndividuals.add(individual); + } + + // project lifetimes for template donors (normalised incomes) + for (int age = 1; age <= endAge; age++) { + + final int currentCapAge = Math.min(age, capAge); + final int currentUncapAge = age; + List whiteNoiseEstimates = whiteNoiseEstimatesAll.stream().filter(w -> w.getAge() == currentCapAge).toList(); + int noiseSampleSize = whiteNoiseEstimates.size(); + double adjFactor = AdjustmentFactors.getWhiteNoise(currentCapAge); + + templateIndividuals.parallelStream().forEach(individual -> { + //for (TemplateIndividual individual : templateIndividuals) { + + WhiteNoiseEstimate noise = whiteNoiseEstimates.get(individual.getNextInt(noiseSampleSize)); + double etaAdj = noise.getEtaHat() * adjFactor; + individual.projectNormIncome(currentUncapAge, etaAdj); + //} + }); + } + + // write template profiles to csv + boolean writeToCSV2 = false; + if (writeToCSV2) { + + List incomes = new ArrayList<>(); + for (TemplateIndividual individual : templateIndividuals) { + + incomes.addAll(individual.getNormIncomes()); + } + try { + String path = outputDir + File.separator + "normalised_incomes.csv"; + Writer writer = new FileWriter(path); + StatefulBeanToCsv beanToCsv = new StatefulBeanToCsvBuilder(writer).build(); + beanToCsv.write(incomes); + writer.close(); + } catch (CsvRequiredFieldEmptyException | CsvDataTypeMismatchException e) { + // a bean had an empty required field or a value that couldn't be converted + System.err.println("Could not write CSV: " + e.getMessage()); + } catch (IOException e) { + e.printStackTrace(); + throw new RuntimeException("Problem reading initialisation observations"); + } catch (Throwable e) { + e.printStackTrace(); + throw e; + } + } + + // finalise + log.info("Completed lifetime income projections"); + } + + public static void imputeProfiles(Logger log, Set personSet, List templateIndividuals, boolean writeToCSV, String outputDir) { + + log.info("Imputing lifetime income profiles for sample"); + System.out.println("Imputing lifetime income profiles for sample"); + + int endAge = templateIndividuals.getFirst().getNormIncomes().size() - 1; + + // loop through each age in order + for (int age = 0; age <= endAge; age++) { + + // sort template individuals by norm income at current age + int ageHere = age; + templateIndividuals.sort(Comparator.comparingDouble(individual -> individual.getNormIncome(ageHere).getZValue())); + final double[] sortedZ = templateIndividuals.stream().mapToDouble(individual -> individual.getNormIncome(ageHere).getZValue()).toArray(); + boolean agePlus = (age==endAge); + + // loop through person set + personSet.parallelStream().forEach(person -> { + //for (Person person : personSet) { + + person.matchLifetimeIncome(ageHere, agePlus, sortedZ, templateIndividuals); + //} + }); + } + + // write template profiles to csv + boolean writeToCSV2 = true; + if (writeToCSV) { + + List incomes = new ArrayList<>(); + for (Person person : personSet) { + + for (int age = 0; age <= endAge; age++) { + + ImputedIncome income = new ImputedIncome(person, age); + incomes.add(income); + } + } + try { + String path = outputDir + File.separator + "imputed_incomes.csv"; + Writer writer = new FileWriter(path); + StatefulBeanToCsv beanToCsv = new StatefulBeanToCsvBuilder(writer).build(); + beanToCsv.write(incomes); + writer.close(); + } catch (CsvRequiredFieldEmptyException | CsvDataTypeMismatchException e) { + // a bean had an empty required field or a value that couldn't be converted + System.err.println("Could not write CSV: " + e.getMessage()); + } catch (IOException e) { + e.printStackTrace(); + throw new RuntimeException("Problem writing imputed incomes"); + } catch (Throwable e) { + e.printStackTrace(); + throw e; + } + } + + // finalise + log.info("Completed imputing lifetime incomes"); + } + + public static void imputeProfiles2(Logger log, Integer capAge, long seed, List templateIndividuals, boolean writeToCSV, String outputDir) { + + log.info("Imputing lifetime income profiles for sample"); + System.out.println("Imputing lifetime income profiles for sample"); + + // define year + int year = 2023; + + // load survey sample + List sampleIndividuals; + try { + String path = Parameters.getInputDirectory() + "lifetime_incomes" + File.separator + "/sample/" + year + ".csv"; + FileReader fileReader = new FileReader(path); + sampleIndividuals = new CsvToBeanBuilder(fileReader).withType(SampleIndividual.class).build().parse(); + } catch (IOException e) { + e.printStackTrace(); + throw new RuntimeException("Problem reading initialisation start sample for analysis"); + } catch (Throwable e) { + e.printStackTrace(); + throw e; + } + long seedHere = seed; + for (SampleIndividual individual : sampleIndividuals) { + + seedHere += 1; + individual.setGender(individual.getMale() == 1 ? Gender.Male : Gender.Female); + individual.setGenerator(seedHere); + } + + // match templates to sample popualtion + // loop through each age in order + for (int age = 0; age <= capAge; age++) { + + // sort template individuals by norm income at current age + int ageHere = age; + templateIndividuals.sort(Comparator.comparingDouble(individual -> individual.getNormIncome(ageHere).getZValue())); + final double[] sortedZ = templateIndividuals.stream().mapToDouble(individual -> individual.getNormIncome(ageHere).getZValue()).toArray(); + boolean agePlus = (age>=capAge); + + // loop through sample + sampleIndividuals.parallelStream().forEach(individual -> { + //for (Person person : personSet) { + + individual.matchLifetimeIncome(ageHere, agePlus, sortedZ, templateIndividuals); + //} + }); + } + + // identify end age of template population + int endAge = templateIndividuals.getFirst().getNormIncomes().size() - 1; + + // write matched income profiles to csv + if (writeToCSV) { + + List incomes = new ArrayList<>(); + for (SampleIndividual individual : sampleIndividuals) { + + individual.projectAgeAtDeath(); + for (int age = 0; age <= endAge; age++) { + + ImputedIncome income = new ImputedIncome(individual, age); + incomes.add(income); + } + } + try { + String path = outputDir + File.separator + "imputed_incomes2.csv"; + Writer writer = new FileWriter(path); + StatefulBeanToCsv beanToCsv = new StatefulBeanToCsvBuilder(writer).build(); + beanToCsv.write(incomes); + writer.close(); + } catch (CsvRequiredFieldEmptyException | CsvDataTypeMismatchException e) { + // a bean had an empty required field or a value that couldn't be converted + System.err.println("Could not write CSV: " + e.getMessage()); + } catch (IOException e) { + e.printStackTrace(); + throw new RuntimeException("Problem writing imputed incomes"); + } catch (Throwable e) { + e.printStackTrace(); + throw e; + } + } + + // finalise + log.info("Completed imputing lifetime incomes"); + } +} diff --git a/src/main/java/simpaths/model/lifetime_incomes/ManagerProjectLifetimeIncomes.java b/src/main/java/simpaths/model/lifetime_incomes/ManagerProjectLifetimeIncomes.java deleted file mode 100644 index 610dc5fc9..000000000 --- a/src/main/java/simpaths/model/lifetime_incomes/ManagerProjectLifetimeIncomes.java +++ /dev/null @@ -1,252 +0,0 @@ -package simpaths.model.lifetime_incomes; - -import com.opencsv.bean.CsvToBeanBuilder; -import com.opencsv.bean.StatefulBeanToCsv; -import com.opencsv.bean.StatefulBeanToCsvBuilder; -import com.opencsv.exceptions.CsvDataTypeMismatchException; -import com.opencsv.exceptions.CsvRequiredFieldEmptyException; -import jakarta.persistence.EntityManager; -import jakarta.persistence.EntityTransaction; -import jakarta.persistence.Persistence; -import simpaths.data.Parameters; -import simpaths.model.enums.Gender; - -import java.io.*; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.*; -import java.util.random.RandomGenerator; -import java.util.stream.IntStream; - -import org.apache.logging.log4j.Logger; - -public class ManagerProjectLifetimeIncomes { - - - /** - * ENTRY POINT FOR MANAGER - */ - public static void run(Logger log, Integer startBirthYear, Integer endBirthYear, - Integer endAge, Integer simCohortSize, boolean writeToCSV, long seed, double age0StdDev, String outputDir) { - - log.info("Initialising lifetime income projections"); - System.out.println("Initialising lifetime income projections"); - - // start projecting lifetime incomes - initialiseLifetimeIncomeDatabase(); - RandomGenerator generator = new Random(seed); - - // load initialisation values - List initialisationObservations; - try { - String path = Parameters.getInputDirectory() + "lifetime_incomes" + File.separator + "initialisation_observations.csv"; - FileReader fileReader = new FileReader(path); - initialisationObservations = new CsvToBeanBuilder(fileReader).withType(InitialisationObservation.class).build().parse(); - } catch (IOException e) { - e.printStackTrace(); - throw new RuntimeException("Problem reading initialisation observations"); - } catch (Throwable e) { - e.printStackTrace(); - throw e; - } - int initSampleSize = initialisationObservations.size(); - - // load white noise estimates - List whiteNoiseEstimates; - try { - String path = Parameters.getInputDirectory() + "lifetime_incomes" + File.separator + "whitenoise_estimates.csv"; - FileReader fileReader = new FileReader(path); - whiteNoiseEstimates = new CsvToBeanBuilder(fileReader).withType(WhiteNoiseEstimate.class).build().parse(); - } catch (IOException e) { - e.printStackTrace(); - throw new RuntimeException("Problem reading initialisation observations"); - } catch (Throwable e) { - e.printStackTrace(); - throw e; - } - int noiseSampleSize = whiteNoiseEstimates.size(); - - // generate common set of z data for all age/gender cohorts - List templateIndividuals = new ArrayList<>(); - - // initialise template individuals - for (int ii=0; ii < simCohortSize; ii++) { - - InitialisationObservation obs = initialisationObservations.get(generator.nextInt(initSampleSize)); - TemplateIndividual individual = new TemplateIndividual(ii, obs, generator.nextLong()); - templateIndividuals.add(individual); - } - - // project lifetimes for template individuals - IntStream.range(0, simCohortSize).parallel().forEach(ii -> { - //for (TemplateIndividual individual : templateIndividuals) { - - TemplateIndividual individual = templateIndividuals.get(ii); - for (int age = 1; age <= endAge; age++) { - - WhiteNoiseEstimate noise = whiteNoiseEstimates.get(individual.getNextInt(noiseSampleSize)); - double etaAdj = noise.getEtaHat() * AdjustmentFactors.getWhiteNoise(age); - individual.projectNormIncome(age, etaAdj); - } - //} - }); - - boolean writeToCSV2 = true; - if (writeToCSV2) { - - List incomes = new ArrayList<>(); - for (TemplateIndividual individual : templateIndividuals) { - - incomes.addAll(individual.getNormIncomes()); - } - try { - String path = outputDir + File.separator + "normalised_incomes.csv"; - Writer writer = new FileWriter(path); - StatefulBeanToCsv beanToCsv = new StatefulBeanToCsvBuilder(writer).build(); - beanToCsv.write(incomes); - writer.close(); - } catch (CsvRequiredFieldEmptyException | CsvDataTypeMismatchException e) { - // a bean had an empty required field or a value that couldn't be converted - System.err.println("Could not write CSV: " + e.getMessage()); - } catch (IOException e) { - e.printStackTrace(); - throw new RuntimeException("Problem reading initialisation observations"); - } catch (Throwable e) { - e.printStackTrace(); - throw e; - } - } - - // loop over birth years - boolean initialised = true; - for (int by = startBirthYear; by <= endBirthYear; by++) { - - log.info("Projecting lifetime incomes for birth year " + by); - System.out.println("Projecting lifetime incomes for birth year " + by); - - // initialise sets for csv reporting - Set cohorts = new LinkedHashSet<>(); - Set individuals = new LinkedHashSet<>(); - Set annualIncomes = new LinkedHashSet<>(); - - // loop over genders - for (Gender gender : Gender.values()) { - - BirthCohort birthCohort = new BirthCohort(by, gender, endAge); - cohorts.add(birthCohort); - Individual[] individualsLocal = new Individual[simCohortSize]; - AnnualIncome[] annualIncomesLocal = new AnnualIncome[simCohortSize*(endAge+1)]; - - // loop over individuals - final int birthYear = by; - IntStream.range(0, simCohortSize).parallel().forEach(ii -> { -// for (int ii=0; ii < simCohortSize; ii++) { - // loop over new individuals - - Individual individual = new Individual(birthCohort); - individualsLocal[ii] = individual; - - // loop over ages - for (int aa = 0; aa <= endAge; aa++) { - - double rnd = generator.nextDouble(); - AnnualIncome income = new AnnualIncome(birthYear + aa, individual, rnd, age0StdDev); - annualIncomesLocal[ii*(endAge+1)+aa] = income; - } -// } - }); - for (Individual individual : individualsLocal) { - birthCohort.addIndividual(individual); - individuals.add(individual); - } - annualIncomes.addAll(Arrays.asList(annualIncomesLocal)); - - // commit to database - writeLifetimeIncomeDatabase(log, birthCohort); - } - - // write to csv - if (writeToCSV) { - ExportCSV exportCohorts = new ExportCSV(by, cohorts); - ExportCSV exportIndividuals = new ExportCSV(by, individuals); - ExportCSV exportAnnualIncomes = new ExportCSV(by, annualIncomes); - } - } - - log.info("Completed lifetime income projections"); - } - - private static void writeLifetimeIncomeDatabase(Logger log, BirthCohort cohort) { - - EntityTransaction txn = null; - try { - // initialise database for storing results - String fileName = Parameters.getInputDirectory() + "input"; - var propertyMap = new HashMap(); - propertyMap.put("hibernate.connection.url", "jdbc:h2:file:" + fileName + ";TRACE_LEVEL_FILE=0;TRACE_LEVEL_SYSTEM_OUT=0;AUTO_SERVER=TRUE"); - EntityManager em = Persistence.createEntityManagerFactory("lifetime-incomes", propertyMap).createEntityManager(); - txn = em.getTransaction(); - txn.begin(); - log.info("Running em.persist()"); - em.persist(cohort); - txn.commit(); - em.close(); - } catch (Exception e) { - if (txn != null) - txn.rollback(); - e.printStackTrace(); - throw new RuntimeException("Problem writing lifetime incomes dataset"); - } - } - - private static void initialiseLifetimeIncomeDatabase() { - - // initialise database connection - Connection conn = null; - try { - String fileName = Parameters.getInputDirectory() + "input"; - Class.forName("org.h2.Driver"); - conn = DriverManager.getConnection("jdbc:h2:file:" + fileName + ";TRACE_LEVEL_FILE=0;TRACE_LEVEL_SYSTEM_OUT=0;AUTO_SERVER=TRUE", "sa", ""); - - // check that database is empty at start - Statement stat = null; - try { - stat = conn.createStatement(); - stat.execute( "DROP TABLE IF EXISTS birthcohort CASCADE;"); - stat.execute( "DROP TABLE IF EXISTS individual CASCADE;"); - stat.execute( "DROP TABLE IF EXISTS annualincome CASCADE;"); - stat.execute( "CREATE TABLE birthcohort (ID BIGINT NOT NULL PRIMARY KEY AUTO_INCREMENT, BIRTH_YEAR INT DEFAULT 2019, END_AGE INT DEFAULT 80, GENDER VARCHAR_IGNORECASE DEFAULT 'Male');"); - } catch(Exception e){ - // throw new IllegalArgumentException("SQL Exception thrown!" + e.getMessage()); - e.printStackTrace(); - } - finally { - try { - if(stat != null) - stat.close(); - } catch (SQLException e) { - e.printStackTrace(); - } - } - } - catch(ClassNotFoundException | SQLException e){ - if(e instanceof ClassNotFoundException) { - System.out.println( "ERROR: Class not found: " + e.getMessage() + "\nCheck that the input.h2.db " - + "exists in the input folder. If not, unzip the input.h2.zip file and store the resulting " - + "input.h2.db in the input folder!\n"); - } - else { - throw new IllegalArgumentException("SQL Exception thrown! " + e.getMessage()); - } - } - finally { - try { - if (conn != null) { conn.close(); } - } catch (SQLException e) { - e.printStackTrace(); - } - } - } -} diff --git a/src/main/java/simpaths/model/lifetime_incomes/RegressionProxy.java b/src/main/java/simpaths/model/lifetime_incomes/RegressionProxy.java new file mode 100644 index 000000000..f59dcf48a --- /dev/null +++ b/src/main/java/simpaths/model/lifetime_incomes/RegressionProxy.java @@ -0,0 +1,374 @@ +package simpaths.model.lifetime_incomes; + +import microsim.statistics.IDoubleSource; +import simpaths.data.Parameters; +import simpaths.model.enums.TimeSeriesVariable; + +public class RegressionProxy implements IDoubleSource { + + private int age; + private int year; + + public RegressionProxy() {} + public RegressionProxy(int age, int year) { + this.age = age; + this.year = year; + } + + public int getAge() { + return age; + } + + public void setAge(int age) { + this.age = age; + } + + public int getYear() { + return year; + } + + public void setYear(int year) { + this.year = year; + } + + + public enum Variables { + Age0, + Age1, + Age2, + Age3, + Age4, + Age5, + Age6, + Age7, + Age8, + Age9, + Age10, + Age11, + Age12, + Age13, + Age14, + Age15, + Age16, + Age17, + Age18, + Age19, + Age20, + Age21, + Age22, + Age23, + Age24, + Age25, + Age26, + Age27, + Age28, + Age29, + Age30, + Age31, + Age32, + Age33, + Age34, + Age35, + Age36, + Age37, + Age38, + Age39, + Age40, + Age41, + Age42, + Age43, + Age44, + Age45, + Age46, + Age47, + Age48, + Age49, + Age50, + Age51, + Age52, + Age53, + Age54, + Age55, + Age56, + Age57, + Age58, + Age59, + Age60, + Age61, + Age62, + Age63, + Age64, + Age65, + Age66, + Age67, + Age68, + Age69, + Age70, + Age71, + Age72, + Age73, + Age74, + Age75, + Age76, + Age77, + Age78, + Age79, + Log_GDP_per_capita, + Age80plus, + } + public double getDoubleValue(Enum variableID) { + + switch ((Variables) variableID) { + + case Age0 -> { + return (age == 0) ? 1 : 0; + } + case Age1 -> { + return (age == 1) ? 1 : 0; + } + case Age2 -> { + return (age == 2) ? 1 : 0; + } + case Age3 -> { + return (age == 3) ? 1 : 0; + } + case Age4 -> { + return (age == 4) ? 1 : 0; + } + case Age5 -> { + return (age == 5) ? 1 : 0; + } + case Age6 -> { + return (age == 6) ? 1 : 0; + } + case Age7 -> { + return (age == 7) ? 1 : 0; + } + case Age8 -> { + return (age == 8) ? 1 : 0; + } + case Age9 -> { + return (age == 9) ? 1 : 0; + } + case Age10 -> { + return (age == 10) ? 1 : 0; + } + case Age11 -> { + return (age == 11) ? 1 : 0; + } + case Age12 -> { + return (age == 12) ? 1 : 0; + } + case Age13 -> { + return (age == 13) ? 1 : 0; + } + case Age14 -> { + return (age == 14) ? 1 : 0; + } + case Age15 -> { + return (age == 15) ? 1 : 0; + } + case Age16 -> { + return (age == 16) ? 1 : 0; + } + case Age17 -> { + return (age == 17) ? 1 : 0; + } + case Age18 -> { + return (age == 18) ? 1 : 0; + } + case Age19 -> { + return (age == 19) ? 1 : 0; + } + case Age20 -> { + return (age == 20) ? 1 : 0; + } + case Age21 -> { + return (age == 21) ? 1 : 0; + } + case Age22 -> { + return (age == 22) ? 1 : 0; + } + case Age23 -> { + return (age == 23) ? 1 : 0; + } + case Age24 -> { + return (age == 24) ? 1 : 0; + } + case Age25 -> { + return (age == 25) ? 1 : 0; + } + case Age26 -> { + return (age == 26) ? 1 : 0; + } + case Age27 -> { + return (age == 27) ? 1 : 0; + } + case Age28 -> { + return (age == 28) ? 1 : 0; + } + case Age29 -> { + return (age == 29) ? 1 : 0; + } + case Age30 -> { + return (age == 30) ? 1 : 0; + } + case Age31 -> { + return (age == 31) ? 1 : 0; + } + case Age32 -> { + return (age == 32) ? 1 : 0; + } + case Age33 -> { + return (age == 33) ? 1 : 0; + } + case Age34 -> { + return (age == 34) ? 1 : 0; + } + case Age35 -> { + return (age == 35) ? 1 : 0; + } + case Age36 -> { + return (age == 36) ? 1 : 0; + } + case Age37 -> { + return (age == 37) ? 1 : 0; + } + case Age38 -> { + return (age == 38) ? 1 : 0; + } + case Age39 -> { + return (age == 39) ? 1 : 0; + } + case Age40 -> { + return (age == 40) ? 1 : 0; + } + case Age41 -> { + return (age == 41) ? 1 : 0; + } + case Age42 -> { + return (age == 42) ? 1 : 0; + } + case Age43 -> { + return (age == 43) ? 1 : 0; + } + case Age44 -> { + return (age == 44) ? 1 : 0; + } + case Age45 -> { + return (age == 45) ? 1 : 0; + } + case Age46 -> { + return (age == 46) ? 1 : 0; + } + case Age47 -> { + return (age == 47) ? 1 : 0; + } + case Age48 -> { + return (age == 48) ? 1 : 0; + } + case Age49 -> { + return (age == 49) ? 1 : 0; + } + case Age50 -> { + return (age == 50) ? 1 : 0; + } + case Age51 -> { + return (age == 51) ? 1 : 0; + } + case Age52 -> { + return (age == 52) ? 1 : 0; + } + case Age53 -> { + return (age == 53) ? 1 : 0; + } + case Age54 -> { + return (age == 54) ? 1 : 0; + } + case Age55 -> { + return (age == 55) ? 1 : 0; + } + case Age56 -> { + return (age == 56) ? 1 : 0; + } + case Age57 -> { + return (age == 57) ? 1 : 0; + } + case Age58 -> { + return (age == 58) ? 1 : 0; + } + case Age59 -> { + return (age == 59) ? 1 : 0; + } + case Age60 -> { + return (age == 60) ? 1 : 0; + } + case Age61 -> { + return (age == 61) ? 1 : 0; + } + case Age62 -> { + return (age == 62) ? 1 : 0; + } + case Age63 -> { + return (age == 63) ? 1 : 0; + } + case Age64 -> { + return (age == 64) ? 1 : 0; + } + case Age65 -> { + return (age == 65) ? 1 : 0; + } + case Age66 -> { + return (age == 66) ? 1 : 0; + } + case Age67 -> { + return (age == 67) ? 1 : 0; + } + case Age68 -> { + return (age == 68) ? 1 : 0; + } + case Age69 -> { + return (age == 69) ? 1 : 0; + } + case Age70 -> { + return (age == 70) ? 1 : 0; + } + case Age71 -> { + return (age == 71) ? 1 : 0; + } + case Age72 -> { + return (age == 72) ? 1 : 0; + } + case Age73 -> { + return (age == 73) ? 1 : 0; + } + case Age74 -> { + return (age == 74) ? 1 : 0; + } + case Age75 -> { + return (age == 75) ? 1 : 0; + } + case Age76 -> { + return (age == 76) ? 1 : 0; + } + case Age77 -> { + return (age == 77) ? 1 : 0; + } + case Age78 -> { + return (age == 78) ? 1 : 0; + } + case Age79 -> { + return (age == 79) ? 1 : 0; + } + case Log_GDP_per_capita -> { + return Math.log(Parameters.getTimeSeriesValue(year, TimeSeriesVariable.GDPperCapita)); + } + case Age80plus -> { + return (age >= 80) ? 1. : 0.; + } + default -> { + return 0.0; + } + } + } +} diff --git a/src/main/java/simpaths/model/lifetime_incomes/SampleIndividual.java b/src/main/java/simpaths/model/lifetime_incomes/SampleIndividual.java new file mode 100644 index 000000000..a5ad8b471 --- /dev/null +++ b/src/main/java/simpaths/model/lifetime_incomes/SampleIndividual.java @@ -0,0 +1,183 @@ +package simpaths.model.lifetime_incomes; + +import com.opencsv.bean.CsvBindByName; +import simpaths.data.Parameters; +import simpaths.model.enums.Gender; +import simpaths.model.enums.TimeSeriesVariable; + +import java.util.Arrays; +import java.util.List; +import java.util.Random; +import java.util.random.RandomGenerator; + +public class SampleIndividual { + + @CsvBindByName private int id; + @CsvBindByName private int age; + @CsvBindByName private int male; + @CsvBindByName private double income; + @CsvBindByName private int year; + private Gender gender; + private int ageAtDeath; + TemplateIndividual ltIncomeDonor; + RandomGenerator generator; // random number generator + + public SampleIndividual() { + } + + public void setGenerator(long seed) { + generator = new Random(seed); + } + public int getAge() { + return age; + } + + public void setAge(int age) { + this.age = age; + } + + public double getIncome() { + return income; + } + + public void setIncome(double income) { + this.income = income; + } + + public int getId() { + return id; + } + + public void setId(int id) { + this.id = id; + } + + public int getMale() { + gender = (male ==1) ? Gender.Male : Gender.Female; + return male; + } + + public void setMale(int male) { + this.male = male; + } + + public int getYear() { + return year; + } + + public void setYear(int year) { + this.year = year; + } + + public Gender getGender() { + return gender; + } + + public void setGender(Gender gender) { + this.gender = gender; + } + + public TemplateIndividual getLtIncomeDonor() { + return ltIncomeDonor; + } + + public void setLtIncomeDonor(TemplateIndividual ltIncomeDonor) { + this.ltIncomeDonor = ltIncomeDonor; + } + + public int getAgeAtDeath() { + return ageAtDeath; + } + + public void setAgeAtDeath(int ageAtDeath) { + this.ageAtDeath = ageAtDeath; + } + + + public void matchLifetimeIncome(int ageH, boolean agePlus, final double[] sortedZ, List templateIndividuals) { + + boolean matchHere; + if (agePlus) + matchHere = (age >= ageH); + else + matchHere = (age == ageH); + if (matchHere) { + + // identify normalised income target + if (income > 0.0) { + + double gmEqDispIncomePerYear = getGMEquivalisedIncome(age, year); + double zTarget = Math.log(income / gmEqDispIncomePerYear); + int ii = Arrays.binarySearch(sortedZ, zTarget); + if (ii >= 0) { + // exact match found + setLtIncomeDonor(templateIndividuals.get(ii)); + } else { + // adopt nearest neighbour + int ins = -ii - 1; + if (ins == 0) { + // lower bound + setLtIncomeDonor(templateIndividuals.get(ins)); + } else if (ins == sortedZ.length) { + // upper bound + setLtIncomeDonor(templateIndividuals.get(ins - 1)); + } else if (zTarget - sortedZ[ins -1] < sortedZ[ins] - zTarget) { + // choose lower bound + setLtIncomeDonor(templateIndividuals.get(ins - 1)); + } else { + // choose upper bound + setLtIncomeDonor(templateIndividuals.get(ins)); + } + } + } + else { + + setLtIncomeDonor(templateIndividuals.get(0)); + } + } + } + + public double getGMEquivalisedIncome(int ageH, int yearH) { + + Double val = Parameters.getEquivalisedIncome(gender, ageH, yearH); + if (!Parameters.isFinite(val)) { + // outside observed range, use regression model + + RegressionProxy proxy = new RegressionProxy(ageH, yearH); + if (gender.toBoolean()) { + val = Math.exp(Parameters.getRegLifetimeIncome1a().getScore(proxy, RegressionProxy.Variables.class)); + } + else { + val = Math.exp(Parameters.getRegLifetimeIncome1b().getScore(proxy, RegressionProxy.Variables.class)); + } + } + return val; + } + + public double getImputedIncome(int ageH) { + + int yearH = year - age + ageH; + double gmIncome = getGMEquivalisedIncome(ageH, yearH); + if (ltIncomeDonor != null) + return Math.exp(ltIncomeDonor.getNormIncome(ageH).getZValue()) * gmIncome; + else + return gmIncome; + } + + public void projectAgeAtDeath() { + + boolean found = false; + int maxAge = ltIncomeDonor.getNormIncomes().size() - 1; + for (int ageHere = age+1; ageHere < maxAge; ageHere++) { + int yearHere = year - age + ageHere; + double mortality = Parameters.getMortalityProbability(gender, ageHere, yearHere); + if (generator.nextDouble() < mortality) { + found = true; + ageAtDeath = ageHere; + break; + } + } + if (!found) + ageAtDeath = maxAge; + } +} diff --git a/src/main/java/simpaths/model/lifetime_incomes/WhiteNoiseEstimate.java b/src/main/java/simpaths/model/lifetime_incomes/WhiteNoiseEstimate.java index 220b8e866..b736d1539 100644 --- a/src/main/java/simpaths/model/lifetime_incomes/WhiteNoiseEstimate.java +++ b/src/main/java/simpaths/model/lifetime_incomes/WhiteNoiseEstimate.java @@ -10,7 +10,7 @@ public class WhiteNoiseEstimate { @CsvBindByName private double pidp; // person id @CsvBindByName private double etaHat; // estimated white noise - @CsvBindByName private double age; // observation age + @CsvBindByName private int age; // observation age @CsvBindByName private double weight; // survey weight // Required by CsvToObjectLoader (reflection-based instantiation). @@ -22,5 +22,5 @@ public WhiteNoiseEstimate() { public void setPidp(double pidp) { this.pidp = pidp; } public double getEtaHat() { return etaHat; } public void setEtaHat(double etaHat) { this.etaHat = etaHat; } - public double getAge() { return age; } + public int getAge() { return age; } }