[
https://issues.apache.org/jira/browse/SPARK-58318?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Marc Le Bihan updated SPARK-58318:
----------------------------------
Description:
I'm following a {{String[]}} of competencies codes, at their indexes :
{code:java}
codesCompetences = (size = 121)
0 = "C1005" 1 = "C1010" 2 = "C1015" 3 = "C1020" 4 = "C1025"
5 = "C1030" 6 = "C1035" 7 = "C1040" 8 = "C1045" 9 = "C1505"
10 = "C1510" 11 = "C1515" 12 = "C1520" 13 = "C2000" 14 = "C2005"
15 = "C2010" 16 = "C2015" 17 = "C2020" 18 = "C2025" 19 = "C2030"
20 = "C2035" 21 = "C2040" 22 = "C2045" 23 = "C2050" 24 = "C2055"
25 = "C2060" 26 = "C2065" 27 = "C2070" 28 = "C2075" 29 = "C2080"
30 = "C2085" 31 = "C2090" 32 = "C2095" 33 = "C2096" 34 = "C2500"
35 = "C2505" 36 = "C2510" 37 = "C3000" 38 = "C3005" 39 = "C3010"
40 = "C3015" 41 = "C3020" 42 = "C3025" 43 = "C3030" 44 = "C3035"
45 = "C3040" 46 = "C3045" 47 = "C3050" 48 = "C3055" 49 = "C3060"
50 = "C3500" 51 = "C4000" 52 = "C4005" 53 = "C4010" 54 = "C4500"
55 = "C4505" 56 = "C4510" 57 = "C4515" 58 = "C4520" 59 = "C5000"
60 = "C5005" 61 = "C5010" 62 = "C5015" 63 = "C5020" 64 = "C5025"
65 = "C5030" 66 = "C5035" 67 = "C5040" 68 = "C5045" 69 = "C5050"
70 = "C5500" 71 = "C5505" 72 = "C5510" 73 = "C5515" 74 = "C5520"
75 = "C5525" 76 = "C5530" 77 = "C5535" 78 = "C5540" 79 = "C6000"
80 = "C6005" 81 = "C6010" 82 = "C6015" 83 = "C6020" 84 = "C6025"
85 = "C6030" 86 = "C6035" 87 = "C6040" 88 = "C6045" 89 = "C6055"
90 = "C6060" 91 = "C6065" 92 = "C6070" 93 = "C6075" 94 = "C6500"
95 = "C6505" 96 = "C6510" 97 = "C6515" 98 = "C6520" 99 = "C6525"
100 = "C6530" 101 = "C6535" 102 = "C7000" 103 = "C7500" 104 = "C7505"
105 = "C7510" 106 = "C7515" 107 = "C7520" 108 = "C7525" 109 = "C7530"
110 = "C7535" 111 = "C7540" 112 = "C7541" 113 = "C7545" 114 = "C7546"
115 = "C7550" 116 = "C7555" 117 = "C7560" 118 = "C7565" 119 = "C7570"
120 = "C7575 {code}
With them, I'm converting values of columns coming from a CSV file:
{code:java}
private Dataset<Row>
extraireCompetencesDelegueesEtInteretCommunautaire(Dataset<Row> dataset, int
anneeCog) {
// Ces valeurs sont celles qu'un distinct() montre possibles sur les
colonnes du fichier csv d'entrée.
final String COMPETENCE_DELEGUEE = "délégué";
final String COMPETENCE_INTERET_COMMUNAUTAIRE = "avec intérêt
communautaire";
List<String> codesCompetences =
this.datasetCompetence.codesCompetencesAsList(anneeCog);
for(String codeCompetence : codesCompetences) {
Column colCompetence = new Column(codeCompetence);
// Créer le champ Cxxx_Deleguee et le one hot encoder en "1" ou "0" selon
que la valeur associée au code compétence vaut 'délégué' ou pas.
Column competenceDeleguee =
when(colCompetence.equalTo(COMPETENCE_DELEGUEE), lit("1")).otherwise(lit("0"));
dataset = dataset.withColumn(codeCompetence + "_Deleguee",
competenceDeleguee);
// Créer le champ Cxxx_InteretCommunautaire et le one hot encoder en "1"
ou "0" selon que la valeur associée au code compétence vaut 'avec intérêt
communautaire' ou pas.
Column competenceInteretCommunautaire =
when(colCompetence.equalTo(COMPETENCE_INTERET_COMMUNAUTAIRE),
lit("1")).otherwise(lit("0"));
dataset = dataset.withColumn(codeCompetence + "_InteretCommunautaire",
competenceInteretCommunautaire);
// one hot encoder la présence de la compétence
Column encoding = when(colCompetence.isin(OUI, COMPETENCE_DELEGUEE,
COMPETENCE_INTERET_COMMUNAUTAIRE), lit("1"))
.when(colCompetence.equalTo(NON), lit("0"))
.otherwise(colCompetence);
dataset = dataset.withColumn(codeCompetence, encoding);
LOGGER.info("added: '{}'", encoding);
}
return dataset;
}
{code}
This loop adds to the {{Dataset}} some columns each time,. Among them one is :
{code:java}
added: 'CASE WHEN in(C6525, 'OUI', 'délégué', 'avec intérêt communautaire')
THEN '1' WHEN =(C6525, 'NON') THEN '0' ELSE C6525 END'{code}
and it's the last log shown before a:
{code:java}
java.lang.StackOverflowError
at java.base/java.lang.ref.Reference.refersToImpl(Reference.java:384)
at java.base/java.lang.ref.Reference.refersTo(Reference.java:375)
at java.base/java.lang.ThreadLocal$ThreadLocalMap.set(ThreadLocal.java:539)
at java.base/java.lang.ThreadLocal.set(ThreadLocal.java:243)
at java.base/java.lang.ThreadLocal.set(ThreadLocal.java:229)
at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.set(origin.scala:93)
at
org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:105)
at
org.apache.spark.sql.catalyst.trees.TreeNode.transformDownWithPruning(TreeNode.scala:495)
at
org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDownWithPruning$3(TreeNode.scala:500)
at
org.apache.spark.sql.catalyst.trees.UnaryLike.mapChildren(TreeNode.scala:1268)
at
org.apache.spark.sql.catalyst.trees.UnaryLike.mapChildren$(TreeNode.scala:1267)
at
org.apache.spark.sql.catalyst.expressions.UnaryExpression.mapChildren(Expression.scala:596)
at
org.apache.spark.sql.catalyst.trees.TreeNode.transformDownWithPruning(TreeNode.scala:500)
at
org.apache.spark.sql.catalyst.plans.QueryPlan.$anonfun$transformExpressionsDownWithPruning$1(QueryPlan.scala:223)
at
org.apache.spark.sql.catalyst.plans.QueryPlan.$anonfun$mapExpressions$1(QueryPlan.scala:264)
at
org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:107)
at
org.apache.spark.sql.catalyst.plans.QueryPlan.transformExpression$1(QueryPlan.scala:264)
at
org.apache.spark.sql.catalyst.plans.QueryPlan.recursiveTransform$1(QueryPlan.scala:276)
at
org.apache.spark.sql.catalyst.plans.QueryPlan.$anonfun$mapExpressions$4(QueryPlan.scala:282)
at scala.collection.immutable.List.map(List.scala:236)
at scala.collection.immutable.List.map(List.scala:79)
at
org.apache.spark.sql.catalyst.plans.QueryPlan.recursiveTransform$1(QueryPlan.scala:282)
at
org.apache.spark.sql.catalyst.plans.QueryPlan.$anonfun$mapExpressions$5(QueryPlan.scala:287)
at
org.apache.spark.sql.catalyst.trees.TreeNode.mapProductIterator(TreeNode.scala:337)
at
org.apache.spark.sql.catalyst.plans.QueryPlan.mapExpressions(QueryPlan.scala:287)
at
org.apache.spark.sql.catalyst.plans.QueryPlan.transformExpressionsDownWithPruning(QueryPlan.scala:223)
at
org.apache.spark.sql.catalyst.plans.QueryPlan.transformExpressionsWithPruning(QueryPlan.scala:194)
at
org.apache.spark.sql.catalyst.plans.QueryPlan.transformExpressions(QueryPlan.scala:172)
at
org.apache.spark.sql.catalyst.analysis.DeduplicateRelations$.deduplicate(DeduplicateRelations.scala:240)
at
org.apache.spark.sql.catalyst.analysis.DeduplicateRelations$.deduplicateAndRenew(DeduplicateRelations.scala:323)
at
org.apache.spark.sql.catalyst.analysis.DeduplicateRelations$.org$apache$spark$sql$catalyst$analysis$DeduplicateRelations$$renewDuplicatedRelations(DeduplicateRelations.scala:126)
at
org.apache.spark.sql.catalyst.analysis.DeduplicateRelations$.$anonfun$deduplicate$1(DeduplicateRelations.scala:233)
at
org.apache.spark.sql.catalyst.analysis.DeduplicateRelations$.$anonfun$deduplicate$1$adapted(DeduplicateRelations.scala:232)
at scala.collection.immutable.Vector.foreach(Vector.scala:2125)
at
org.apache.spark.sql.catalyst.analysis.DeduplicateRelations$.deduplicate(DeduplicateRelations.scala:232)
at
org.apache.spark.sql.catalyst.analysis.DeduplicateRelations$.deduplicateAndRenew(DeduplicateRelations.scala:323)
at
org.apache.spark.sql.catalyst.analysis.DeduplicateRelations$.org$apache$spark$sql$catalyst$analysis$DeduplicateRelations$$renewDuplicatedRelations(DeduplicateRelations.scala:126)
at
org.apache.spark.sql.catalyst.analysis.DeduplicateRelations$.$anonfun$deduplicate$1(DeduplicateRelations.scala:233)
at
org.apache.spark.sql.catalyst.analysis.DeduplicateRelations$.$anonfun$deduplicate$1$adapted(DeduplicateRelations.scala:232)
[...] {code}
What I've noticed during my debugs is that:
* If I do a step over step debugging, without allowing my program to resume
from `C6525`, I can pursue through `C6530`, `C6535`... But as soon as do a
Resume program, the stack overflow happens.
So I guess it's an helper thread that starts to work on the request being
prepared that causes the trouble.
The problem didn't exist with Spark {{4.1.1}} I was using before, with {{Java
21}} then.
Remark:
this code is included in a larger one, in the {{if (this.anneeCOG >= 2025)}
below:
{code:java}
@Override
public Dataset<Row> get() {
LOGGER.info("Constitution du dataset des périmètres de groupements
(intercommunalités seulement : {}) par {} pour l'année {}...",
this.epciSeulement, this.tri, this.anneeCOG);
this.session.sparkContext().setJobDescription("EPCIPerimetreWorker");
this.session.sparkContext().setLocalProperty("callSite.short", toString());
// Obtention des SIREN des communes d'après le COG.
Dataset<Row> sirenCommunes =
this.datasetSirenCommunaux.rowSirenCommunes(this.options, this.historique, new
SirenCommunesTriSiren(), this.anneeCOG);
Dataset<Row> perimetres = this.loader.loadOpenData(this.anneeCOG);
// Corriger les catégories de communes "Commune" en commune
perimetres = perimetres.withColumn(CATEGORIE_MEMBRES_GROUPEMENT.champ(),
when(CATEGORIE_MEMBRES_GROUPEMENT.col().equalTo(lit("Commune")),
lit("commune"))
.otherwise(CATEGORIE_MEMBRES_GROUPEMENT.col())); // Celles "Groupement"
en groupement
perimetres = perimetres.withColumn(CATEGORIE_MEMBRES_GROUPEMENT.champ(),
when(CATEGORIE_MEMBRES_GROUPEMENT.col().equalTo(lit("Groupement")),
lit("groupement"))
.otherwise(CATEGORIE_MEMBRES_GROUPEMENT.col()));
// Et celles "Autre organisme" en autre organisme
perimetres = perimetres.withColumn(CATEGORIE_MEMBRES_GROUPEMENT.champ(),
when(CATEGORIE_MEMBRES_GROUPEMENT.col().equalTo(lit("Autre organisme")),
lit("autre organisme"))
.otherwise(CATEGORIE_MEMBRES_GROUPEMENT.col())); perimetres =
perimetres.persist();
boolean versionBanatic2024etAnterieure = this.anneeCOG <= 2024;
PerimetreAdaptateurAnnuel adaptateurAnnuel = versionBanatic2024etAnterieure
? new PerimetreAdaptateurAnnuel2024EtAnterieur() : new
PerimetreAdaptateurAnnuel2025EtPosterieur();
// Si l'année porte sur une version de fichiers BANATIC ancienne,
// les champs de code portant sur les région, département, commune sont
encodés sous la forme 'code - libellé'
perimetres =
adaptateurAnnuel.extraireCodesRegionDepartementArrondissementSiren(perimetres,
this.session);
perimetres = retrouverRegion(perimetres, sirenCommunes, this.anneeCOG);
// TODO : ici, est-ce sûr que c'est POPULATION_TOTALE ?
perimetres = perimetres.withColumnRenamed(POPULATION_TOTALE.champ(),
POPULATION_GROUPEMENT.champ());
perimetres = joinSirenCommunes(perimetres);
perimetres = adaptateurAnnuel.codePostalEtVilleAdministrative(perimetres);
Column joinCommuneSiege =
SIREN_COMMUNE_SIEGE.col(perimetres).equalTo(SIREN_COMMUNE.col(sirenCommunes));
perimetres = perimetres.join(sirenCommunes, joinCommuneSiege, LEFT_OUTER)
.withColumnRenamed(CODE_COMMUNE.champ(), CODE_COMMUNE_SIEGE.champ())
.withColumnRenamed(POPULATION_TOTALE.champ(),
POPULATION_COMMUNE_SIEGE_TOTALE.champ())
.withColumnRenamed(POPULATION_MUNICIPALE.champ(),
POPULATION_COMMUNE_SIEGE_MUNICIPALE.champ())
.withColumnRenamed(POPULATION_COMPTEE_A_PART.champ(),
POPULATION_COMMUNE_SIEGE_COMPTEE_A_PART.champ())
.drop(NOM_REGION.champ())
.drop(CODE_REGION.champ())
.drop(CODE_DEPARTEMENT.champ())
.drop(NOM_DEPARTEMENT.champ())
.drop(SIREN_COMMUNE.champ());
perimetres = adaptateurAnnuel.codePostalEtVilleSiege(perimetres);
// FIXME En 2019, Choisy-le-Roi est inscrit comme EPT lié à la fois à
Grand-Orly Seine, ce qui est exact, mais aussi d'Est-Ensemble comme EPT, ce qui
n'est pas vrai.
Column eptChoisyLeRoiConfusion =
NATURE_JURIDIQUE.col().equalTo("EPT").and(SIREN_GROUPEMENT.col(perimetres).equalTo("200057875"));
if (this.epciSeulement) {
perimetres = perimetres.filter(NATURE_JURIDIQUE.col().isin("CA", "CC",
"CU", "EPT", "METRO", "MET69")
.and(not(eptChoisyLeRoiConfusion)));
}
if (this.anneeCOG >= 2025) {
perimetres = hotEncodingAvecNull(perimetres,
SYNDICAT_A_LA_CARTE.champ()); // Contient des valeurs nulles
perimetres = hotEncodingAvecNull(perimetres, INTERDEPARTEMENTAL.champ());
// Contient des valeurs nulles
perimetres = hotEncodingAvecNull(perimetres, ZONE_DE_MONTAGNE.champ());
// Contient des valeurs nulles
perimetres = hotEncodingAvecNull(perimetres, GESTION_DES_EAUX.champ());
// Contient des valeurs nulles
perimetres = hotEncodingAvecNull(perimetres,
ETABLISSEMENT_PUBLIC_TERRITORIAL_DE_BASSIN.champ()); // Contient des valeurs
nulles
perimetres = hotEncodingAvecNull(perimetres,
TAXE_ENLEVEMENT_ORDURES_MENAGERES.champ()); // Sans valeurs nulles
perimetres = hotEncodingAvecNull(perimetres,
REDEVANCE_ENLEVEMENT_ORDURES_MENAGERES.champ()); // Sans valeurs nulles
perimetres =
extraireCompetencesDelegueesEtInteretCommunautaire(perimetres, this.anneeCOG);
}
perimetres = perimetres.filter((FilterFunction<Row>) perimetre ->
this.validator.validerPerimetre(this.historique, perimetre, this.anneeCOG));
perimetres = ajouterCompetencesPasEncoreExistantesOuDisparues(perimetres,
this.anneeCOG);
List<Column> colonnesSelect = adaptateurAnnuel.select(perimetres,
this.datasetCompetence, this.anneeCOG);
perimetres = perimetres.select(colonnesSelect.toArray(new Column[]{}));
return perimetres;
}{code}
was:
I'm following a {{String[]}} of competencies codes, at their indexes :
{code:java}
codesCompetences = (size = 121)
0 = "C1005" 1 = "C1010" 2 = "C1015" 3 = "C1020" 4 = "C1025"
5 = "C1030" 6 = "C1035" 7 = "C1040" 8 = "C1045" 9 = "C1505"
10 = "C1510" 11 = "C1515" 12 = "C1520" 13 = "C2000" 14 = "C2005"
15 = "C2010" 16 = "C2015" 17 = "C2020" 18 = "C2025" 19 = "C2030"
20 = "C2035" 21 = "C2040" 22 = "C2045" 23 = "C2050" 24 = "C2055"
25 = "C2060" 26 = "C2065" 27 = "C2070" 28 = "C2075" 29 = "C2080"
30 = "C2085" 31 = "C2090" 32 = "C2095" 33 = "C2096" 34 = "C2500"
35 = "C2505" 36 = "C2510" 37 = "C3000" 38 = "C3005" 39 = "C3010"
40 = "C3015" 41 = "C3020" 42 = "C3025" 43 = "C3030" 44 = "C3035"
45 = "C3040" 46 = "C3045" 47 = "C3050" 48 = "C3055" 49 = "C3060"
50 = "C3500" 51 = "C4000" 52 = "C4005" 53 = "C4010" 54 = "C4500"
55 = "C4505" 56 = "C4510" 57 = "C4515" 58 = "C4520" 59 = "C5000"
60 = "C5005" 61 = "C5010" 62 = "C5015" 63 = "C5020" 64 = "C5025"
65 = "C5030" 66 = "C5035" 67 = "C5040" 68 = "C5045" 69 = "C5050"
70 = "C5500" 71 = "C5505" 72 = "C5510" 73 = "C5515" 74 = "C5520"
75 = "C5525" 76 = "C5530" 77 = "C5535" 78 = "C5540" 79 = "C6000"
80 = "C6005" 81 = "C6010" 82 = "C6015" 83 = "C6020" 84 = "C6025"
85 = "C6030" 86 = "C6035" 87 = "C6040" 88 = "C6045" 89 = "C6055"
90 = "C6060" 91 = "C6065" 92 = "C6070" 93 = "C6075" 94 = "C6500"
95 = "C6505" 96 = "C6510" 97 = "C6515" 98 = "C6520" 99 = "C6525"
100 = "C6530" 101 = "C6535" 102 = "C7000" 103 = "C7500" 104 = "C7505"
105 = "C7510" 106 = "C7515" 107 = "C7520" 108 = "C7525" 109 = "C7530"
110 = "C7535" 111 = "C7540" 112 = "C7541" 113 = "C7545" 114 = "C7546"
115 = "C7550" 116 = "C7555" 117 = "C7560" 118 = "C7565" 119 = "C7570"
120 = "C7575 {code}
With them, I'm converting values of columns coming from a CSV file:
{code:java}
private Dataset<Row>
extraireCompetencesDelegueesEtInteretCommunautaire(Dataset<Row> dataset, int
anneeCog) {
// Ces valeurs sont celles qu'un distinct() montre possibles sur les
colonnes du fichier csv d'entrée.
final String COMPETENCE_DELEGUEE = "délégué";
final String COMPETENCE_INTERET_COMMUNAUTAIRE = "avec intérêt
communautaire";
List<String> codesCompetences =
this.datasetCompetence.codesCompetencesAsList(anneeCog);
for(String codeCompetence : codesCompetences) {
Column colCompetence = new Column(codeCompetence);
// Créer le champ Cxxx_Deleguee et le one hot encoder en "1" ou "0" selon
que la valeur associée au code compétence vaut 'délégué' ou pas.
Column competenceDeleguee =
when(colCompetence.equalTo(COMPETENCE_DELEGUEE), lit("1")).otherwise(lit("0"));
dataset = dataset.withColumn(codeCompetence + "_Deleguee",
competenceDeleguee);
// Créer le champ Cxxx_InteretCommunautaire et le one hot encoder en "1"
ou "0" selon que la valeur associée au code compétence vaut 'avec intérêt
communautaire' ou pas.
Column competenceInteretCommunautaire =
when(colCompetence.equalTo(COMPETENCE_INTERET_COMMUNAUTAIRE),
lit("1")).otherwise(lit("0"));
dataset = dataset.withColumn(codeCompetence + "_InteretCommunautaire",
competenceInteretCommunautaire);
// one hot encoder la présence de la compétence
Column encoding = when(colCompetence.isin(OUI, COMPETENCE_DELEGUEE,
COMPETENCE_INTERET_COMMUNAUTAIRE), lit("1"))
.when(colCompetence.equalTo(NON), lit("0"))
.otherwise(colCompetence);
dataset = dataset.withColumn(codeCompetence, encoding);
LOGGER.info("added: '{}'", encoding);
}
return dataset;
}
{code}
This loop adds to the {{Dataset}} some columns each time,. Among them one is :
{code:java}
added: 'CASE WHEN in(C6525, 'OUI', 'délégué', 'avec intérêt communautaire')
THEN '1' WHEN =(C6525, 'NON') THEN '0' ELSE C6525 END'{code}
and it's the last log shown before a:
{code:java}
java.lang.StackOverflowError
at java.base/java.lang.ref.Reference.refersToImpl(Reference.java:384)
at java.base/java.lang.ref.Reference.refersTo(Reference.java:375)
at java.base/java.lang.ThreadLocal$ThreadLocalMap.set(ThreadLocal.java:539)
at java.base/java.lang.ThreadLocal.set(ThreadLocal.java:243)
at java.base/java.lang.ThreadLocal.set(ThreadLocal.java:229)
at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.set(origin.scala:93)
at
org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:105)
at
org.apache.spark.sql.catalyst.trees.TreeNode.transformDownWithPruning(TreeNode.scala:495)
at
org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDownWithPruning$3(TreeNode.scala:500)
at
org.apache.spark.sql.catalyst.trees.UnaryLike.mapChildren(TreeNode.scala:1268)
at
org.apache.spark.sql.catalyst.trees.UnaryLike.mapChildren$(TreeNode.scala:1267)
at
org.apache.spark.sql.catalyst.expressions.UnaryExpression.mapChildren(Expression.scala:596)
at
org.apache.spark.sql.catalyst.trees.TreeNode.transformDownWithPruning(TreeNode.scala:500)
at
org.apache.spark.sql.catalyst.plans.QueryPlan.$anonfun$transformExpressionsDownWithPruning$1(QueryPlan.scala:223)
at
org.apache.spark.sql.catalyst.plans.QueryPlan.$anonfun$mapExpressions$1(QueryPlan.scala:264)
at
org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:107)
at
org.apache.spark.sql.catalyst.plans.QueryPlan.transformExpression$1(QueryPlan.scala:264)
at
org.apache.spark.sql.catalyst.plans.QueryPlan.recursiveTransform$1(QueryPlan.scala:276)
at
org.apache.spark.sql.catalyst.plans.QueryPlan.$anonfun$mapExpressions$4(QueryPlan.scala:282)
at scala.collection.immutable.List.map(List.scala:236)
at scala.collection.immutable.List.map(List.scala:79)
at
org.apache.spark.sql.catalyst.plans.QueryPlan.recursiveTransform$1(QueryPlan.scala:282)
at
org.apache.spark.sql.catalyst.plans.QueryPlan.$anonfun$mapExpressions$5(QueryPlan.scala:287)
at
org.apache.spark.sql.catalyst.trees.TreeNode.mapProductIterator(TreeNode.scala:337)
at
org.apache.spark.sql.catalyst.plans.QueryPlan.mapExpressions(QueryPlan.scala:287)
at
org.apache.spark.sql.catalyst.plans.QueryPlan.transformExpressionsDownWithPruning(QueryPlan.scala:223)
at
org.apache.spark.sql.catalyst.plans.QueryPlan.transformExpressionsWithPruning(QueryPlan.scala:194)
at
org.apache.spark.sql.catalyst.plans.QueryPlan.transformExpressions(QueryPlan.scala:172)
at
org.apache.spark.sql.catalyst.analysis.DeduplicateRelations$.deduplicate(DeduplicateRelations.scala:240)
at
org.apache.spark.sql.catalyst.analysis.DeduplicateRelations$.deduplicateAndRenew(DeduplicateRelations.scala:323)
at
org.apache.spark.sql.catalyst.analysis.DeduplicateRelations$.org$apache$spark$sql$catalyst$analysis$DeduplicateRelations$$renewDuplicatedRelations(DeduplicateRelations.scala:126)
at
org.apache.spark.sql.catalyst.analysis.DeduplicateRelations$.$anonfun$deduplicate$1(DeduplicateRelations.scala:233)
at
org.apache.spark.sql.catalyst.analysis.DeduplicateRelations$.$anonfun$deduplicate$1$adapted(DeduplicateRelations.scala:232)
at scala.collection.immutable.Vector.foreach(Vector.scala:2125)
at
org.apache.spark.sql.catalyst.analysis.DeduplicateRelations$.deduplicate(DeduplicateRelations.scala:232)
at
org.apache.spark.sql.catalyst.analysis.DeduplicateRelations$.deduplicateAndRenew(DeduplicateRelations.scala:323)
at
org.apache.spark.sql.catalyst.analysis.DeduplicateRelations$.org$apache$spark$sql$catalyst$analysis$DeduplicateRelations$$renewDuplicatedRelations(DeduplicateRelations.scala:126)
at
org.apache.spark.sql.catalyst.analysis.DeduplicateRelations$.$anonfun$deduplicate$1(DeduplicateRelations.scala:233)
at
org.apache.spark.sql.catalyst.analysis.DeduplicateRelations$.$anonfun$deduplicate$1$adapted(DeduplicateRelations.scala:232)
[...] {code}
What I've noticed during my debugs is that:
* If I do a step over step debugging, without allowing my program to resume
from `C6525`, I can pursue through `C6530`, `C6535`... But as soon as do a
Resume program, the stack overflow happens.
* If I assign each of these new columns to {{lit("0")}}, avoiding these
{{when...otherwise}}, the StackOverflow comes after the loop, but it comes
quickly after. So eventually, it triggers all the time.
So I guess it's an helper thread that starts to work on the request being
prepared that causes the trouble.
The problem didn't exist with Spark {{4.1.1}} I was using before, with {{Java
21}} then.
Remark:
this code is included in a larger one, in the {{if (this.anneeCOG >= 2025)}
below:
{code:java}
@Override
public Dataset<Row> get() {
LOGGER.info("Constitution du dataset des périmètres de groupements
(intercommunalités seulement : {}) par {} pour l'année {}...",
this.epciSeulement, this.tri, this.anneeCOG);
this.session.sparkContext().setJobDescription("EPCIPerimetreWorker");
this.session.sparkContext().setLocalProperty("callSite.short", toString());
// Obtention des SIREN des communes d'après le COG.
Dataset<Row> sirenCommunes =
this.datasetSirenCommunaux.rowSirenCommunes(this.options, this.historique, new
SirenCommunesTriSiren(), this.anneeCOG);
Dataset<Row> perimetres = this.loader.loadOpenData(this.anneeCOG);
// Corriger les catégories de communes "Commune" en commune
perimetres = perimetres.withColumn(CATEGORIE_MEMBRES_GROUPEMENT.champ(),
when(CATEGORIE_MEMBRES_GROUPEMENT.col().equalTo(lit("Commune")),
lit("commune"))
.otherwise(CATEGORIE_MEMBRES_GROUPEMENT.col())); // Celles "Groupement"
en groupement
perimetres = perimetres.withColumn(CATEGORIE_MEMBRES_GROUPEMENT.champ(),
when(CATEGORIE_MEMBRES_GROUPEMENT.col().equalTo(lit("Groupement")),
lit("groupement"))
.otherwise(CATEGORIE_MEMBRES_GROUPEMENT.col()));
// Et celles "Autre organisme" en autre organisme
perimetres = perimetres.withColumn(CATEGORIE_MEMBRES_GROUPEMENT.champ(),
when(CATEGORIE_MEMBRES_GROUPEMENT.col().equalTo(lit("Autre organisme")),
lit("autre organisme"))
.otherwise(CATEGORIE_MEMBRES_GROUPEMENT.col())); perimetres =
perimetres.persist();
boolean versionBanatic2024etAnterieure = this.anneeCOG <= 2024;
PerimetreAdaptateurAnnuel adaptateurAnnuel = versionBanatic2024etAnterieure
? new PerimetreAdaptateurAnnuel2024EtAnterieur() : new
PerimetreAdaptateurAnnuel2025EtPosterieur();
// Si l'année porte sur une version de fichiers BANATIC ancienne,
// les champs de code portant sur les région, département, commune sont
encodés sous la forme 'code - libellé'
perimetres =
adaptateurAnnuel.extraireCodesRegionDepartementArrondissementSiren(perimetres,
this.session);
perimetres = retrouverRegion(perimetres, sirenCommunes, this.anneeCOG);
// TODO : ici, est-ce sûr que c'est POPULATION_TOTALE ?
perimetres = perimetres.withColumnRenamed(POPULATION_TOTALE.champ(),
POPULATION_GROUPEMENT.champ());
perimetres = joinSirenCommunes(perimetres);
perimetres = adaptateurAnnuel.codePostalEtVilleAdministrative(perimetres);
Column joinCommuneSiege =
SIREN_COMMUNE_SIEGE.col(perimetres).equalTo(SIREN_COMMUNE.col(sirenCommunes));
perimetres = perimetres.join(sirenCommunes, joinCommuneSiege, LEFT_OUTER)
.withColumnRenamed(CODE_COMMUNE.champ(), CODE_COMMUNE_SIEGE.champ())
.withColumnRenamed(POPULATION_TOTALE.champ(),
POPULATION_COMMUNE_SIEGE_TOTALE.champ())
.withColumnRenamed(POPULATION_MUNICIPALE.champ(),
POPULATION_COMMUNE_SIEGE_MUNICIPALE.champ())
.withColumnRenamed(POPULATION_COMPTEE_A_PART.champ(),
POPULATION_COMMUNE_SIEGE_COMPTEE_A_PART.champ())
.drop(NOM_REGION.champ())
.drop(CODE_REGION.champ())
.drop(CODE_DEPARTEMENT.champ())
.drop(NOM_DEPARTEMENT.champ())
.drop(SIREN_COMMUNE.champ());
perimetres = adaptateurAnnuel.codePostalEtVilleSiege(perimetres);
// FIXME En 2019, Choisy-le-Roi est inscrit comme EPT lié à la fois à
Grand-Orly Seine, ce qui est exact, mais aussi d'Est-Ensemble comme EPT, ce qui
n'est pas vrai.
Column eptChoisyLeRoiConfusion =
NATURE_JURIDIQUE.col().equalTo("EPT").and(SIREN_GROUPEMENT.col(perimetres).equalTo("200057875"));
if (this.epciSeulement) {
perimetres = perimetres.filter(NATURE_JURIDIQUE.col().isin("CA", "CC",
"CU", "EPT", "METRO", "MET69")
.and(not(eptChoisyLeRoiConfusion)));
}
if (this.anneeCOG >= 2025) {
perimetres = hotEncodingAvecNull(perimetres,
SYNDICAT_A_LA_CARTE.champ()); // Contient des valeurs nulles
perimetres = hotEncodingAvecNull(perimetres, INTERDEPARTEMENTAL.champ());
// Contient des valeurs nulles
perimetres = hotEncodingAvecNull(perimetres, ZONE_DE_MONTAGNE.champ());
// Contient des valeurs nulles
perimetres = hotEncodingAvecNull(perimetres, GESTION_DES_EAUX.champ());
// Contient des valeurs nulles
perimetres = hotEncodingAvecNull(perimetres,
ETABLISSEMENT_PUBLIC_TERRITORIAL_DE_BASSIN.champ()); // Contient des valeurs
nulles
perimetres = hotEncodingAvecNull(perimetres,
TAXE_ENLEVEMENT_ORDURES_MENAGERES.champ()); // Sans valeurs nulles
perimetres = hotEncodingAvecNull(perimetres,
REDEVANCE_ENLEVEMENT_ORDURES_MENAGERES.champ()); // Sans valeurs nulles
perimetres =
extraireCompetencesDelegueesEtInteretCommunautaire(perimetres, this.anneeCOG);
}
perimetres = perimetres.filter((FilterFunction<Row>) perimetre ->
this.validator.validerPerimetre(this.historique, perimetre, this.anneeCOG));
perimetres = ajouterCompetencesPasEncoreExistantesOuDisparues(perimetres,
this.anneeCOG);
List<Column> colonnesSelect = adaptateurAnnuel.select(perimetres,
this.datasetCompetence, this.anneeCOG);
perimetres = perimetres.select(colonnesSelect.toArray(new Column[]{}));
return perimetres;
}{code}
> SQL Catalyst 4.2.0 causes a StackOverflow error when it attempts to optimize
> a request it is preparing
> ------------------------------------------------------------------------------------------------------
>
> Key: SPARK-58318
> URL: https://issues.apache.org/jira/browse/SPARK-58318
> Project: Spark
> Issue Type: Bug
> Components: SQL
> Affects Versions: 4.2.0
> Environment: Debian 13
> Spark 4.2.0
> Java 25
> Reporter: Marc Le Bihan
> Priority: Major
>
> I'm following a {{String[]}} of competencies codes, at their indexes :
> {code:java}
> codesCompetences = (size = 121)
> 0 = "C1005" 1 = "C1010" 2 = "C1015" 3 = "C1020" 4 = "C1025"
> 5 = "C1030" 6 = "C1035" 7 = "C1040" 8 = "C1045" 9 = "C1505"
> 10 = "C1510" 11 = "C1515" 12 = "C1520" 13 = "C2000" 14 = "C2005"
> 15 = "C2010" 16 = "C2015" 17 = "C2020" 18 = "C2025" 19 = "C2030"
> 20 = "C2035" 21 = "C2040" 22 = "C2045" 23 = "C2050" 24 = "C2055"
> 25 = "C2060" 26 = "C2065" 27 = "C2070" 28 = "C2075" 29 = "C2080"
> 30 = "C2085" 31 = "C2090" 32 = "C2095" 33 = "C2096" 34 = "C2500"
> 35 = "C2505" 36 = "C2510" 37 = "C3000" 38 = "C3005" 39 = "C3010"
> 40 = "C3015" 41 = "C3020" 42 = "C3025" 43 = "C3030" 44 = "C3035"
> 45 = "C3040" 46 = "C3045" 47 = "C3050" 48 = "C3055" 49 = "C3060"
> 50 = "C3500" 51 = "C4000" 52 = "C4005" 53 = "C4010" 54 = "C4500"
> 55 = "C4505" 56 = "C4510" 57 = "C4515" 58 = "C4520" 59 = "C5000"
> 60 = "C5005" 61 = "C5010" 62 = "C5015" 63 = "C5020" 64 = "C5025"
> 65 = "C5030" 66 = "C5035" 67 = "C5040" 68 = "C5045" 69 = "C5050"
> 70 = "C5500" 71 = "C5505" 72 = "C5510" 73 = "C5515" 74 = "C5520"
> 75 = "C5525" 76 = "C5530" 77 = "C5535" 78 = "C5540" 79 = "C6000"
> 80 = "C6005" 81 = "C6010" 82 = "C6015" 83 = "C6020" 84 = "C6025"
> 85 = "C6030" 86 = "C6035" 87 = "C6040" 88 = "C6045" 89 = "C6055"
> 90 = "C6060" 91 = "C6065" 92 = "C6070" 93 = "C6075" 94 = "C6500"
> 95 = "C6505" 96 = "C6510" 97 = "C6515" 98 = "C6520" 99 = "C6525"
> 100 = "C6530" 101 = "C6535" 102 = "C7000" 103 = "C7500" 104 = "C7505"
> 105 = "C7510" 106 = "C7515" 107 = "C7520" 108 = "C7525" 109 = "C7530"
> 110 = "C7535" 111 = "C7540" 112 = "C7541" 113 = "C7545" 114 = "C7546"
> 115 = "C7550" 116 = "C7555" 117 = "C7560" 118 = "C7565" 119 = "C7570"
> 120 = "C7575 {code}
> With them, I'm converting values of columns coming from a CSV file:
> {code:java}
> private Dataset<Row>
> extraireCompetencesDelegueesEtInteretCommunautaire(Dataset<Row> dataset, int
> anneeCog) {
> // Ces valeurs sont celles qu'un distinct() montre possibles sur les
> colonnes du fichier csv d'entrée.
> final String COMPETENCE_DELEGUEE = "délégué";
> final String COMPETENCE_INTERET_COMMUNAUTAIRE = "avec intérêt
> communautaire";
> List<String> codesCompetences =
> this.datasetCompetence.codesCompetencesAsList(anneeCog);
> for(String codeCompetence : codesCompetences) {
> Column colCompetence = new Column(codeCompetence);
> // Créer le champ Cxxx_Deleguee et le one hot encoder en "1" ou "0"
> selon que la valeur associée au code compétence vaut 'délégué' ou pas.
> Column competenceDeleguee =
> when(colCompetence.equalTo(COMPETENCE_DELEGUEE),
> lit("1")).otherwise(lit("0"));
> dataset = dataset.withColumn(codeCompetence + "_Deleguee",
> competenceDeleguee);
> // Créer le champ Cxxx_InteretCommunautaire et le one hot encoder en
> "1" ou "0" selon que la valeur associée au code compétence vaut 'avec intérêt
> communautaire' ou pas.
> Column competenceInteretCommunautaire =
> when(colCompetence.equalTo(COMPETENCE_INTERET_COMMUNAUTAIRE),
> lit("1")).otherwise(lit("0"));
> dataset = dataset.withColumn(codeCompetence + "_InteretCommunautaire",
> competenceInteretCommunautaire);
> // one hot encoder la présence de la compétence
> Column encoding = when(colCompetence.isin(OUI, COMPETENCE_DELEGUEE,
> COMPETENCE_INTERET_COMMUNAUTAIRE), lit("1"))
> .when(colCompetence.equalTo(NON), lit("0"))
> .otherwise(colCompetence);
> dataset = dataset.withColumn(codeCompetence, encoding);
> LOGGER.info("added: '{}'", encoding);
> }
> return dataset;
> }
> {code}
> This loop adds to the {{Dataset}} some columns each time,. Among them one is :
> {code:java}
> added: 'CASE WHEN in(C6525, 'OUI', 'délégué', 'avec intérêt communautaire')
> THEN '1' WHEN =(C6525, 'NON') THEN '0' ELSE C6525 END'{code}
> and it's the last log shown before a:
> {code:java}
> java.lang.StackOverflowError
> at java.base/java.lang.ref.Reference.refersToImpl(Reference.java:384)
> at java.base/java.lang.ref.Reference.refersTo(Reference.java:375)
> at
> java.base/java.lang.ThreadLocal$ThreadLocalMap.set(ThreadLocal.java:539)
> at java.base/java.lang.ThreadLocal.set(ThreadLocal.java:243)
> at java.base/java.lang.ThreadLocal.set(ThreadLocal.java:229)
> at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.set(origin.scala:93)
> at
> org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:105)
> at
> org.apache.spark.sql.catalyst.trees.TreeNode.transformDownWithPruning(TreeNode.scala:495)
> at
> org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDownWithPruning$3(TreeNode.scala:500)
> at
> org.apache.spark.sql.catalyst.trees.UnaryLike.mapChildren(TreeNode.scala:1268)
> at
> org.apache.spark.sql.catalyst.trees.UnaryLike.mapChildren$(TreeNode.scala:1267)
> at
> org.apache.spark.sql.catalyst.expressions.UnaryExpression.mapChildren(Expression.scala:596)
> at
> org.apache.spark.sql.catalyst.trees.TreeNode.transformDownWithPruning(TreeNode.scala:500)
> at
> org.apache.spark.sql.catalyst.plans.QueryPlan.$anonfun$transformExpressionsDownWithPruning$1(QueryPlan.scala:223)
> at
> org.apache.spark.sql.catalyst.plans.QueryPlan.$anonfun$mapExpressions$1(QueryPlan.scala:264)
> at
> org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:107)
> at
> org.apache.spark.sql.catalyst.plans.QueryPlan.transformExpression$1(QueryPlan.scala:264)
> at
> org.apache.spark.sql.catalyst.plans.QueryPlan.recursiveTransform$1(QueryPlan.scala:276)
> at
> org.apache.spark.sql.catalyst.plans.QueryPlan.$anonfun$mapExpressions$4(QueryPlan.scala:282)
> at scala.collection.immutable.List.map(List.scala:236)
> at scala.collection.immutable.List.map(List.scala:79)
> at
> org.apache.spark.sql.catalyst.plans.QueryPlan.recursiveTransform$1(QueryPlan.scala:282)
> at
> org.apache.spark.sql.catalyst.plans.QueryPlan.$anonfun$mapExpressions$5(QueryPlan.scala:287)
> at
> org.apache.spark.sql.catalyst.trees.TreeNode.mapProductIterator(TreeNode.scala:337)
> at
> org.apache.spark.sql.catalyst.plans.QueryPlan.mapExpressions(QueryPlan.scala:287)
> at
> org.apache.spark.sql.catalyst.plans.QueryPlan.transformExpressionsDownWithPruning(QueryPlan.scala:223)
> at
> org.apache.spark.sql.catalyst.plans.QueryPlan.transformExpressionsWithPruning(QueryPlan.scala:194)
> at
> org.apache.spark.sql.catalyst.plans.QueryPlan.transformExpressions(QueryPlan.scala:172)
> at
> org.apache.spark.sql.catalyst.analysis.DeduplicateRelations$.deduplicate(DeduplicateRelations.scala:240)
> at
> org.apache.spark.sql.catalyst.analysis.DeduplicateRelations$.deduplicateAndRenew(DeduplicateRelations.scala:323)
> at
> org.apache.spark.sql.catalyst.analysis.DeduplicateRelations$.org$apache$spark$sql$catalyst$analysis$DeduplicateRelations$$renewDuplicatedRelations(DeduplicateRelations.scala:126)
> at
> org.apache.spark.sql.catalyst.analysis.DeduplicateRelations$.$anonfun$deduplicate$1(DeduplicateRelations.scala:233)
> at
> org.apache.spark.sql.catalyst.analysis.DeduplicateRelations$.$anonfun$deduplicate$1$adapted(DeduplicateRelations.scala:232)
> at scala.collection.immutable.Vector.foreach(Vector.scala:2125)
> at
> org.apache.spark.sql.catalyst.analysis.DeduplicateRelations$.deduplicate(DeduplicateRelations.scala:232)
> at
> org.apache.spark.sql.catalyst.analysis.DeduplicateRelations$.deduplicateAndRenew(DeduplicateRelations.scala:323)
> at
> org.apache.spark.sql.catalyst.analysis.DeduplicateRelations$.org$apache$spark$sql$catalyst$analysis$DeduplicateRelations$$renewDuplicatedRelations(DeduplicateRelations.scala:126)
> at
> org.apache.spark.sql.catalyst.analysis.DeduplicateRelations$.$anonfun$deduplicate$1(DeduplicateRelations.scala:233)
> at
> org.apache.spark.sql.catalyst.analysis.DeduplicateRelations$.$anonfun$deduplicate$1$adapted(DeduplicateRelations.scala:232)
> [...] {code}
> What I've noticed during my debugs is that:
>
> * If I do a step over step debugging, without allowing my program to resume
> from `C6525`, I can pursue through `C6530`, `C6535`... But as soon as do a
> Resume program, the stack overflow happens.
> So I guess it's an helper thread that starts to work on the request being
> prepared that causes the trouble.
> The problem didn't exist with Spark {{4.1.1}} I was using before, with {{Java
> 21}} then.
> Remark:
> this code is included in a larger one, in the {{if (this.anneeCOG >= 2025)}
> below:
> {code:java}
> @Override
> public Dataset<Row> get() {
> LOGGER.info("Constitution du dataset des périmètres de groupements
> (intercommunalités seulement : {}) par {} pour l'année {}...",
> this.epciSeulement, this.tri, this.anneeCOG);
> this.session.sparkContext().setJobDescription("EPCIPerimetreWorker");
> this.session.sparkContext().setLocalProperty("callSite.short",
> toString());
> // Obtention des SIREN des communes d'après le COG.
> Dataset<Row> sirenCommunes =
> this.datasetSirenCommunaux.rowSirenCommunes(this.options, this.historique,
> new SirenCommunesTriSiren(), this.anneeCOG);
> Dataset<Row> perimetres = this.loader.loadOpenData(this.anneeCOG);
> // Corriger les catégories de communes "Commune" en commune
> perimetres = perimetres.withColumn(CATEGORIE_MEMBRES_GROUPEMENT.champ(),
> when(CATEGORIE_MEMBRES_GROUPEMENT.col().equalTo(lit("Commune")),
> lit("commune"))
> .otherwise(CATEGORIE_MEMBRES_GROUPEMENT.col())); // Celles
> "Groupement" en groupement
> perimetres = perimetres.withColumn(CATEGORIE_MEMBRES_GROUPEMENT.champ(),
> when(CATEGORIE_MEMBRES_GROUPEMENT.col().equalTo(lit("Groupement")),
> lit("groupement"))
> .otherwise(CATEGORIE_MEMBRES_GROUPEMENT.col()));
> // Et celles "Autre organisme" en autre organisme
> perimetres = perimetres.withColumn(CATEGORIE_MEMBRES_GROUPEMENT.champ(),
> when(CATEGORIE_MEMBRES_GROUPEMENT.col().equalTo(lit("Autre
> organisme")), lit("autre organisme"))
> .otherwise(CATEGORIE_MEMBRES_GROUPEMENT.col())); perimetres =
> perimetres.persist();
> boolean versionBanatic2024etAnterieure = this.anneeCOG <= 2024;
> PerimetreAdaptateurAnnuel adaptateurAnnuel =
> versionBanatic2024etAnterieure ? new
> PerimetreAdaptateurAnnuel2024EtAnterieur() : new
> PerimetreAdaptateurAnnuel2025EtPosterieur();
> // Si l'année porte sur une version de fichiers BANATIC ancienne,
> // les champs de code portant sur les région, département, commune sont
> encodés sous la forme 'code - libellé'
> perimetres =
> adaptateurAnnuel.extraireCodesRegionDepartementArrondissementSiren(perimetres,
> this.session);
> perimetres = retrouverRegion(perimetres, sirenCommunes, this.anneeCOG);
> // TODO : ici, est-ce sûr que c'est POPULATION_TOTALE ?
> perimetres = perimetres.withColumnRenamed(POPULATION_TOTALE.champ(),
> POPULATION_GROUPEMENT.champ());
> perimetres = joinSirenCommunes(perimetres);
> perimetres = adaptateurAnnuel.codePostalEtVilleAdministrative(perimetres);
>
> Column joinCommuneSiege =
> SIREN_COMMUNE_SIEGE.col(perimetres).equalTo(SIREN_COMMUNE.col(sirenCommunes));
> perimetres = perimetres.join(sirenCommunes, joinCommuneSiege, LEFT_OUTER)
> .withColumnRenamed(CODE_COMMUNE.champ(), CODE_COMMUNE_SIEGE.champ())
> .withColumnRenamed(POPULATION_TOTALE.champ(),
> POPULATION_COMMUNE_SIEGE_TOTALE.champ())
> .withColumnRenamed(POPULATION_MUNICIPALE.champ(),
> POPULATION_COMMUNE_SIEGE_MUNICIPALE.champ())
> .withColumnRenamed(POPULATION_COMPTEE_A_PART.champ(),
> POPULATION_COMMUNE_SIEGE_COMPTEE_A_PART.champ())
> .drop(NOM_REGION.champ())
> .drop(CODE_REGION.champ())
> .drop(CODE_DEPARTEMENT.champ())
> .drop(NOM_DEPARTEMENT.champ())
> .drop(SIREN_COMMUNE.champ());
> perimetres = adaptateurAnnuel.codePostalEtVilleSiege(perimetres);
> // FIXME En 2019, Choisy-le-Roi est inscrit comme EPT lié à la fois à
> Grand-Orly Seine, ce qui est exact, mais aussi d'Est-Ensemble comme EPT, ce
> qui n'est pas vrai.
> Column eptChoisyLeRoiConfusion =
> NATURE_JURIDIQUE.col().equalTo("EPT").and(SIREN_GROUPEMENT.col(perimetres).equalTo("200057875"));
>
> if (this.epciSeulement) {
> perimetres = perimetres.filter(NATURE_JURIDIQUE.col().isin("CA", "CC",
> "CU", "EPT", "METRO", "MET69")
> .and(not(eptChoisyLeRoiConfusion)));
> }
> if (this.anneeCOG >= 2025) {
> perimetres = hotEncodingAvecNull(perimetres,
> SYNDICAT_A_LA_CARTE.champ()); // Contient des valeurs nulles
> perimetres = hotEncodingAvecNull(perimetres,
> INTERDEPARTEMENTAL.champ()); // Contient des valeurs nulles
> perimetres = hotEncodingAvecNull(perimetres, ZONE_DE_MONTAGNE.champ());
> // Contient des valeurs nulles
> perimetres = hotEncodingAvecNull(perimetres, GESTION_DES_EAUX.champ());
> // Contient des valeurs nulles
> perimetres = hotEncodingAvecNull(perimetres,
> ETABLISSEMENT_PUBLIC_TERRITORIAL_DE_BASSIN.champ()); // Contient des valeurs
> nulles
> perimetres = hotEncodingAvecNull(perimetres,
> TAXE_ENLEVEMENT_ORDURES_MENAGERES.champ()); // Sans valeurs nulles
> perimetres = hotEncodingAvecNull(perimetres,
> REDEVANCE_ENLEVEMENT_ORDURES_MENAGERES.champ()); // Sans valeurs nulles
> perimetres =
> extraireCompetencesDelegueesEtInteretCommunautaire(perimetres, this.anneeCOG);
> }
> perimetres = perimetres.filter((FilterFunction<Row>) perimetre ->
> this.validator.validerPerimetre(this.historique, perimetre, this.anneeCOG));
>
> perimetres = ajouterCompetencesPasEncoreExistantesOuDisparues(perimetres,
> this.anneeCOG);
> List<Column> colonnesSelect = adaptateurAnnuel.select(perimetres,
> this.datasetCompetence, this.anneeCOG);
> perimetres = perimetres.select(colonnesSelect.toArray(new Column[]{}));
> return perimetres;
> }{code}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
