Spark History MCP Server

{ "app_id": "application_1768524288842_0024", "skew_analysis": [], "spill_analysis": [], "resource_analysis": [], "partitioning_analysis": [], "join_analysis": [], "recommendations": [ { "category": "Configuration", "issue": "Several stages (4, 12) exhibit noticeable GC times, suggesting memory pressure. Increasing executor memory can alleviate this. Given the small initial size, doubling it is a reasonable starting point. Monitor GC behavior after the change.", "suggestion": "Set spark.executor.memory to 2g", "evidence": "Current: 1g", "impact_level": "High" }, { "category": "Configuration", "issue": "Stage 3 has only 1 task, which took an exceptionally long time (691ms). This indicates poor parallelism. Since it depends on the size of the data being processed in that stage, a higher default parallelism increases initial distribution.", "suggestion": "Set spark.default.parallelism to 200", "evidence": "Current: Not explicitly set, defaults to number of cores on the cluster", "impact_level": "High" }, { "category": "Configuration", "issue": "Stage 7 also had a single task with shuffle read. Increasing shuffle partitions will divide the data into smaller chunks and allow greater parallelism during shuffle operations. Match with `spark.default.parallelism`.", "suggestion": "Set spark.sql.shuffle.partitions to 200", "evidence": "Current: Not explicitly set, defaults to spark.default.parallelism", "impact_level": "High" }, { "category": "Configuration", "issue": "Stage 12 exhibited serialization time. Kryo serialization is often faster and more compact than Java serialization, improving performance and reducing memory usage. This is especially beneficial when caching or shuffling data.", "suggestion": "Set spark.serializer to org.apache.spark.serializer.KryoSerializer", "evidence": "Current: JavaSerializer", "impact_level": "High" }, { "category": "Configuration", "issue": "If KryoSerializer is enabled, increase its maximum buffer size. This prevents 'too large frame' exceptions and improves serialization efficiency, especially when dealing with large objects.", "suggestion": "Set spark.kryoserializer.buffer.max to 256m", "evidence": "Current: Not explicitly set, defaults to 64k", "impact_level": "High" }, { "category": "Configuration", "issue": "Add `-XX:+UseG1GC -XX:InitiatingHeapOccupancyPercent=35` to driver and executor extraJavaOptions. G1GC is generally more efficient for larger heaps and reduces pause times. Setting InitiatingHeapOccupancyPercent proactively triggers GC when the heap is 35% full, preventing larger, more disruptive GC pauses.", "suggestion": "Set spark.driver.extraJavaOptions to -Djava.net.preferIPv6Addresses=false -XX:+UseG1GC -XX:InitiatingHeapOccupancyPercent=35 -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false", "evidence": "Current: -Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false", "impact_level": "High" }, { "category": "Configuration", "issue": "Add `-XX:+UseG1GC -XX:InitiatingHeapOccupancyPercent=35` to driver and executor extraJavaOptions. G1GC is generally more efficient for larger heaps and reduces pause times. Setting InitiatingHeapOccupancyPercent proactively triggers GC when the heap is 35% full, preventing larger, more disruptive GC pauses.", "suggestion": "Set spark.executor.extraJavaOptions to -Djava.net.preferIPv6Addresses=false -XX:+UseG1GC -XX:InitiatingHeapOccupancyPercent=35 -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false", "evidence": "Current: -Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false", "impact_level": "High" }, { "category": "Code", "issue": "Caching inside a loop without unpersisting leads to memory leaks. Each iteration caches a new DataFrame, accumulating memory usage and potentially causing OOM errors.", "suggestion": "Unpersist `temp_df` at the end of each loop iteration using `temp_df.unpersist()` or `temp_df.unpersist(blocking=True)` to ensure immediate release of memory. Alternatively, consider if caching inside the loop is truly necessary; often, intermediate results can be recomputed more efficiently.", "evidence": "Line: 16", "impact_level": "Medium" }, { "category": "Code", "issue": "The `temp_df.count()` action forces caching but doesn't utilize the cached data subsequently. It only contributes to the memory leak.", "suggestion": "Remove `temp_df.count()` if the result is not used. If an action is needed, ensure the cached data is meaningfully used later in the code.", "evidence": "Line: 18", "impact_level": "Medium" }, { "category": "Code", "issue": "Caching a small DataFrame like `small_df` can introduce more overhead than benefit. The caching mechanism itself consumes resources.", "suggestion": "Remove the `.cache()` call for `small_df`. For DataFrames of this size, it's generally more efficient to recompute when needed.", "evidence": "Line: 22", "impact_level": "Medium" } ] }