Spark History MCP Server

from pyspark.sql import SparkSession import pyspark.sql.functions as F def main(): """ FIXED: Proper caching with unpersist. This job properly manages cached DataFrames. """ spark = SparkSession.builder \ .appName("CachingFixedJob") \ .config("spark.executor.memory", "1g") \ .getOrCreate() # Create base DataFrame df = spark.range(100000).toDF("id") # FIXED: Properly unpersist after use for i in range(10): temp_df = df.withColumn(f"col_{i}", F.rand() * 100) temp_df.cache() temp_df.count() # Force caching # FIXED: Unpersist when done temp_df.unpersist() # FIXED: Don't cache tiny DataFrames small_df = spark.range(10).toDF("id") # No caching for small data small_df.show() spark.stop() if __name__ == "__main__": main()