spark性能調優之RDD緩存模型

/**
   * Mark this RDD for persisting using the specified level.
   *
   * @param newLevel the target storage level
   * @param allowOverride whether to override any existing level with the new one
   * 這個方法適用於以前咱們設置過了RDD的緩存級別，如今想要修改RDD的緩存級別的狀況，只須要把allowOverride設置爲true
   */
  private def persist(newLevel: StorageLevel, allowOverride: Boolean): this.type = {
    // TODO: Handle changes of StorageLevel
    //這一段程序也解釋了上面第二種方法的第一個特殊狀況爲何不會報錯
    if (storageLevel != StorageLevel.NONE && newLevel != storageLevel && !allowOverride) {
      throw new UnsupportedOperationException(
        "Cannot change storage level of an RDD after it was already assigned a level")
    }
    // If this is the first time this RDD is marked for persisting, register it
    // with the SparkContext for cleanups and accounting. Do this only once.
    if (storageLevel == StorageLevel.NONE) {
      sc.cleaner.foreach(_.registerRDDForCleanup(this))
      sc.persistRDD(this)
    }
    storageLevel = newLevel
    this
  }

 

StorageLevel這個類裏面設置了RDD的各類緩存級別，總共有12種，實際上是它的多個構造參數的組合造成的，先看一下它的相關構造參數，源碼以下：

@DeveloperApi
class StorageLevel private(
    private var _useDisk: Boolean,    //是否使用磁盤
    private var _useMemory: Boolean,  //是否使用內存
    private var _useOffHeap: Boolean, //是否使用堆外內存
    private var _deserialized: Boolean, //是否反序列化
    private var _replication: Int = 1)  //備份因子，默認爲1
  extends Externalizable {

 

 

而它設定的12種級別就是這些參數取值的組合：

object StorageLevel {
  val NONE = new StorageLevel(false, false, false, false)
  val DISK_ONLY = new StorageLevel(true, false, false, false)
  val DISK_ONLY_2 = new StorageLevel(true, false, false, false, 2)
  val MEMORY_ONLY = new StorageLevel(false, true, false, true)
  val MEMORY_ONLY_2 = new StorageLevel(false, true, false, true, 2)
  val MEMORY_ONLY_SER = new StorageLevel(false, true, false, false)
  val MEMORY_ONLY_SER_2 = new StorageLevel(false, true, false, false, 2)
  val MEMORY_AND_DISK = new StorageLevel(true, true, false, true)
  val MEMORY_AND_DISK_2 = new StorageLevel(true, true, false, true, 2)
  val MEMORY_AND_DISK_SER = new StorageLevel(true, true, false, false)
  val MEMORY_AND_DISK_SER_2 = new StorageLevel(true, true, false, false, 2)
  val OFF_HEAP = new StorageLevel(true, true, true, false, 1)