scala json fastjson json4s

 

最近有個spark任務涉及到scala操做json，大概流程是這樣：從hbase取數據，每條數據先parse json，而後刪除一個多餘的key，最後在弄成json字符串，輸出到hdfs。

json大概長這樣，{「@type」:{"version":"1.0.2","name":"application-content","data":[]},"key-to-remove":[{"blah":"more blah"}],"@value":[]}

邏輯不復雜，讀取hbase的部分在此略去，json相關代碼以下，用fastjson解析json

package dev.json

import com.alibaba.fastjson.JSON

object Course1 {
  def main(args: Array[String]): Unit = {
    val key = "key-to-remove"
val s =
      """
        |{"@type":{"version":"1.0.2","name":"application-content","data":[]},"key-to-remove":[{"blah":"more blah"}],"@value":[]}
        |""".stripMargin
    val obj = JSON.parseObject(s)
    obj.remove(key)
    val out = obj.toJSONString
    println(out)
  }
}

而後就是一頓報錯

Exception in thread "main" com.alibaba.fastjson.JSONException: expect ':' at 2, actual "

at com.alibaba.fastjson.parser.DefaultJSONParser.parseObject(DefaultJSONParser.java:296)

at com.alibaba.fastjson.parser.DefaultJSONParser.parse(DefaultJSONParser.java:1401)

at com.alibaba.fastjson.parser.DefaultJSONParser.parse(DefaultJSONParser.java:1367)

at com.alibaba.fastjson.JSON.parse(JSON.java:183)

at com.alibaba.fastjson.JSON.parse(JSON.java:193)

at com.alibaba.fastjson.JSON.parse(JSON.java:149)

at com.alibaba.fastjson.JSON.parseObject(JSON.java:254)

at dev.json.Course1$.main(Course1.scala:12)

at dev.json.Course1.main(Course1.scala)

之前用fastjson歷來沒碰到這樣的問題，一頓百度，而後才知道是裏面包含了@type的key，autotype is not supported，阿里出於安全考慮，@type容易注入一些不安全操做，因此拋出錯誤。查了一些資料，總算是修復了，須要加上一些選項，把所在包添加白名單，從而關閉對@type的解析。代碼以下：

package dev.json

import com.alibaba.fastjson.JSON
import com.alibaba.fastjson.parser.{Feature, ParserConfig}

object Course1 {
  // 添加包白名單
ParserConfig.getGlobalInstance.addAccept("dev.json")

  def main(args: Array[String]): Unit = {
    val key = "key-to-remove"
val s =
      """
        |{"@type":{"version":"1.0.2","name":"application-content","data":[]},"key-to-remove":[{"blah":"more blah"}],"@value":[]}
        |""".stripMargin
    // 關閉特殊key檢查
val obj = JSON.parseObject(s, Feature.DisableSpecialKeyDetect)
    obj.remove(key)
    val out = obj.toJSONString
    println(out)
  }
}

而後結果就能夠正常解析，輸出以下：

{"@value":[],"@type":{"data":[],"name":"application-content","version":"1.0.2"}}

 

此次報錯，加上前段時間由於fastjson漏洞事件，公司要求緊急升級fastjson版本，瞬間對fastjson印象不那麼好了，說不定哪天就全面禁止在項目中使用fastjson。因此順便嘗試了jackson的scala版本，json4s。json4s使用起來也不那麼順手，但也夠用。上面的功能，用json4s重寫了一個版本。

package dev.json

import org.json4s.DefaultFormats
import org.json4s.JsonDSL._
import org.json4s.jackson.JsonMethods._

object Course2 {

  implicit val formats = DefaultFormats

  def main(args: Array[String]): Unit = {
    val key = "key-to-remove"
val s =
      """
        |{"@type":{"version":"1.0.2","name":"application-content","data":[]},"key-to-remove":[{"blah":"more blah"}],"@value":[]}
        |""".stripMargin
    val obj = parse(s)
    if (null != obj) {
      val obj2 = obj.removeField(_._1.equals(key))
      val out = compact(render(obj2))
      println(out)    }  }}