房價在手，天下我有 --反手就擼一個爬蟲(終)

時間 2019-11-17

原文原文鏈接

接上篇，科科，好，咱們繼續前端

咱們在這裏先把json數據入庫吧～

首先，database/scheme裏定義好數據類型。node

const mongoose = require('mongoose')

const detailHouseSchema = new mongoose.Schema({ //定義數據模式
  link:String,
  text:String,
  _id:String,
  areaDetail:[
    {
      link: String,
      text: String,
      _id: String,
      house:[
        {
          name: String,
          huxing: String,
          favorPos: String,
          aroundPrice: Number,
          adress: String,
          area: String 
        }
      ]
    }
  ]
})

mongoose.model('detailHouse',detailHouseSchema)

複製代碼

而後咱們須要到中間件裏去創建鏈接數據庫和執行插入的動做。git

middleWares/database.js數據庫

import mongoose from 'mongoose'
import config from '../config'
import fs from 'fs'
import { resolve } from 'path'

const r = path => resolve(__dirname,path) //將路徑片斷轉成一個絕對路徑
const models = r('../database/schema')
/**
 * 依次引入本地爬去好的json文件，插入數據庫
 */
var areaJson = require('database/json/AreaDetail.json')
var areaHouseJson = require('database/json/AreaHouse.json')
var detailHouseJson = require('database/json/detailHouse.json')

/**
 * 依次引入schema
 */
fs.readdirSync(models) //讀取文件
    .filter(file => ~file.search(/^[^\.].*js$/)) //篩選出後綴是js的文件
    .forEach(file => require(resolve(models,file))) 

export const database = app =>{
    mongoose.set('debug',true)
    mongoose.connect(config.db)
    mongoose.connection.on('disconnected', ()=>{
        mongoose.connect(config.db)
    })
    mongoose.connection.on('error', err =>{
        console.log(err)
    })
    mongoose.connection.on('open', async () =>{
        console.log('connected to MongoDb',config.db)
        /**
         * 杭州主城區數據入庫
         */
        let area = mongoose.model('area')
        let areaDataBase = await area.find({}).exec()
        if (!areaDataBase.length) area.insertMany(areaJson)
        /**
         * 杭州主城區的房價數據入庫
         */
        let areaHouse = mongoose.model('areaHouse')
        let areaHouseDataBase = await areaHouse.find({}).exec()
        if(!areaHouseDataBase.length) areaHouse.insertMany(areaHouseJson)

        /**
         * 杭州主城區裏包括了分區的房價數據入庫
         */
        let detailHouse = mongoose.model('detailHouse')
        let detailHouseDataBase = await detailHouse.find({}).exec()
        if(!detailHouseDataBase.length) detailHouse.insertMany(detailHouseJson)

    })
}
複製代碼

成功的話，以下圖～ bling～～～ json

走到這裏，咱們要停下來對後端的路由作一個提取個封裝。首先，我這項目頁面量不大，若是單純的用koa-router去原生去寫是沒有問題的，可是若是你是實際的項目，路由不少，這個時候再去那麼寫，代碼的可讀性就不好了。後端

Decorator能夠動態地給一個對象添加額外的職責。雖然,利用子類繼承也能夠實現這樣的功能,可是Decorator提供了一個更靈活的方式。由於繼承會爲類型引入的靜態特質,使得這種擴展方式缺少靈活性;而且隨着子類的增多(擴展功能的增多),各類子類的組合(擴展功能的組合)會致使更多子類的膨脹。api

那麼咱們要在decorator/router.js裏要定義一些公用的方法，其中還添加了打印日誌功能，在調試的時候也是美滋滋的一匹。bash

先去middlewares/routers/router.js裏去調用咱們用修飾起封裝好的方法和Route。app

import  Route  from '../decorator/router'
import { resolve } from 'path'

const r = path => resolve(__dirname, path)

export const router = app => {
  const apiPath = r('../routes')
  /**
   * 路由分離
   */
  const router = new Route(app, apiPath)

  router.init()
}
複製代碼

如今去封裝Routekoa

decorator／router.js

import Router from 'koa-router'
import { resolve } from 'path'
import _ from 'lodash'
import { glob } from 'glob' //用正則去匹配文件

export let routesMap = new Map()
export const symbolPrefix = Symbol('prefix')
export const normalizePath = path => path.startsWith('/') ? path : `/${path}`
export const isArray = c => _.isArray(c) ? c : [c]

export default class Route{
    constructor(app,apipath){
        this.app = app
        this.router = new Router()
        this.apipath = apipath
    }
    init(){
        /**
         * 這裏利用傳進來的apipath去引入後綴爲js的文件
         */
        glob
          .sync(resolve(this.apipath,'./*.js'))
          .forEach(require);
        
        for(let [ conf , controller ] of routesMap){
            /*
            *思路就是把每個路由文件的controller拎出來
            * 而後跟它的路由作一個一一匹配
            * */
            const controllers = isArray(controller) 
            let prefixPath = conf.target[symbolPrefix]
            if(prefixPath) prefixPath = normalizePath(prefixPath)

            const routerPath = prefixPath + conf.path
            this.router[conf.method](routerPath,...controllers) //function (name, path, middlewares)
        }
        this.app.use(this.router.routes()) // 添加路由中間件
        this.app.use(this.router.allowedMethods()) // 對請求進行一些限制處理
    }
}

/**
 * 
 * @param {path,target} 
 * 保證每個controller都是獨一無二的 
 */
export const controller = path => target => target.prototype[symbolPrefix] = path

/**
 * 
 * @param {conf} 
 * 定義簡單的route 
 */
export const route = conf => (target, key, desc) =>{
    conf.path = normalizePath(conf.path)
    routesMap.set({
        target:target,
        ...conf,
    },target[key])
}

/**
 * 
 * @param {path} 
 * 定義get方法
 */
export const get = path => route({
    method:'get',
    path:path
})

/**
 * 
 * @param {path} 
 * 定義post方法 
 */
export const post = path => route({
    method:'post',
    path:path
})

/**
 * 
 *  打印日誌
 */
let reqID = 0

const decorate = (args, middleware) => {
  let [ target, key, descriptor ] = args
  target[key] = isArray(target[key])
  target[key].unshift(middleware)
  return descriptor
}

export const convert = middleware => (...args) => decorate(args, middleware)

export const log = convert(async (ctx, next) => {
    let currentReqID = reqID++
    console.time(`${currentReqID} ${ctx.method} ${ctx.url}`)
    await next()
    console.timeEnd(`${currentReqID} ${ctx.method} ${ctx.url}`)
  })
  
複製代碼

而後再來看看咱們接口定義的文件，代碼趕忙簡潔的一匹.

routes/crawler.js

import { controller, get , log} from '../decorator/router'
import mongoose from 'mongoose'

const areaDataBase = mongoose.model('area')
const areaHouseDataBase = mongoose.model('areaHouse')
const detailHouse = mongoose.model('detailHouse')

@controller('')
export class Crawler{
    /**
     * 獲取杭州城區下的房子信息
     */
    @get('/getDetail')
    @log
    async detailHouse (ctx,next){
        let query = ctx.query
        let { _id } = query;
        if (!_id) return (ctx.body = '_id is required')

        let area = await detailHouse
        .findById(_id)
        .exec()
        ctx.body = {
            code:0,
            area
        }
    }
/**
 * 獲取杭州城區下的房子信息
 */
    @get('/getAreaHouse')
    @log
    async areaHouse (ctx,next){
        let areaHouse = await areaHouseDataBase
        .find({})
        .exec()
        ctx.body = {
            code:0,
            areaHouse
        }
    }
/**
 * 獲取杭州城區單條的名稱
 */
    @get('/getArea/:_id')
    @log
    async getArea (ctx,next){
        const { params } = ctx
        const { _id } = params

        if (!_id) return (ctx.body = '_id is required')

        let area = await areaDataBase
        .findById(_id)
        .exec()
        ctx.body = area
    }
/**
 * 獲取杭州城區的名稱
 */
    @get('/getArea')
    @log
    async Area (ctx,next){
        let area = await areaDataBase
        .find({})
        .exec()
        ctx.body = {
            code:0,
            area
        }
    }

}

複製代碼