js實現unicode碼字符串與utf8字節數據互轉

時間 2019-11-06

標籤實現 unicode 碼字符串 utf8 utf 字節數據欄目 JavaScript 简体版

原文原文鏈接

js的string變量存儲字符串使用的是unicode編碼，要保存時必須選擇其餘編碼後進行傳輸，好比轉成utf-8,utf-32等。存儲到數據庫中爲utf-8編碼，讀取出來如何轉換成正確的字符串就成了問題。如今給出解決方案，能夠正確支持中文、emoji表情、英文混合的字符串編碼互轉。數據庫

/**
 * Created by hdwang on 2019/1/28.
 */
var convertUtf8 = (function() {

    /**
     * unicode string to utf-8
     * @param text 字符串
     * @returns {*} utf-8編碼
     */
    function toBytes(text) {
        var result = [], i = 0;
        text = encodeURI(text);
        while (i < text.length) {
            var c = text.charCodeAt(i++);

            // if it is a % sign, encode the following 2 bytes as a hex value
            if (c === 37) {
                result.push(parseInt(text.substr(i, 2), 16))
                i += 2;

                // otherwise, just the actual byte
            } else {
                result.push(c)
            }
        }

        return coerceArray(result);
    }


    /**
     * utf8 byte to unicode string
     * @param utf8Bytes
     * @returns {string}
     */
    function utf8ByteToUnicodeStr(utf8Bytes){
        var unicodeStr ="";
        for (var pos = 0; pos < utf8Bytes.length;){
            var flag= utf8Bytes[pos];
            var unicode = 0 ;
            if ((flag >>>7) === 0 ) {
                unicodeStr+= String.fromCharCode(utf8Bytes[pos]);
                pos += 1;

            } else if ((flag &0xFC) === 0xFC ){
                unicode = (utf8Bytes[pos] & 0x3) << 30;
                unicode |= (utf8Bytes[pos+1] & 0x3F) << 24;
                unicode |= (utf8Bytes[pos+2] & 0x3F) << 18;
                unicode |= (utf8Bytes[pos+3] & 0x3F) << 12;
                unicode |= (utf8Bytes[pos+4] & 0x3F) << 6;
                unicode |= (utf8Bytes[pos+5] & 0x3F);
                unicodeStr+= String.fromCodePoint(unicode) ;
                pos += 6;

            }else if ((flag &0xF8) === 0xF8 ){
                unicode = (utf8Bytes[pos] & 0x7) << 24;
                unicode |= (utf8Bytes[pos+1] & 0x3F) << 18;
                unicode |= (utf8Bytes[pos+2] & 0x3F) << 12;
                unicode |= (utf8Bytes[pos+3] & 0x3F) << 6;
                unicode |= (utf8Bytes[pos+4] & 0x3F);
                unicodeStr+= String.fromCodePoint(unicode) ;
                pos += 5;

            } else if ((flag &0xF0) === 0xF0 ){
                unicode = (utf8Bytes[pos] & 0xF) << 18;
                unicode |= (utf8Bytes[pos+1] & 0x3F) << 12;
                unicode |= (utf8Bytes[pos+2] & 0x3F) << 6;
                unicode |= (utf8Bytes[pos+3] & 0x3F);
                unicodeStr+= String.fromCodePoint(unicode) ;
                pos += 4;

            } else if ((flag &0xE0) === 0xE0 ){
                unicode = (utf8Bytes[pos] & 0x1F) << 12;;
                unicode |= (utf8Bytes[pos+1] & 0x3F) << 6;
                unicode |= (utf8Bytes[pos+2] & 0x3F);
                unicodeStr+= String.fromCharCode(unicode) ;
                pos += 3;

            } else if ((flag &0xC0) === 0xC0 ){ //110
                unicode = (utf8Bytes[pos] & 0x3F) << 6;
                unicode |= (utf8Bytes[pos+1] & 0x3F);
                unicodeStr+= String.fromCharCode(unicode) ;
                pos += 2;

            } else{
                unicodeStr+= String.fromCharCode(utf8Bytes[pos]);
                pos += 1;
            }
        }
        return unicodeStr;
    }



    function checkInt(value) {
        return (parseInt(value) === value);
    }

    function checkInts(arrayish) {
        if (!checkInt(arrayish.length)) { return false; }

        for (var i = 0; i < arrayish.length; i++) {
            if (!checkInt(arrayish[i]) || arrayish[i] < 0 || arrayish[i] > 255) {
                return false;
            }
        }

        return true;
    }

    function coerceArray(arg, copy) {

        // ArrayBuffer view
        if (arg.buffer && arg.name === 'Uint8Array') {

            if (copy) {
                if (arg.slice) {
                    arg = arg.slice();
                } else {
                    arg = Array.prototype.slice.call(arg);
                }
            }

            return arg;
        }

        // It's an array; check it is a valid representation of a byte
        if (Array.isArray(arg)) {
            if (!checkInts(arg)) {
                throw new Error('Array contains invalid value: ' + arg);
            }

            return new Uint8Array(arg);
        }

        // Something else, but behaves like an array (maybe a Buffer? Arguments?)
        if (checkInt(arg.length) && checkInts(arg)) {
            return new Uint8Array(arg);
        }

        throw new Error('unsupported array-like object');
    }

    return {
        toBytes: toBytes,
        fromBytes: utf8ByteToUnicodeStr
    }
})()

針對emoji的字節字符，佔兩個unicode字符。使用String.fromCharCode也能夠實現，須要進行兩次fromCharCode，沒有fromPointCode方便。下面展現了utf-8的4字節轉換爲unicode(utf-16)的過程。編碼

//高char10位[一個unicode字符] (2+6+2=10)
unicode =   ((utf8Bytes[pos] & 0x3)) << 8 |((utf8Bytes[pos+1] & 0x3f) << 2) |((utf8Bytes[pos+2] >> 4) & 0x03);

//減去‭1F600‬中的1，這裏減去6個0便可，低位char已經佔據10位
unicode = unicode - parseInt('1000000',2)

//加上utf-16高char的標識符
unicode =  0xD800 + unicode;
console.log(unicode);
unicodeStr +=  String.fromCharCode(unicode);

//低char10位[一個unicode字符](4+6)
unicode =  ((utf8Bytes[pos+2] & 0x0F) << 6) | (utf8Bytes[pos+3] & 0x3F);
//加上utf-16低char的標識符
unicode = 0xDC00 + unicode;
console.log(unicode);
unicodeStr+=  String.fromCharCode(unicode);
pos += 4;