diff --git a/src/encoding-convert.js b/src/encoding-convert.js index a7722d0..bcdd154 100644 --- a/src/encoding-convert.js +++ b/src/encoding-convert.js @@ -154,6 +154,17 @@ function SJISToJIS(data) { } results[results.length] = b1 - 0x80 & 0xFF; } else if (b1 >= 0x80) { + b2 = data[++i]; + if (b2 === 0x7F) { + if (index !== 0) { + index = 0; + results[results.length] = esc[0]; + results[results.length] = esc[1]; + results[results.length] = esc[2]; + } + results[results.length] = config.FALLBACK_CHARACTER; + continue; + } if (index !== 1) { index = 1; results[results.length] = esc[3]; @@ -161,7 +172,6 @@ function SJISToJIS(data) { results[results.length] = esc[5]; } - b2 = data[++i]; if (sjisExt.hasCP932DuplicateCode(b1)) { // Remap CP932 duplicate codes and IBM extended characters remapped = sjisExt.remapCP932DuplicateCode(b1, b2); @@ -228,6 +238,10 @@ function SJISToEUCJP(data) { results[results.length] = b1; } else if (b1 >= 0x81) { b2 = data[++i]; + if (b2 === 0x7F) { + results[results.length] = config.FALLBACK_CHARACTER; + continue; + } if (sjisExt.hasCP932DuplicateCode(b1)) { // Remap CP932 duplicate codes and IBM extended characters remapped = sjisExt.remapCP932DuplicateCode(b1, b2); @@ -409,6 +423,10 @@ function SJISToUTF8(data) { results[results.length] = u3 & 0xFF; } else if (b >= 0x80) { b2 = data[++i]; + if (b2 === 0x7F) { + results[results.length] = config.FALLBACK_CHARACTER; + continue; + } if (sjisExt.hasCP932DuplicateCode(b)) { // Remap CP932 duplicate codes and IBM extended characters remapped = sjisExt.remapCP932DuplicateCode(b, b2); @@ -642,7 +660,8 @@ function UTF8ToSJIS(data, options) { (data[++i] & 0xFF); } - jis = EncodingTable.UTF8_TO_JIS_TABLE[utf8]; + jis = EncodingTable.UTF8_TO_JIS_TABLE[utf8] || + EncodingTable.UTF8_TO_JIS_ALIAS_TABLE[utf8]; if (jis == null) { if (fallbackOption) { handleFallback(results, bytes, fallbackOption); @@ -653,10 +672,6 @@ function UTF8ToSJIS(data, options) { if (jis < 0xFF) { results[results.length] = jis + 0x80; } else { - if (jis > 0x10000) { - jis -= 0x10000; - } - b1 = jis >> 8; b2 = jis & 0xFF; if (b1 & 0x01) { @@ -701,7 +716,7 @@ function UTF8ToEUCJP(data, options) { var results = []; var i = 0; var len = data && data.length; - var b, bytes, utf8, jis; + var b, bytes, utf8, jis, isJISX0212; var fallbackOption = options && options.fallback; for (; i < len; i++) { @@ -726,32 +741,28 @@ function UTF8ToEUCJP(data, options) { (data[++i] & 0xFF); } - jis = EncodingTable.UTF8_TO_JIS_TABLE[utf8]; // JIS X 0212 chars duplicated into JIS X 0208 unassigned rows: use the JIS X 0212 form. - if (jis != null && EncodingTable.UTF8_TO_JISX0212_TABLE[utf8] != null) { - jis = null; + jis = EncodingTable.UTF8_TO_JISX0212_TABLE[utf8]; + isJISX0212 = jis != null; + + if (jis == null) { + jis = EncodingTable.UTF8_TO_JIS_TABLE[utf8] || + EncodingTable.UTF8_TO_JIS_ALIAS_TABLE[utf8]; } if (jis == null) { - jis = EncodingTable.UTF8_TO_JISX0212_TABLE[utf8]; - if (jis == null) { - if (fallbackOption) { - handleFallback(results, bytes, fallbackOption); - } else { - results[results.length] = config.FALLBACK_CHARACTER; - } + if (fallbackOption) { + handleFallback(results, bytes, fallbackOption); } else { - results[results.length] = 0x8F; - results[results.length] = (jis >> 8) - 0x80 & 0xFF; - results[results.length] = (jis & 0xFF) - 0x80 & 0xFF; + results[results.length] = config.FALLBACK_CHARACTER; } } else { - if (jis > 0x10000) { - jis -= 0x10000; - } if (jis < 0xFF) { results[results.length] = 0x8E; results[results.length] = jis - 0x80 & 0xFF; } else { + if (isJISX0212) { + results[results.length] = 0x8F; + } results[results.length] = (jis >> 8) - 0x80 & 0xFF; results[results.length] = (jis & 0xFF) - 0x80 & 0xFF; } @@ -773,7 +784,7 @@ function UTF8ToJIS(data, options) { var index = 0; var len = data && data.length; var i = 0; - var b, bytes, utf8, jis; + var b, bytes, utf8, jis, isJISX0212; var fallbackOption = options && options.fallback; var esc = [ @@ -813,42 +824,29 @@ function UTF8ToJIS(data, options) { (data[++i] & 0xFF); } - jis = EncodingTable.UTF8_TO_JIS_TABLE[utf8]; // JIS X 0212 chars duplicated into JIS X 0208 unassigned rows: use the JIS X 0212 form. - if (jis != null && EncodingTable.UTF8_TO_JISX0212_TABLE[utf8] != null) { - jis = null; + jis = EncodingTable.UTF8_TO_JISX0212_TABLE[utf8]; + isJISX0212 = jis != null; + + if (jis == null) { + jis = EncodingTable.UTF8_TO_JIS_TABLE[utf8] || + EncodingTable.UTF8_TO_JIS_ALIAS_TABLE[utf8]; } + if (jis == null) { - jis = EncodingTable.UTF8_TO_JISX0212_TABLE[utf8]; - if (jis == null) { - if (index !== 0) { - index = 0; - results[results.length] = esc[0]; - results[results.length] = esc[1]; - results[results.length] = esc[2]; - } + if (index !== 0) { + index = 0; + results[results.length] = esc[0]; + results[results.length] = esc[1]; + results[results.length] = esc[2]; + } - if (fallbackOption) { - handleFallback(results, bytes, fallbackOption); - } else { - results[results.length] = config.FALLBACK_CHARACTER; - } + if (fallbackOption) { + handleFallback(results, bytes, fallbackOption); } else { - // JIS X 0212:1990 - if (index !== 3) { - index = 3; - results[results.length] = esc[9]; - results[results.length] = esc[10]; - results[results.length] = esc[11]; - results[results.length] = esc[12]; - } - results[results.length] = jis >> 8 & 0xFF; - results[results.length] = jis & 0xFF; + results[results.length] = config.FALLBACK_CHARACTER; } } else { - if (jis > 0x10000) { - jis -= 0x10000; - } if (jis < 0xFF) { // Halfwidth Katakana if (index !== 2) { @@ -859,7 +857,17 @@ function UTF8ToJIS(data, options) { } results[results.length] = jis & 0xFF; } else { - if (index !== 1) { + if (isJISX0212) { + // JIS X 0212:1990 + if (index !== 3) { + index = 3; + results[results.length] = esc[9]; + results[results.length] = esc[10]; + results[results.length] = esc[11]; + results[results.length] = esc[12]; + } + } else if (index !== 1) { + // JIS X 0208 index = 1; results[results.length] = esc[3]; results[results.length] = esc[4]; @@ -933,8 +941,9 @@ function UTF8ToUNICODE(data, options) { var i = 0; var len = data && data.length; var n, c, c2, c3, c4, code; - // For internal usage only - var ignoreSurrogatePair = options && options.ignoreSurrogatePair; + // For internal use only + // Returns raw Unicode code points without splitting into surrogate pairs + var asCodePoint = options && options.asCodePoint; while (i < len) { c = data[i++]; @@ -970,7 +979,7 @@ function UTF8ToUNICODE(data, options) { (c4 & 0x3F); } - if (code <= 0xFFFF || ignoreSurrogatePair) { + if (code <= 0xFFFF || asCodePoint) { results[results.length] = code; } else { // Split in surrogate halves @@ -1685,7 +1694,7 @@ function handleFallback(results, bytes, fallbackOption) { switch (fallbackOption) { case 'html-entity': case 'html-entity-hex': - var unicode = UTF8ToUNICODE(bytes, { ignoreSurrogatePair: true })[0]; + var unicode = UTF8ToUNICODE(bytes, { asCodePoint: true })[0]; if (unicode) { results[results.length] = 0x26; // & results[results.length] = 0x23; // # diff --git a/src/encoding-table.js b/src/encoding-table.js index 44de98c..8823e8c 100644 --- a/src/encoding-table.js +++ b/src/encoding-table.js @@ -1,4 +1,5 @@ exports.UTF8_TO_JIS_TABLE = require('./utf8-to-jis-table'); +exports.UTF8_TO_JIS_ALIAS_TABLE = require('./utf8-to-jis-alias-table'); exports.UTF8_TO_JISX0212_TABLE = require('./utf8-to-jisx0212-table'); exports.JIS_TO_UTF8_TABLE = require('./jis-to-utf8-table'); exports.JISX0212_TO_UTF8_TABLE = require('./jisx0212-to-utf8-table'); diff --git a/src/utf8-to-jis-alias-table.js b/src/utf8-to-jis-alias-table.js new file mode 100644 index 0000000..98b4fb4 --- /dev/null +++ b/src/utf8-to-jis-alias-table.js @@ -0,0 +1,14 @@ +/** + * Encoding conversion table for UTF-8 to JIS (encode only) + * + * Accepts Unicode characters that differ between CP932 and JIS X 0208 mappings + * (e.g., WAVE DASH vs. FULLWIDTH TILDE). + * + * { UTF-8 : JIS } + */ +module.exports = { + 0xE28892: 0x215D, // − U+2212 MINUS SIGN same cell as - U+FF0D (0xEFBC8D) -> SJIS 0x817C + 0xE3809C: 0x2141, // 〜 U+301C WAVE DASH same cell as ~ U+FF5E (0xEFBD9E) -> SJIS 0x8160 + 0xC2A2: 0x2171, // ¢ U+00A2 CENT SIGN same cell as ¢ U+FFE0 (0xEFBFA0) -> SJIS 0x8191 + 0xC2A3: 0x2172 // £ U+00A3 POUND SIGN same cell as £ U+FFE1 (0xEFBFA1) -> SJIS 0x8192 +}; diff --git a/src/utf8-to-jis-table.js b/src/utf8-to-jis-table.js index cfc8d05..b25c292 100644 --- a/src/utf8-to-jis-table.js +++ b/src/utf8-to-jis-table.js @@ -1,6 +1,7 @@ /* eslint-disable @stylistic/indent, @stylistic/key-spacing */ /** * Encoding conversion table for UTF-8 to JIS + * { UTF-8 : JIS } */ module.exports = { 0xEFBDA1:0x21,0xEFBDA2:0x22,0xEFBDA3:0x23,0xEFBDA4:0x24,0xEFBDA5:0x25, @@ -1486,8 +1487,8 @@ module.exports = { 0xE285B5:0x7C76,0xE285B6:0x7C77,0xE285B7:0x7C78,0xE285B8:0x7C79,0xE285B9:0x7C7A, 0xEFBFA4:0x7C7C,0xEFBC87:0x7C7D,0xEFBC82:0x7C7E, -//FIXME: mojibake -0xE288A5:0x2142, -0xEFBFA2:0x224C, -0xE28892:0x1215D +// Remaps Unicode characters that differ between CP932 and JIS X 0208 mappings +// (encode and decode) See also `utf8-to-jis-alias-table.js` +0xE288A5:0x2142, // ∥ U+2225 PARALLEL TO same cell as ‖ U+2016 (0xE28096) -> SJIS 0x8161 +0xEFBFA2:0x224C // ¬ U+FFE2 FULLWIDTH NOT SIGN same cell as ¬ U+00AC (0xC2AC) -> SJIS 0x81CA }; diff --git a/src/utf8-to-jisx0212-table.js b/src/utf8-to-jisx0212-table.js index 3043199..dd1a329 100644 --- a/src/utf8-to-jisx0212-table.js +++ b/src/utf8-to-jisx0212-table.js @@ -1,6 +1,11 @@ /* eslint-disable @stylistic/indent,@stylistic/key-spacing */ /** * Encoding conversion table for UTF-8 to JIS X 0212:1990 (Hojo-Kanji) + * + * Character mappings based on: + * https://www.unicode.org/Public/MAPPINGS/OBSOLETE/EASTASIA/JIS/JIS0212.TXT + * Keys have been converted to UTF-8 + * { UTF-8 : JIS } */ module.exports = { 0xCB98:0x222F,0xCB87:0x2230,0xC2B8:0x2231,0xCB99:0x2232,0xCB9D:0x2233, @@ -1217,8 +1222,5 @@ module.exports = { 0xE9BDB1:0x6D53,0xE9BDB3:0x6D54,0xE9BDB5:0x6D55,0xE9BDBA:0x6D56,0xE9BDBD:0x6D57, 0xE9BE8F:0x6D58,0xE9BE90:0x6D59,0xE9BE91:0x6D5A,0xE9BE92:0x6D5B,0xE9BE94:0x6D5C, 0xE9BE96:0x6D5D,0xE9BE97:0x6D5E,0xE9BE9E:0x6D5F,0xE9BEA1:0x6D60,0xE9BEA2:0x6D61, -0xE9BEA3:0x6D62,0xE9BEA5:0x6D63, - -//FIXME: mojibake -0xE3809C:0x2141 +0xE9BEA3:0x6D62,0xE9BEA5:0x6D63 }; diff --git a/tests/test.js b/tests/test.js index 34813c8..f55e487 100644 --- a/tests/test.js +++ b/tests/test.js @@ -1200,6 +1200,66 @@ describe('encoding', function() { }); }); }); + it('Unicode aliases to SJIS', function() { + var aliasMaps = [ + { input: [0x301C], sjis: [0x81, 0x60], decoded: [0xFF5E] }, // 〜 -> ~ + { input: [0xFF5E], sjis: [0x81, 0x60], decoded: [0xFF5E] }, // ~ -> ~ + { input: [0x00A2], sjis: [0x81, 0x91], decoded: [0xFFE0] }, // ¢ -> ¢ + { input: [0xFFE0], sjis: [0x81, 0x91], decoded: [0xFFE0] }, // ¢ -> ¢ + { input: [0x00A3], sjis: [0x81, 0x92], decoded: [0xFFE1] }, // £ -> £ + { input: [0xFFE1], sjis: [0x81, 0x92], decoded: [0xFFE1] }, // £ -> £ + { input: [0x2212], sjis: [0x81, 0x7C], decoded: [0xFF0D] }, // − -> - + { input: [0xFF0D], sjis: [0x81, 0x7C], decoded: [0xFF0D] } // - -> - + ]; + + aliasMaps.forEach(function(alias) { + var sjis = encoding.convert(alias.input, { to: 'sjis', from: 'unicode' }); + assert.deepEqual(sjis, alias.sjis); + + var decodedUnicode = encoding.convert(sjis, { to: 'unicode', from: 'sjis' }); + assert.deepEqual(decodedUnicode, alias.decoded); + }); + }); + + it('Unicode aliases to EUC-JP and JIS', function() { + var aliasMaps = [ + { input: [0x301C], eucjp: [0xA1, 0xC1], jis: [0x21, 0x41], decoded: [0xFF5E] }, // 〜 -> ~ + { input: [0x00A2], eucjp: [0xA1, 0xF1], jis: [0x21, 0x71], decoded: [0xFFE0] }, // ¢ -> ¢ + { input: [0x00A3], eucjp: [0xA1, 0xF2], jis: [0x21, 0x72], decoded: [0xFFE1] }, // £ -> £ + { input: [0x2212], eucjp: [0xA1, 0xDD], jis: [0x21, 0x5D], decoded: [0xFF0D] } // − -> - + ]; + + aliasMaps.forEach(function(alias) { + var eucjp = encoding.convert(alias.input, { to: 'euc-jp', from: 'unicode' }); + assert.deepEqual(eucjp, alias.eucjp); + + var decodedUnicode = encoding.convert(eucjp, { to: 'unicode', from: 'euc-jp' }); + assert.deepEqual(decodedUnicode, alias.decoded); + + var jis = encoding.convert(alias.input, { to: 'jis', from: 'unicode' }); + assert.deepEqual(jis, [0x1B, 0x24, 0x42].concat(alias.jis, [0x1B, 0x28, 0x42])); + }); + }); + + it('JIS X 0212 undefined code point', function() { + var undefinedCodePoint = [0x8F, 0xA1, 0xC1]; + var decoded = encoding.convert(undefinedCodePoint, { + to: 'unicode', from: 'euc-jp', type: 'string' + }); + assert.equal(decoded, '?'); + }); + + it('SJIS invalid trail byte 0x7F', function() { + var invalidSjisCode = [0x81, 0x7F]; + var validSjisCode = [0x81, 0x7E]; + + assert.equal(encoding.convert(invalidSjisCode, { + to: 'unicode', from: 'sjis', type: 'string' + }), '?'); + + var unicode = encoding.convert(validSjisCode, { to: 'unicode', from: 'sjis' }); + assert.deepEqual(unicode, [0xD7]); + }); }); describe('urlEncode/urlDecode', function() {