Skip to content

fix(convert): handle Unicode aliases for CP932 mapping differences #67

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
polygonplanet merged 3 commits into from
Sep 8, 2026
Merged

fix(convert): handle Unicode aliases for CP932 mapping differences #67

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
3f4c428
feat(convert): add Unicode aliases for CP932 mapping differences
polygonplanet Sep 7, 2026
9e6ecfc
refactor: rename internal option ignoreSurrogatePair to asCodePoint i…
polygonplanet Sep 8, 2026
b848595
fix(convert): fallback on invalid SJIS trail byte 0x7F
polygonplanet Sep 8, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
125 changes: 67 additions & 58 deletions src/encoding-convert.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -154,14 +154,24 @@ function SJISToJIS(data) {
}
results[results.length] = b1 - 0x80 & 0xFF;
} else if (b1 >= 0x80) {
b2 = data[++i];
if (b2 === 0x7F) {
if (index !== 0) {
index = 0;
results[results.length] = esc[0];
results[results.length] = esc[1];
results[results.length] = esc[2];
}
results[results.length] = config.FALLBACK_CHARACTER;
continue;
}
if (index !== 1) {
index = 1;
results[results.length] = esc[3];
results[results.length] = esc[4];
results[results.length] = esc[5];
}

b2 = data[++i];
if (sjisExt.hasCP932DuplicateCode(b1)) {
// Remap CP932 duplicate codes and IBM extended characters
remapped = sjisExt.remapCP932DuplicateCode(b1, b2);
Expand Down Expand Up @@ -228,6 +238,10 @@ function SJISToEUCJP(data) {
results[results.length] = b1;
} else if (b1 >= 0x81) {
b2 = data[++i];
if (b2 === 0x7F) {
results[results.length] = config.FALLBACK_CHARACTER;
continue;
}
if (sjisExt.hasCP932DuplicateCode(b1)) {
// Remap CP932 duplicate codes and IBM extended characters
remapped = sjisExt.remapCP932DuplicateCode(b1, b2);
Expand Down Expand Up @@ -409,6 +423,10 @@ function SJISToUTF8(data) {
results[results.length] = u3 & 0xFF;
} else if (b >= 0x80) {
b2 = data[++i];
if (b2 === 0x7F) {
results[results.length] = config.FALLBACK_CHARACTER;
continue;
}
if (sjisExt.hasCP932DuplicateCode(b)) {
// Remap CP932 duplicate codes and IBM extended characters
remapped = sjisExt.remapCP932DuplicateCode(b, b2);
Expand Down Expand Up @@ -642,7 +660,8 @@ function UTF8ToSJIS(data, options) {
(data[++i] & 0xFF);
}

jis = EncodingTable.UTF8_TO_JIS_TABLE[utf8];
jis = EncodingTable.UTF8_TO_JIS_TABLE[utf8] ||
EncodingTable.UTF8_TO_JIS_ALIAS_TABLE[utf8];
if (jis == null) {
if (fallbackOption) {
handleFallback(results, bytes, fallbackOption);
Expand All @@ -653,10 +672,6 @@ function UTF8ToSJIS(data, options) {
if (jis < 0xFF) {
results[results.length] = jis + 0x80;
} else {
if (jis > 0x10000) {
jis -= 0x10000;
}

b1 = jis >> 8;
b2 = jis & 0xFF;
if (b1 & 0x01) {
Expand Down Expand Up @@ -701,7 +716,7 @@ function UTF8ToEUCJP(data, options) {
var results = [];
var i = 0;
var len = data && data.length;
var b, bytes, utf8, jis;
var b, bytes, utf8, jis, isJISX0212;
var fallbackOption = options && options.fallback;

for (; i < len; i++) {
Expand All @@ -726,32 +741,28 @@ function UTF8ToEUCJP(data, options) {
(data[++i] & 0xFF);
}

jis = EncodingTable.UTF8_TO_JIS_TABLE[utf8];
// JIS X 0212 chars duplicated into JIS X 0208 unassigned rows: use the JIS X 0212 form.
if (jis != null && EncodingTable.UTF8_TO_JISX0212_TABLE[utf8] != null) {
jis = null;
jis = EncodingTable.UTF8_TO_JISX0212_TABLE[utf8];
isJISX0212 = jis != null;

if (jis == null) {
jis = EncodingTable.UTF8_TO_JIS_TABLE[utf8] ||
EncodingTable.UTF8_TO_JIS_ALIAS_TABLE[utf8];
}
if (jis == null) {
jis = EncodingTable.UTF8_TO_JISX0212_TABLE[utf8];
if (jis == null) {
if (fallbackOption) {
handleFallback(results, bytes, fallbackOption);
} else {
results[results.length] = config.FALLBACK_CHARACTER;
}
if (fallbackOption) {
handleFallback(results, bytes, fallbackOption);
} else {
results[results.length] = 0x8F;
results[results.length] = (jis >> 8) - 0x80 & 0xFF;
results[results.length] = (jis & 0xFF) - 0x80 & 0xFF;
results[results.length] = config.FALLBACK_CHARACTER;
}
} else {
if (jis > 0x10000) {
jis -= 0x10000;
}
if (jis < 0xFF) {
results[results.length] = 0x8E;
results[results.length] = jis - 0x80 & 0xFF;
} else {
if (isJISX0212) {
results[results.length] = 0x8F;
}
results[results.length] = (jis >> 8) - 0x80 & 0xFF;
results[results.length] = (jis & 0xFF) - 0x80 & 0xFF;
}
Expand All @@ -773,7 +784,7 @@ function UTF8ToJIS(data, options) {
var index = 0;
var len = data && data.length;
var i = 0;
var b, bytes, utf8, jis;
var b, bytes, utf8, jis, isJISX0212;
var fallbackOption = options && options.fallback;

var esc = [
Expand Down Expand Up @@ -813,42 +824,29 @@ function UTF8ToJIS(data, options) {
(data[++i] & 0xFF);
}

jis = EncodingTable.UTF8_TO_JIS_TABLE[utf8];
// JIS X 0212 chars duplicated into JIS X 0208 unassigned rows: use the JIS X 0212 form.
if (jis != null && EncodingTable.UTF8_TO_JISX0212_TABLE[utf8] != null) {
jis = null;
jis = EncodingTable.UTF8_TO_JISX0212_TABLE[utf8];
isJISX0212 = jis != null;

if (jis == null) {
jis = EncodingTable.UTF8_TO_JIS_TABLE[utf8] ||
EncodingTable.UTF8_TO_JIS_ALIAS_TABLE[utf8];
}

if (jis == null) {
jis = EncodingTable.UTF8_TO_JISX0212_TABLE[utf8];
if (jis == null) {
if (index !== 0) {
index = 0;
results[results.length] = esc[0];
results[results.length] = esc[1];
results[results.length] = esc[2];
}
if (index !== 0) {
index = 0;
results[results.length] = esc[0];
results[results.length] = esc[1];
results[results.length] = esc[2];
}

if (fallbackOption) {
handleFallback(results, bytes, fallbackOption);
} else {
results[results.length] = config.FALLBACK_CHARACTER;
}
if (fallbackOption) {
handleFallback(results, bytes, fallbackOption);
} else {
// JIS X 0212:1990
if (index !== 3) {
index = 3;
results[results.length] = esc[9];
results[results.length] = esc[10];
results[results.length] = esc[11];
results[results.length] = esc[12];
}
results[results.length] = jis >> 8 & 0xFF;
results[results.length] = jis & 0xFF;
results[results.length] = config.FALLBACK_CHARACTER;
}
} else {
if (jis > 0x10000) {
jis -= 0x10000;
}
if (jis < 0xFF) {
// Halfwidth Katakana
if (index !== 2) {
Expand All @@ -859,7 +857,17 @@ function UTF8ToJIS(data, options) {
}
results[results.length] = jis & 0xFF;
} else {
if (index !== 1) {
if (isJISX0212) {
// JIS X 0212:1990
if (index !== 3) {
index = 3;
results[results.length] = esc[9];
results[results.length] = esc[10];
results[results.length] = esc[11];
results[results.length] = esc[12];
}
} else if (index !== 1) {
// JIS X 0208
index = 1;
results[results.length] = esc[3];
results[results.length] = esc[4];
Expand Down Expand Up @@ -933,8 +941,9 @@ function UTF8ToUNICODE(data, options) {
var i = 0;
var len = data && data.length;
var n, c, c2, c3, c4, code;
// For internal usage only
var ignoreSurrogatePair = options && options.ignoreSurrogatePair;
// For internal use only
// Returns raw Unicode code points without splitting into surrogate pairs
var asCodePoint = options && options.asCodePoint;

while (i < len) {
c = data[i++];
Expand Down Expand Up @@ -970,7 +979,7 @@ function UTF8ToUNICODE(data, options) {
(c4 & 0x3F);
}

if (code <= 0xFFFF || ignoreSurrogatePair) {
if (code <= 0xFFFF || asCodePoint) {
results[results.length] = code;
} else {
// Split in surrogate halves
Expand Down Expand Up @@ -1685,7 +1694,7 @@ function handleFallback(results, bytes, fallbackOption) {
switch (fallbackOption) {
case 'html-entity':
case 'html-entity-hex':
var unicode = UTF8ToUNICODE(bytes, { ignoreSurrogatePair: true })[0];
var unicode = UTF8ToUNICODE(bytes, { asCodePoint: true })[0];
if (unicode) {
results[results.length] = 0x26; // &
results[results.length] = 0x23; // #
Expand Down
1 change: 1 addition & 0 deletions src/encoding-table.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
exports.UTF8_TO_JIS_TABLE = require('./utf8-to-jis-table');
exports.UTF8_TO_JIS_ALIAS_TABLE = require('./utf8-to-jis-alias-table');
exports.UTF8_TO_JISX0212_TABLE = require('./utf8-to-jisx0212-table');
exports.JIS_TO_UTF8_TABLE = require('./jis-to-utf8-table');
exports.JISX0212_TO_UTF8_TABLE = require('./jisx0212-to-utf8-table');
14 changes: 14 additions & 0 deletions src/utf8-to-jis-alias-table.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
/**
* Encoding conversion table for UTF-8 to JIS (encode only)
*
* Accepts Unicode characters that differ between CP932 and JIS X 0208 mappings
* (e.g., WAVE DASH vs. FULLWIDTH TILDE).
*
* { UTF-8 : JIS }
*/
module.exports = {
0xE28892: 0x215D, // − U+2212 MINUS SIGN same cell as － U+FF0D (0xEFBC8D) -> SJIS 0x817C
0xE3809C: 0x2141, // 〜 U+301C WAVE DASH same cell as ～ U+FF5E (0xEFBD9E) -> SJIS 0x8160
0xC2A2: 0x2171, // ¢ U+00A2 CENT SIGN same cell as ￠ U+FFE0 (0xEFBFA0) -> SJIS 0x8191
0xC2A3: 0x2172 // £ U+00A3 POUND SIGN same cell as ￡ U+FFE1 (0xEFBFA1) -> SJIS 0x8192
};
9 changes: 5 additions & 4 deletions src/utf8-to-jis-table.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
/* eslint-disable @stylistic/indent, @stylistic/key-spacing */
/**
* Encoding conversion table for UTF-8 to JIS
* { UTF-8 : JIS }
*/
module.exports = {
0xEFBDA1:0x21,0xEFBDA2:0x22,0xEFBDA3:0x23,0xEFBDA4:0x24,0xEFBDA5:0x25,
Expand Down Expand Up @@ -1486,8 +1487,8 @@ module.exports = {
0xE285B5:0x7C76,0xE285B6:0x7C77,0xE285B7:0x7C78,0xE285B8:0x7C79,0xE285B9:0x7C7A,
0xEFBFA4:0x7C7C,0xEFBC87:0x7C7D,0xEFBC82:0x7C7E,

//FIXME: mojibake
0xE288A5:0x2142,
0xEFBFA2:0x224C,
0xE28892:0x1215D
// Remaps Unicode characters that differ between CP932 and JIS X 0208 mappings
// (encode and decode) See also `utf8-to-jis-alias-table.js`
0xE288A5:0x2142, // ∥ U+2225 PARALLEL TO same cell as ‖ U+2016 (0xE28096) -> SJIS 0x8161
0xEFBFA2:0x224C // ￢ U+FFE2 FULLWIDTH NOT SIGN same cell as ¬ U+00AC (0xC2AC) -> SJIS 0x81CA
};
10 changes: 6 additions & 4 deletions src/utf8-to-jisx0212-table.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,11 @@
/* eslint-disable @stylistic/indent,@stylistic/key-spacing */
/**
* Encoding conversion table for UTF-8 to JIS X 0212:1990 (Hojo-Kanji)
*
* Character mappings based on:
* https://www.unicode.org/Public/MAPPINGS/OBSOLETE/EASTASIA/JIS/JIS0212.TXT
* Keys have been converted to UTF-8
* { UTF-8 : JIS }
*/
module.exports = {
0xCB98:0x222F,0xCB87:0x2230,0xC2B8:0x2231,0xCB99:0x2232,0xCB9D:0x2233,
Expand Down Expand Up @@ -1217,8 +1222,5 @@ module.exports = {
0xE9BDB1:0x6D53,0xE9BDB3:0x6D54,0xE9BDB5:0x6D55,0xE9BDBA:0x6D56,0xE9BDBD:0x6D57,
0xE9BE8F:0x6D58,0xE9BE90:0x6D59,0xE9BE91:0x6D5A,0xE9BE92:0x6D5B,0xE9BE94:0x6D5C,
0xE9BE96:0x6D5D,0xE9BE97:0x6D5E,0xE9BE9E:0x6D5F,0xE9BEA1:0x6D60,0xE9BEA2:0x6D61,
0xE9BEA3:0x6D62,0xE9BEA5:0x6D63,

//FIXME: mojibake
0xE3809C:0x2141
0xE9BEA3:0x6D62,0xE9BEA5:0x6D63
};
60 changes: 60 additions & 0 deletions tests/test.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1200,6 +1200,66 @@ describe('encoding', function() {
});
});
});
it('Unicode aliases to SJIS', function() {
var aliasMaps = [
{ input: [0x301C], sjis: [0x81, 0x60], decoded: [0xFF5E] }, // 〜 -> ～
{ input: [0xFF5E], sjis: [0x81, 0x60], decoded: [0xFF5E] }, // ～ -> ～
{ input: [0x00A2], sjis: [0x81, 0x91], decoded: [0xFFE0] }, // ¢ -> ￠
{ input: [0xFFE0], sjis: [0x81, 0x91], decoded: [0xFFE0] }, // ￠ -> ￠
{ input: [0x00A3], sjis: [0x81, 0x92], decoded: [0xFFE1] }, // £ -> ￡
{ input: [0xFFE1], sjis: [0x81, 0x92], decoded: [0xFFE1] }, // ￡ -> ￡
{ input: [0x2212], sjis: [0x81, 0x7C], decoded: [0xFF0D] }, // − -> －
{ input: [0xFF0D], sjis: [0x81, 0x7C], decoded: [0xFF0D] } // － -> －
];

aliasMaps.forEach(function(alias) {
var sjis = encoding.convert(alias.input, { to: 'sjis', from: 'unicode' });
assert.deepEqual(sjis, alias.sjis);

var decodedUnicode = encoding.convert(sjis, { to: 'unicode', from: 'sjis' });
assert.deepEqual(decodedUnicode, alias.decoded);
});
});

it('Unicode aliases to EUC-JP and JIS', function() {
var aliasMaps = [
{ input: [0x301C], eucjp: [0xA1, 0xC1], jis: [0x21, 0x41], decoded: [0xFF5E] }, // 〜 -> ～
{ input: [0x00A2], eucjp: [0xA1, 0xF1], jis: [0x21, 0x71], decoded: [0xFFE0] }, // ¢ -> ￠
{ input: [0x00A3], eucjp: [0xA1, 0xF2], jis: [0x21, 0x72], decoded: [0xFFE1] }, // £ -> ￡
{ input: [0x2212], eucjp: [0xA1, 0xDD], jis: [0x21, 0x5D], decoded: [0xFF0D] } // − -> －
];

aliasMaps.forEach(function(alias) {
var eucjp = encoding.convert(alias.input, { to: 'euc-jp', from: 'unicode' });
assert.deepEqual(eucjp, alias.eucjp);

var decodedUnicode = encoding.convert(eucjp, { to: 'unicode', from: 'euc-jp' });
assert.deepEqual(decodedUnicode, alias.decoded);

var jis = encoding.convert(alias.input, { to: 'jis', from: 'unicode' });
assert.deepEqual(jis, [0x1B, 0x24, 0x42].concat(alias.jis, [0x1B, 0x28, 0x42]));
});
});

it('JIS X 0212 undefined code point', function() {
var undefinedCodePoint = [0x8F, 0xA1, 0xC1];
var decoded = encoding.convert(undefinedCodePoint, {
to: 'unicode', from: 'euc-jp', type: 'string'
});
assert.equal(decoded, '?');
});

it('SJIS invalid trail byte 0x7F', function() {
var invalidSjisCode = [0x81, 0x7F];
var validSjisCode = [0x81, 0x7E];

assert.equal(encoding.convert(invalidSjisCode, {
to: 'unicode', from: 'sjis', type: 'string'
}), '?');

var unicode = encoding.convert(validSjisCode, { to: 'unicode', from: 'sjis' });
assert.deepEqual(unicode, [0xD7]);
});
});

describe('urlEncode/urlDecode', function() {
Expand Down