Source: lib/helper/xml-encoding-sniffer.js

  1. /**
  2.  * W3C Extensible Markup Language (XML) 1.0 (Fifth Edition)
  3.  * W3C Recommendation 26 November 2008
  4.  * url: https://www.w3.org/TR/xml/#charencoding
  5.  *
  6.  * function xmlEncodingSniffer(sample)
  7.  * function getXmlDeclaredEncoding(sample,  guessedEncoding)
  8.  * function getStringFromByteArray(message, guessedEncoding)
  9.  * function detectUnicodeInByteSampleByHeuristics(sampleBytes)
  10.  * function detectSuspiciousUTF8SequenceLength(sampleBytes, currentPos)
  11.  * function detectBOMBytes(bomBytes)
  12.  * function isCommonUSASCIIByte(byte)
  13.  *
  14.  * xmlEncodingSniffer                    => detectBOMBytes, getXmlDeclaredEncoding
  15.  * getXmlDeclaredEncoding                => getStringFromByteArray
  16.  * getStringFromByteArray                => detectBOMBytes, detectUnicodeInByteSampleByHeuristics
  17.  * detectUnicodeInByteSampleByHeuristics => detectSuspiciousUTF8SequenceLength
  18.  * detectSuspiciousUTF8SequenceLength    => NULL
  19.  * detectBOMBytes                        => NULL
  20.  * isCommonUSASCIIByte(byte)             => NULL
  21.  *
  22.  * @module  helper-xml-encoding-sniffer
  23.  * @desc    Helper module - Helper functions for the main module {@link module:whatwg-xhr whatwg-xhr}.
  24.  * @version 1.0.0
  25.  * @author  Essam A. El-Sherif
  26.  */
  28. /**
  29.  * @func    xmlEncodingSniffer
  30.  * @static
  31.  * @param   {object} Sample Buffer data.
  32.  * @return  {string} Encoding detected.
  33.  * @desc    Auto-detection of character encoding of XML data.
  34.  */
  35. export function xmlEncodingSniffer(sample){
  37. 	let encodingFound1 = null;
  38. 	let encodingFound2 = null;
  40. 	// look for the BOM in the read sample
  41. 	encodingFound1 = detectBOMBytes(sample);
  43. 	// if the encoding was not detected due to a missing or unrecognizable BOM,
  44. 	// try to detect from the binary representation of the string "<?xml"
  45. 	let checkPseudoAttribute = false;
  47. 	if(encodingFound1 === null){
  49. 		if(sample[0] === 0x00 && sample[1] === 0x3C && sample[2] === 0x00 && sample[3] === 0x3F){
  51. 			// UTF-16BE or big-endian ISO-10646-UCS-2 or other encoding with a 16-bit code unit
  52. 			// in big-endian order and ASCII characters encoded as ASCII values.
  53. 			// (the encoding declaration must be read to determine which)
  55. 			encodingFound1 = 'UTF-16BE';
  56. 			checkPseudoAttribute = true;
  57. 		}
  58. 		else
  59. 		if(sample[0] === 0x00 && sample[1] === 0x00 && sample[2] === 0x00 && sample[3] === 0x3C){
  61. 			// UTF-32BE
  62. 			// (the encoding declaration must be read to determine which)
  64. 			encodingFound1 = 'UTF-32BE';
  65. 			checkPseudoAttribute = true;
  66. 		}
  67. 		else
  68. 		if(sample[0] === 0xFF && sample[1] === 0xFE){
  70. 			encodingFound1 = 'UTF-16LE';
  71. 		}
  72. 		else
  73. 		if(sample[0] === 0xFE && sample[1] === 0xFF){
  75. 			encodingFound1 = 'UTF-16BE';
  76. 		}
  77. 		else
  78. 		if(sample[0] === 0x3C && sample[1] === 0x00 && sample[2] === 0x00 && sample[3] === 0x00){
  80. 			// (the encoding declaration must be read to determine which)
  82. 			encodingFound1 = 'UTF-32';
  83. 			checkPseudoAttribute = true;
  84. 		}
  85. 		else
  86. 		if(sample[0] === 0x3C && sample[1] === 0x00 && sample[2] === 0x3F && sample[3] === 0x00){
  88. 			// UTF-16LE or little-endian ISO-10646-UCS-2 or other encoding with a 16-bit code unit
  89. 			// in little-endian order and ASCII characters encoded as ASCII values
  90. 			// (the encoding declaration must be read to determine which)
  92. 			encodingFound1 = 'UTF-16LE';
  93. 			checkPseudoAttribute = true;
  94. 		}
  95. 		else
  96. 		if(sample[0] === 0x3C && sample[1] === 0x3F && sample[2] === 0x78 && sample[3] === 0x6D){
  98. 			// UTF-8, ISO 646, ASCII, some part of ISO 8859 or any other 7-bit, 8-bit
  99. 			// (the encoding declaration must be read to determine which)
  101. 			encodingFound1 = 'ASCII';
  102. 			checkPseudoAttribute = true;
  103. 		}
  104. 		else
  105. 		if(sample[0] === 0x4C && sample[1] === 0x6F && sample[2] === 0xA7 && sample[3] === 0x94){
  107. 		    // IBM037 - IBM EBCDIC US-Canada"CP037";
  109. 			encodingFound1 = 'IBM037';
  110. 		}
  111. 	}	// if (encodingFound1 === null)
  113. 	// Now read the encoding pseudoattribute in the XML header, if present
  114. 	encodingFound2 = getXmlDeclaredEncoding(sample, encodingFound1 || 'UTF-8');
  116. 	// when not declared, w3c says it is utf-8
  117. 	if(encodingFound2 === null)
  118. 		encodingFound2 = 'UTF-8';
  120. 	// compare the 2 found encoding and decided which is the right one
  122. 	let winner = null;
  123. 	if(encodingFound1 === encodingFound2){
  124. 		winner = encodingFound2;
  125. 	}
  126. 	else
  127. 	if(encodingFound1 === null){
  128. 		winner = encodingFound2;
  129. 	}
  130. 	else
  131. 	if(encodingFound2 === null){
  132. 		winner = encodingFound1;
  133. 	}
  134. 	else
  135. 	if(checkPseudoAttribute){
  136. 		// Fine-tune the winner encoding. This is the most heuristic part, as some encoding
  137. 		// can be overloaded. E.g. ASCII might be UTF-7, UTF-8, ISO-8859...
  139. 		if(
  140. 			(encodingFound1 === 'ASCII') &&
  141. 			(encodingFound2 === 'UTF-7' || encodingFound2 === 'UTF-8' || encodingFound2.toUpperCase().includes('ISO-8859'))
  142. 		){
  143. 			winner = encodingFound2;
  144. 		}
  145. 		else{
  146. 			// I'm not sure here if throw an exception or accept encodingFound1 or encodingFound2,
  147. 			// as both are not null and not equals
  148. 			// throw new Error(
  149. 			//	`The text encoding and the encoding pseudo-attribute of the XML header mismatch ${encodingFound1} ${encodingFound2}`);
  151. 			winner = encodingFound2;
  152. 		}
  153. 	}
  154. 	else{
  155. 			// encodingFound1 and encodingFound2 are different so none win
  156. 			// throw new Error(
  157. 			//	`The text encoding and the encoding pseudo-attribute of the XML header mismatch ${encodingFound1} ${encodingFound2}`);
  159. 			winner = encodingFound2;
  160. 	}
  162. 	// return the detected encoding
  163. 	return winner;
  164. }
  166. function getXmlDeclaredEncoding(sample, guessedEncoding){
  168. 	// capture the encoding from the xml declaraion
  169. 	let contents = getStringFromByteArray(sample, guessedEncoding);
  171. 	let pattern = /<\?xml\s+version\=["']1\.0["']\s+encoding\=["'](?<encoding>[\w\-]+)["']/;
  173. 	let m = contents.match(pattern);
  175. 	return m && m.groups['encoding'];
  176. }
  178. function getStringFromByteArray(message, guessedEncoding){
  180. 	// try to get the encoding from the byte array
  181. 	let encodingFound = detectBOMBytes(message);
  183. 	if(encodingFound){
  184. 		let preamble;
  186. 		switch(encodingFound){
  187. 			case 'UTF-16LE':
  188. 			case 'UTF-16LE':
  189. 				preamble = 2;
  190. 				break;
  191. 			case 'UTF-7':
  192. 			case 'UTF-8':
  193. 				preamble = 3;
  194. 				break;
  195. 			case 'UTF-32LE':
  196. 			case 'UTF-32BE':
  197. 				preamble = 4;
  198. 				break;
  199. 			default:
  200. 				preamble = 0;
  201. 		}
  203. 		let decoder = new TextDecoder(encodingFound);
  205. 		return new TextDecoder(encodingFound).decode(message.subarray(preamble));
  206. 	}
  208. 	encodingFound = detectUnicodeInByteSampleByHeuristics(message) || guessedEncoding;
  210. 	return new TextDecoder(encodingFound).decode( new Uint8Array(message).buffer );
  211. }
  213. function detectUnicodeInByteSampleByHeuristics(sampleBytes){
  215. 	let oddBinaryNullsInSample      = 0;
  216. 	let evenBinaryNullsInSample     = 0;
  218. 	let suspiciousUTF8SequenceCount = 0;
  219. 	let suspiciousUTF8BytesTotal    = 0;
  221. 	let likelyUSASCIIBytesInSample  = 0;
  223. 	// Cycle through, keeping count of binary null positions, possible UTF-8
  224. 	// sequences from upper ranges of Windows-1252, and probable US-ASCII
  225. 	// character counts.
  227. 	let currentPos    = 0;
  228. 	let skipUTF8Bytes = 0;
  230. 	while(currentPos < sampleBytes.length){
  232. 		//binary null distribution
  233. 		if(sampleBytes[currentPos] === 0){
  234. 			if(currentPos % 2 === 0)
  235. 				evenBinaryNullsInSample++;
  236. 			else
  237. 				oddBinaryNullsInSample++;
  238. 		}
  240. 		//likely US-ASCII characters
  241. 		if(isCommonUSASCIIByte(sampleBytes[currentPos])){
  242. 			likelyUSASCIIBytesInSample++;
  243. 		}
  245. 		//suspicious sequences (look like UTF-8)
  246. 		if(skipUTF8Bytes === 0){
  248. 			let lengthFound = detectSuspiciousUTF8SequenceLength(sampleBytes, currentPos);
  250. 			if(lengthFound > 0){
  252. 				suspiciousUTF8SequenceCount++;
  253. 				suspiciousUTF8BytesTotal += lengthFound;
  254. 				skipUTF8Bytes = lengthFound - 1;
  255. 			}
  256. 		}
  257. 		else{
  258. 			skipUTF8Bytes--;
  259. 		}
  261. 		currentPos++;
  262. 	}
  264. 	// UTF-16 LE - in english / european environments, this is usually characterized by a
  265. 	// high proportion of odd binary nulls (starting at 0), with (as this is text) a low
  266. 	// proportion of even binary nulls.
  267. 	// The thresholds here used (less than 20% nulls where you expect non-nulls, and more than
  268. 	// 60% nulls where you do expect nulls) are completely arbitrary.
  270. 	if(
  271. 		((evenBinaryNullsInSample * 2.0) / sampleBytes.length) < 0.2 &&
  272. 		((oddBinaryNullsInSample * 2.0)  / sampleBytes.length) > 0.6
  273. 	)
  274. 		return 'UTF-16LE';
  276. 	// UTF-16 BE - in english / european environments, this is usually characterized by a
  277. 	// high proportion of even binary nulls (starting at 0), with (as this is text) a low
  278. 	// proportion of odd binary nulls.
  279. 	// The thresholds here used (less than 20% nulls where you expect non-nulls, and more than
  280. 	// 60% nulls where you do expect nulls) are completely arbitrary.
  282. 	if(
  283. 		((oddBinaryNullsInSample * 2.0) / sampleBytes.length)  < 0.2 &&
  284. 		((evenBinaryNullsInSample * 2.0) / sampleBytes.length) > 0.6
  285. 	)
  286. 		return 'UTF-16BE';
  288. 	// UTF-8 - Martin Dürst outlines a method for detecting whether something CAN be UTF-8 content
  289. 	// using regexp, in his w3c.org unicode FAQ entry:
  290. 	// http://www.w3.org/International/questions/qa-forms-utf-8
  292. 	let potentiallyMangledString = sampleBytes.toString('ascii');
  294. 	let utf8Validator = '';
  296. 	utf8Validator += '^(';
  297. 	utf8Validator += '[\\x00-\\x7F]';                            // ASCII
  298. 	utf8Validator += '|[\\xC2-\\xDF][\\x80-\\xBF]';              // non-overlong 2-byte
  299. 	utf8Validator += '|\\xE0[\\xA0-\\xBF][\\x80-\\xBF]';         // excluding overlongs
  300. 	utf8Validator += '|[\\xE1-\\xEC\\xEE\\xEF][\\x80-\\xBF]{2}'; // straight 3-byte
  301. 	utf8Validator += '|\\xED[\\x80-\\x9F][\\x80-\\xBF]';         // excluding surrogates
  302. 	utf8Validator += '|\\xF0[\\x90-\\xBF][\\x80-\\xBF]{2}';      // planes 1-3
  303. 	utf8Validator += '|[\\xF1-\\xF3][\\x80-\\xBF]{3}';           // planes 4-15
  304. 	utf8Validator += '|\\xF4[\\x80-\\x8F][\\x80-\\xBF]{2}';      // plane 16
  305. 	utf8Validator += ')*$';
  307. 	utf8Validator = new RegExp(utf8Validator);
  309. 	if(utf8Validator.test(potentiallyMangledString)){
  311. 		// Unfortunately, just the fact that it CAN be UTF-8 doesn't tell you much about probabilities.
  312. 		// If all the characters are in the 0-127 range, no harm done, most western charsets are same
  313. 		// as UTF-8 in these ranges.
  314. 		// If some of the characters were in the upper range (western accented characters), however,
  315. 		// they would likely be mangled to 2-byte by the UTF-8 encoding process.
  316. 		// So, we need to play stats.
  318. 		// The "Random" likelihood of any pair of randomly generated characters being one
  319. 		// of these "suspicious" character sequences is:
  320. 		// 128 / (256 * 256) = 0.2%.
  321. 		//
  322. 		// In western text data, that is SIGNIFICANTLY reduced - most text data stays in the <127
  323. 		// character range, so we assume that more than 1 in 500,000 of these character
  324. 		// sequences indicates UTF-8. The number 500,000 is completely arbitrary - so sue me.
  325. 		//
  326. 		// We can only assume these character sequences will be rare if we ALSO assume that this
  327. 		// IS in fact western text - in which case the bulk of the UTF-8 encoded data (that is
  328. 		// not already suspicious sequences) should be plain US-ASCII bytes. This, I
  329. 		// arbitrarily decided, should be 80% (a random distribution, eg binary data, would yield
  330. 		// approx 40%, so the chances of hitting this threshold by accident in random data are
  331. 		// VERY low).
  333. 		if
  334. 			(
  335. 				(suspiciousUTF8SequenceCount * 500000.0 / sampleBytes.length >= 1) && //suspicious sequences
  336. 				( //all suspicious, so cannot evaluate proportion of US-Ascii
  337. 				    (sampleBytes.length - suspiciousUTF8BytesTotal === 0) ||
  338. 					likelyUSASCIIBytesInSample * 1.0 / (sampleBytes.length - suspiciousUTF8BytesTotal) >= 0.8
  339. 				)
  340. 		)
  341. 			return 'UTF-8';
  342. 	}
  344. 	return null;
  345. }
  347. function detectSuspiciousUTF8SequenceLength(sampleBytes, currentPos){
  349. 	let lengthFound = 0;
  351. 		if(
  352. 			sampleBytes.length >= currentPos + 1 &&
  353. 			sampleBytes[currentPos] === 0xC2
  354. 		){
  355. 			if(
  356. 				sampleBytes[currentPos + 1] === 0x81 ||
  357. 				sampleBytes[currentPos + 1] === 0x8D ||
  358. 				sampleBytes[currentPos + 1] === 0x8F
  359. 			)
  360. 				lengthFound = 2;
  361. 			else
  362. 			if(
  363. 				sampleBytes[currentPos + 1] === 0x90 ||
  364. 				sampleBytes[currentPos + 1] === 0x9D
  365. 			)
  366. 				lengthFound = 2;
  367. 			else
  368. 			if(
  369. 				sampleBytes[currentPos + 1] >= 0xA0 &&
  370. 				sampleBytes[currentPos + 1] <= 0xBF
  371. 			)
  372. 				lengthFound = 2;
  373. 		}
  374. 		else
  375. 		if(
  376. 			sampleBytes.length >= currentPos + 1 &&
  377. 			sampleBytes[currentPos] === 0xC3
  378. 		){
  379. 			if(
  380. 				sampleBytes[currentPos + 1] >= 0x80 &&
  381. 				sampleBytes[currentPos + 1] <= 0xBF
  382. 			)
  383. 				lengthFound = 2;
  384. 		}
  385. 		else
  386. 		if(
  387. 			sampleBytes.length >= currentPos + 1 &&
  388. 			sampleBytes[currentPos] === 0xC5
  389. 		){
  390. 			if(
  391. 				sampleBytes[currentPos + 1] === 0x92 ||
  392. 				sampleBytes[currentPos + 1] === 0x93
  393. 			)
  394. 				lengthFound = 2;
  395. 			else
  396. 			if(
  397. 				sampleBytes[currentPos + 1] === 0xA0 ||
  398. 				sampleBytes[currentPos + 1] === 0xA1
  399. 			)
  400. 				lengthFound = 2;
  401. 			else
  402. 			if(
  403. 				sampleBytes[currentPos + 1] === 0xB8 ||
  404. 				sampleBytes[currentPos + 1] === 0xBD ||
  405. 				sampleBytes[currentPos + 1] === 0xBE
  406. 			)
  407. 				lengthFound = 2;
  408. 		}
  409. 		else
  410. 		if(
  411. 			sampleBytes.length >= currentPos + 1 &&
  412. 			sampleBytes[currentPos] === 0xC6
  413. 		){
  414. 			if(sampleBytes[currentPos + 1] === 0x92)
  415. 				lengthFound = 2;
  416. 		}
  417. 		else
  418. 		if(
  419. 			sampleBytes.length >= currentPos + 1 &&
  420. 			sampleBytes[currentPos] === 0xCB
  421. 		){
  422. 			if(
  423. 				sampleBytes[currentPos + 1] === 0x86 ||
  424. 				sampleBytes[currentPos + 1] === 0x9C
  425. 			)
  426. 				lengthFound = 2;
  427. 		}
  428. 		else
  429. 		if(
  430. 			sampleBytes.length >= currentPos + 2 &&
  431. 			sampleBytes[currentPos] === 0xE2
  432. 		){
  433. 			if(sampleBytes[currentPos + 1] === 0x80){
  434. 				if(
  435. 					sampleBytes[currentPos + 2] === 0x93 ||
  436. 					sampleBytes[currentPos + 2] === 0x94
  437. 				)
  438. 					lengthFound = 3;
  440. 				if(
  441. 					sampleBytes[currentPos + 2] === 0x98 ||
  442. 					sampleBytes[currentPos + 2] === 0x99 ||
  443. 					sampleBytes[currentPos + 2] === 0x9A
  444. 				)
  445. 					lengthFound = 3;
  447. 				if(
  448. 					sampleBytes[currentPos + 2] === 0x9C ||
  449. 					sampleBytes[currentPos + 2] === 0x9D ||
  450. 					sampleBytes[currentPos + 2] === 0x9E
  451. 				)
  452. 					lengthFound = 3;
  454. 				if(
  455. 					sampleBytes[currentPos + 2] === 0xA0 ||
  456. 					sampleBytes[currentPos + 2] === 0xA1 ||
  457. 					sampleBytes[currentPos + 2] === 0xA2
  458. 				)
  459. 					lengthFound = 3;
  461. 				if(sampleBytes[currentPos + 2] === 0xA6)
  462. 					lengthFound = 3;
  464. 				if(sampleBytes[currentPos + 2] === 0xB0)
  465. 					lengthFound = 3;
  467. 				if(
  468. 					sampleBytes[currentPos + 2] === 0xB9 ||
  469. 					sampleBytes[currentPos + 2] === 0xBA
  470. 				)
  471. 					lengthFound = 3;
  472. 			}
  473. 			else
  474. 			if(
  475. 				sampleBytes[currentPos + 1] === 0x82 &&
  476. 				sampleBytes[currentPos + 2] === 0xAC
  477. 			)
  478. 				lengthFound = 3;
  479. 			else
  480. 			if(
  481. 				sampleBytes[currentPos + 1] === 0x84 &&
  482. 				sampleBytes[currentPos + 2] === 0xA2
  483. 			)
  484. 				lengthFound = 3;
  485. 		}
  487. 	return lengthFound;
  488. }
  490. function detectBOMBytes(bomBytes){
  492. 	if (bomBytes.length < 2) return null;
  494. 	// UTF-16LE - Unicode UTF-16 little endian byte order
  495. 	if (
  496. 		bomBytes[0] === 0xFF &&
  497. 		bomBytes[1] === 0xFE &&
  498. 		(bomBytes.length < 4 || bomBytes[2] != 0x00 || bomBytes[3] != 0x00)
  499. 	)
  500. 		return 'UTF-16LE';
  502. 	// UTF-16BE - Unicode UTF-16 big endian byte order
  503. 	if (bomBytes[0] === 0xFE && bomBytes[1] === 0xFF)
  504. 		return 'UTF-16BE';
  506. 	if (bomBytes.length < 3) return null;
  508. 	// UTF-8
  509. 	if (bomBytes[0] === 0xEF && bomBytes[1] === 0xBB && bomBytes[2] === 0xBF)
  510. 		return 'UTF-8';
  512. 	// Character encodings such as UTF-7 that make overloaded usage of ASCII-valued
  513. 	// bytes may fail to be reliably detected
  514. 	if (bomBytes[0] === 0x2B && bomBytes[1] === 0x2F && bomBytes[2] === 0x76)
  515. 		return 'UTF-7';
  517. 	if (bomBytes.length < 4) return null;
  519. 	// UTF-32LE - Unicode UTF-32 little endian byte order
  520. 	if (bomBytes[0] === 0xFF && bomBytes[1] === 0xFE && bomBytes[2] === 0x00 && bomBytes[3] === 0x00)
  521. 		return 'UTF-32LE';
  523. 	// UTF-32BE - Unicode UTF-32 big endian byte order
  524. 	if (bomBytes[0] === 0x00 && bomBytes[1] === 0x00 && bomBytes[2] === 0xFE && bomBytes[3] === 0xFF)
  525. 		return 'UTF-32BE';
  527. 	return null;
  528. }
  530. function isCommonUSASCIIByte(byte){
  531. 	return (
  532. 		byte === 0x0A || //lf
  533. 		byte === 0x0D || //cr
  534. 		byte === 0x09 || //tab
  535. 		(byte >= 0x20 && byte <= 0x2F) || //common punctuation
  536. 		(byte >= 0x30 && byte <= 0x39) || //digits
  537. 		(byte >= 0x3A && byte <= 0x40) || //common punctuation
  538. 		(byte >= 0x41 && byte <= 0x5A) || //capital letters
  539. 		(byte >= 0x5B && byte <= 0x60) || //common punctuation
  540. 		(byte >= 0x61 && byte <= 0x7A) || //lowercase letters
  541. 		(byte >= 0x7B && byte <= 0x7E)    //common punctuation
  542. 	);
  543. }

A Node.js implementation of the WHATWG XMLHttpRequest Living Standard for non-browser environments.