Nelze vybrat více než 25 témat Téma musí začínat písmenem nebo číslem, může obsahovat pomlčky („-“) a může být dlouhé až 35 znaků.

rcube_charset.php 27KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820
  1. <?php
  2. /**
  3. +-----------------------------------------------------------------------+
  4. | This file is part of the Roundcube Webmail client |
  5. | Copyright (C) 2005-2012, The Roundcube Dev Team |
  6. | Copyright (C) 2011-2012, Kolab Systems AG |
  7. | Copyright (C) 2000 Edmund Grimley Evans <edmundo@rano.org> |
  8. | |
  9. | Licensed under the GNU General Public License version 3 or |
  10. | any later version with exceptions for skins & plugins. |
  11. | See the README file for a full license statement. |
  12. | |
  13. | PURPOSE: |
  14. | Provide charset conversion functionality |
  15. +-----------------------------------------------------------------------+
  16. | Author: Thomas Bruederli <roundcube@gmail.com> |
  17. | Author: Aleksander Machniak <alec@alec.pl> |
  18. +-----------------------------------------------------------------------+
  19. */
  20. /**
  21. * Character sets conversion functionality
  22. *
  23. * @package Framework
  24. * @subpackage Core
  25. * @author Thomas Bruederli <roundcube@gmail.com>
  26. * @author Aleksander Machniak <alec@alec.pl>
  27. * @author Edmund Grimley Evans <edmundo@rano.org>
  28. */
  29. class rcube_charset
  30. {
  31. // Aliases: some of them from HTML5 spec.
  32. static public $aliases = array(
  33. 'USASCII' => 'WINDOWS-1252',
  34. 'ANSIX31101983' => 'WINDOWS-1252',
  35. 'ANSIX341968' => 'WINDOWS-1252',
  36. 'UNKNOWN8BIT' => 'ISO-8859-15',
  37. 'UNKNOWN' => 'ISO-8859-15',
  38. 'USERDEFINED' => 'ISO-8859-15',
  39. 'KSC56011987' => 'EUC-KR',
  40. 'GB2312' => 'GBK',
  41. 'GB231280' => 'GBK',
  42. 'UNICODE' => 'UTF-8',
  43. 'UTF7IMAP' => 'UTF7-IMAP',
  44. 'TIS620' => 'WINDOWS-874',
  45. 'ISO88599' => 'WINDOWS-1254',
  46. 'ISO885911' => 'WINDOWS-874',
  47. 'MACROMAN' => 'MACINTOSH',
  48. '77' => 'MAC',
  49. '128' => 'SHIFT-JIS',
  50. '129' => 'CP949',
  51. '130' => 'CP1361',
  52. '134' => 'GBK',
  53. '136' => 'BIG5',
  54. '161' => 'WINDOWS-1253',
  55. '162' => 'WINDOWS-1254',
  56. '163' => 'WINDOWS-1258',
  57. '177' => 'WINDOWS-1255',
  58. '178' => 'WINDOWS-1256',
  59. '186' => 'WINDOWS-1257',
  60. '204' => 'WINDOWS-1251',
  61. '222' => 'WINDOWS-874',
  62. '238' => 'WINDOWS-1250',
  63. 'MS950' => 'CP950',
  64. 'WINDOWS949' => 'UHC',
  65. );
  66. /**
  67. * Catch an error and throw an exception.
  68. *
  69. * @param int $errno Level of the error
  70. * @param string $errstr Error message
  71. */
  72. public static function error_handler($errno, $errstr)
  73. {
  74. throw new ErrorException($errstr, 0, $errno);
  75. }
  76. /**
  77. * Parse and validate charset name string (see #1485758).
  78. * Sometimes charset string is malformed, there are also charset aliases
  79. * but we need strict names for charset conversion (specially utf8 class)
  80. *
  81. * @param string $input Input charset name
  82. *
  83. * @return string The validated charset name
  84. */
  85. public static function parse_charset($input)
  86. {
  87. static $charsets = array();
  88. $charset = strtoupper($input);
  89. if (isset($charsets[$input])) {
  90. return $charsets[$input];
  91. }
  92. $charset = preg_replace(array(
  93. '/^[^0-9A-Z]+/', // e.g. _ISO-8859-JP$SIO
  94. '/\$.*$/', // e.g. _ISO-8859-JP$SIO
  95. '/UNICODE-1-1-*/', // RFC1641/1642
  96. '/^X-/', // X- prefix (e.g. X-ROMAN8 => ROMAN8)
  97. ), '', $charset);
  98. if ($charset == 'BINARY') {
  99. return $charsets[$input] = null;
  100. }
  101. // allow A-Z and 0-9 only
  102. $str = preg_replace('/[^A-Z0-9]/', '', $charset);
  103. if (isset(self::$aliases[$str])) {
  104. $result = self::$aliases[$str];
  105. }
  106. // UTF
  107. else if (preg_match('/U[A-Z][A-Z](7|8|16|32)(BE|LE)*/', $str, $m)) {
  108. $result = 'UTF-' . $m[1] . $m[2];
  109. }
  110. // ISO-8859
  111. else if (preg_match('/ISO8859([0-9]{0,2})/', $str, $m)) {
  112. $iso = 'ISO-8859-' . ($m[1] ?: 1);
  113. // some clients sends windows-1252 text as latin1,
  114. // it is safe to use windows-1252 for all latin1
  115. $result = $iso == 'ISO-8859-1' ? 'WINDOWS-1252' : $iso;
  116. }
  117. // handle broken charset names e.g. WINDOWS-1250HTTP-EQUIVCONTENT-TYPE
  118. else if (preg_match('/(WIN|WINDOWS)([0-9]+)/', $str, $m)) {
  119. $result = 'WINDOWS-' . $m[2];
  120. }
  121. // LATIN
  122. else if (preg_match('/LATIN(.*)/', $str, $m)) {
  123. $aliases = array('2' => 2, '3' => 3, '4' => 4, '5' => 9, '6' => 10,
  124. '7' => 13, '8' => 14, '9' => 15, '10' => 16,
  125. 'ARABIC' => 6, 'CYRILLIC' => 5, 'GREEK' => 7, 'GREEK1' => 7, 'HEBREW' => 8
  126. );
  127. // some clients sends windows-1252 text as latin1,
  128. // it is safe to use windows-1252 for all latin1
  129. if ($m[1] == 1) {
  130. $result = 'WINDOWS-1252';
  131. }
  132. // if iconv is not supported we need ISO labels, it's also safe for iconv
  133. else if (!empty($aliases[$m[1]])) {
  134. $result = 'ISO-8859-'.$aliases[$m[1]];
  135. }
  136. // iconv requires convertion of e.g. LATIN-1 to LATIN1
  137. else {
  138. $result = $str;
  139. }
  140. }
  141. else {
  142. $result = $charset;
  143. }
  144. $charsets[$input] = $result;
  145. return $result;
  146. }
  147. /**
  148. * Convert a string from one charset to another.
  149. * Uses mbstring and iconv functions if possible
  150. *
  151. * @param string $str Input string
  152. * @param string $from Suspected charset of the input string
  153. * @param string $to Target charset to convert to; defaults to RCUBE_CHARSET
  154. *
  155. * @return string Converted string
  156. */
  157. public static function convert($str, $from, $to = null)
  158. {
  159. static $iconv_options = null;
  160. static $mbstring_list = null;
  161. static $mbstring_sch = null;
  162. $to = empty($to) ? RCUBE_CHARSET : strtoupper($to);
  163. $from = self::parse_charset($from);
  164. // It is a common case when UTF-16 charset is used with US-ASCII content (#1488654)
  165. // In that case we can just skip the conversion (use UTF-8)
  166. if ($from == 'UTF-16' && !preg_match('/[^\x00-\x7F]/', $str)) {
  167. $from = 'UTF-8';
  168. }
  169. if ($from == $to || empty($str) || empty($from)) {
  170. return $str;
  171. }
  172. if ($iconv_options === null) {
  173. if (function_exists('iconv')) {
  174. // ignore characters not available in output charset
  175. $iconv_options = '//IGNORE';
  176. if (iconv('', $iconv_options, '') === false) {
  177. // iconv implementation does not support options
  178. $iconv_options = '';
  179. }
  180. }
  181. else {
  182. $iconv_options = false;
  183. }
  184. }
  185. // convert charset using iconv module
  186. if ($iconv_options !== false && $from != 'UTF7-IMAP' && $to != 'UTF7-IMAP') {
  187. // throw an exception if iconv reports an illegal character in input
  188. // it means that input string has been truncated
  189. set_error_handler(array('rcube_charset', 'error_handler'), E_NOTICE);
  190. try {
  191. $out = iconv($from, $to . $iconv_options, $str);
  192. }
  193. catch (ErrorException $e) {
  194. $out = false;
  195. }
  196. restore_error_handler();
  197. if ($out !== false) {
  198. return $out;
  199. }
  200. }
  201. if ($mbstring_list === null) {
  202. if (extension_loaded('mbstring')) {
  203. $mbstring_sch = mb_substitute_character();
  204. $mbstring_list = mb_list_encodings();
  205. $mbstring_list = array_map('strtoupper', $mbstring_list);
  206. }
  207. else {
  208. $mbstring_list = false;
  209. }
  210. }
  211. // convert charset using mbstring module
  212. if ($mbstring_list !== false) {
  213. $aliases['WINDOWS-1257'] = 'ISO-8859-13';
  214. // it happens that mbstring supports ASCII but not US-ASCII
  215. if (($from == 'US-ASCII' || $to == 'US-ASCII') && !in_array('US-ASCII', $mbstring_list)) {
  216. $aliases['US-ASCII'] = 'ASCII';
  217. }
  218. $mb_from = $aliases[$from] ?: $from;
  219. $mb_to = $aliases[$to] ?: $to;
  220. // return if encoding found, string matches encoding and convert succeeded
  221. if (in_array($mb_from, $mbstring_list) && in_array($mb_to, $mbstring_list)) {
  222. // Do the same as //IGNORE with iconv
  223. mb_substitute_character('none');
  224. // throw an exception if mbstring reports an illegal character in input
  225. // using mb_check_encoding() is much slower
  226. set_error_handler(array('rcube_charset', 'error_handler'), E_WARNING);
  227. try {
  228. $out = mb_convert_encoding($str, $mb_to, $mb_from);
  229. }
  230. catch (ErrorException $e) {
  231. $out = false;
  232. }
  233. restore_error_handler();
  234. mb_substitute_character($mbstring_sch);
  235. if ($out !== false) {
  236. return $out;
  237. }
  238. }
  239. }
  240. // convert charset using bundled classes/functions
  241. if ($to == 'UTF-8') {
  242. if ($from == 'UTF7-IMAP') {
  243. if ($out = self::utf7imap_to_utf8($str)) {
  244. return $out;
  245. }
  246. }
  247. else if ($from == 'UTF-7') {
  248. if ($out = self::utf7_to_utf8($str)) {
  249. return $out;
  250. }
  251. }
  252. else if ($from == 'ISO-8859-1' && function_exists('utf8_encode')) {
  253. return utf8_encode($str);
  254. }
  255. }
  256. // encode string for output
  257. if ($from == 'UTF-8') {
  258. // @TODO: we need a function for UTF-7 (RFC2152) conversion
  259. if ($to == 'UTF7-IMAP' || $to == 'UTF-7') {
  260. if ($out = self::utf8_to_utf7imap($str)) {
  261. return $out;
  262. }
  263. }
  264. else if ($to == 'ISO-8859-1' && function_exists('utf8_decode')) {
  265. return utf8_decode($str);
  266. }
  267. }
  268. if (!isset($out)) {
  269. trigger_error("No suitable function found for '$from' to '$to' conversion");
  270. }
  271. // return original string
  272. return $str;
  273. }
  274. /**
  275. * Converts string from standard UTF-7 (RFC 2152) to UTF-8.
  276. *
  277. * @param string $str Input string (UTF-7)
  278. *
  279. * @return string Converted string (UTF-8)
  280. */
  281. public static function utf7_to_utf8($str)
  282. {
  283. $Index_64 = array(
  284. 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,
  285. 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,
  286. 0,0,0,0, 0,0,0,0, 0,0,0,1, 0,0,0,0,
  287. 1,1,1,1, 1,1,1,1, 1,1,0,0, 0,0,0,0,
  288. 0,1,1,1, 1,1,1,1, 1,1,1,1, 1,1,1,1,
  289. 1,1,1,1, 1,1,1,1, 1,1,1,0, 0,0,0,0,
  290. 0,1,1,1, 1,1,1,1, 1,1,1,1, 1,1,1,1,
  291. 1,1,1,1, 1,1,1,1, 1,1,1,0, 0,0,0,0,
  292. );
  293. $u7len = strlen($str);
  294. $str = strval($str);
  295. $res = '';
  296. for ($i=0; $u7len > 0; $i++, $u7len--) {
  297. $u7 = $str[$i];
  298. if ($u7 == '+') {
  299. $i++;
  300. $u7len--;
  301. $ch = '';
  302. for (; $u7len > 0; $i++, $u7len--) {
  303. $u7 = $str[$i];
  304. if (!$Index_64[ord($u7)]) {
  305. break;
  306. }
  307. $ch .= $u7;
  308. }
  309. if ($ch == '') {
  310. if ($u7 == '-') {
  311. $res .= '+';
  312. }
  313. continue;
  314. }
  315. $res .= self::utf16_to_utf8(base64_decode($ch));
  316. }
  317. else {
  318. $res .= $u7;
  319. }
  320. }
  321. return $res;
  322. }
  323. /**
  324. * Converts string from UTF-16 to UTF-8 (helper for utf-7 to utf-8 conversion)
  325. *
  326. * @param string $str Input string
  327. *
  328. * @return string The converted string
  329. */
  330. public static function utf16_to_utf8($str)
  331. {
  332. $len = strlen($str);
  333. $dec = '';
  334. for ($i = 0; $i < $len; $i += 2) {
  335. $c = ord($str[$i]) << 8 | ord($str[$i + 1]);
  336. if ($c >= 0x0001 && $c <= 0x007F) {
  337. $dec .= chr($c);
  338. }
  339. else if ($c > 0x07FF) {
  340. $dec .= chr(0xE0 | (($c >> 12) & 0x0F));
  341. $dec .= chr(0x80 | (($c >> 6) & 0x3F));
  342. $dec .= chr(0x80 | (($c >> 0) & 0x3F));
  343. }
  344. else {
  345. $dec .= chr(0xC0 | (($c >> 6) & 0x1F));
  346. $dec .= chr(0x80 | (($c >> 0) & 0x3F));
  347. }
  348. }
  349. return $dec;
  350. }
  351. /**
  352. * Convert the data ($str) from RFC 2060's UTF-7 to UTF-8.
  353. * If input data is invalid, return the original input string.
  354. * RFC 2060 obviously intends the encoding to be unique (see
  355. * point 5 in section 5.1.3), so we reject any non-canonical
  356. * form, such as &ACY- (instead of &-) or &AMA-&AMA- (instead
  357. * of &AMAAwA-).
  358. *
  359. * Translated from C to PHP by Thomas Bruederli <roundcube@gmail.com>
  360. *
  361. * @param string $str Input string (UTF7-IMAP)
  362. *
  363. * @return string Output string (UTF-8)
  364. */
  365. public static function utf7imap_to_utf8($str)
  366. {
  367. $Index_64 = array(
  368. -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1,
  369. -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1,
  370. -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,62, 63,-1,-1,-1,
  371. 52,53,54,55, 56,57,58,59, 60,61,-1,-1, -1,-1,-1,-1,
  372. -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10, 11,12,13,14,
  373. 15,16,17,18, 19,20,21,22, 23,24,25,-1, -1,-1,-1,-1,
  374. -1,26,27,28, 29,30,31,32, 33,34,35,36, 37,38,39,40,
  375. 41,42,43,44, 45,46,47,48, 49,50,51,-1, -1,-1,-1,-1
  376. );
  377. $u7len = strlen($str);
  378. $str = strval($str);
  379. $p = '';
  380. $err = '';
  381. for ($i=0; $u7len > 0; $i++, $u7len--) {
  382. $u7 = $str[$i];
  383. if ($u7 == '&') {
  384. $i++;
  385. $u7len--;
  386. $u7 = $str[$i];
  387. if ($u7len && $u7 == '-') {
  388. $p .= '&';
  389. continue;
  390. }
  391. $ch = 0;
  392. $k = 10;
  393. for (; $u7len > 0; $i++, $u7len--) {
  394. $u7 = $str[$i];
  395. if ((ord($u7) & 0x80) || ($b = $Index_64[ord($u7)]) == -1) {
  396. break;
  397. }
  398. if ($k > 0) {
  399. $ch |= $b << $k;
  400. $k -= 6;
  401. }
  402. else {
  403. $ch |= $b >> (-$k);
  404. if ($ch < 0x80) {
  405. // Printable US-ASCII
  406. if (0x20 <= $ch && $ch < 0x7f) {
  407. return $err;
  408. }
  409. $p .= chr($ch);
  410. }
  411. else if ($ch < 0x800) {
  412. $p .= chr(0xc0 | ($ch >> 6));
  413. $p .= chr(0x80 | ($ch & 0x3f));
  414. }
  415. else {
  416. $p .= chr(0xe0 | ($ch >> 12));
  417. $p .= chr(0x80 | (($ch >> 6) & 0x3f));
  418. $p .= chr(0x80 | ($ch & 0x3f));
  419. }
  420. $ch = ($b << (16 + $k)) & 0xffff;
  421. $k += 10;
  422. }
  423. }
  424. // Non-zero or too many extra bits
  425. if ($ch || $k < 6) {
  426. return $err;
  427. }
  428. // BASE64 not properly terminated
  429. if (!$u7len || $u7 != '-') {
  430. return $err;
  431. }
  432. // Adjacent BASE64 sections
  433. if ($u7len > 2 && $str[$i+1] == '&' && $str[$i+2] != '-') {
  434. return $err;
  435. }
  436. }
  437. // Not printable US-ASCII
  438. else if (ord($u7) < 0x20 || ord($u7) >= 0x7f) {
  439. return $err;
  440. }
  441. else {
  442. $p .= $u7;
  443. }
  444. }
  445. return $p;
  446. }
  447. /**
  448. * Convert the data ($str) from UTF-8 to RFC 2060's UTF-7.
  449. * Unicode characters above U+FFFF are replaced by U+FFFE.
  450. * If input data is invalid, return an empty string.
  451. *
  452. * Translated from C to PHP by Thomas Bruederli <roundcube@gmail.com>
  453. *
  454. * @param string $str Input string (UTF-8)
  455. *
  456. * @return string Output string (UTF7-IMAP)
  457. */
  458. public static function utf8_to_utf7imap($str)
  459. {
  460. $B64Chars = array(
  461. 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O',
  462. 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd',
  463. 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's',
  464. 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7',
  465. '8', '9', '+', ','
  466. );
  467. $u8len = strlen($str);
  468. $base64 = 0;
  469. $i = 0;
  470. $p = '';
  471. $err = '';
  472. while ($u8len) {
  473. $u8 = $str[$i];
  474. $c = ord($u8);
  475. if ($c < 0x80) {
  476. $ch = $c;
  477. $n = 0;
  478. }
  479. else if ($c < 0xc2) {
  480. return $err;
  481. }
  482. else if ($c < 0xe0) {
  483. $ch = $c & 0x1f;
  484. $n = 1;
  485. }
  486. else if ($c < 0xf0) {
  487. $ch = $c & 0x0f;
  488. $n = 2;
  489. }
  490. else if ($c < 0xf8) {
  491. $ch = $c & 0x07;
  492. $n = 3;
  493. }
  494. else if ($c < 0xfc) {
  495. $ch = $c & 0x03;
  496. $n = 4;
  497. }
  498. else if ($c < 0xfe) {
  499. $ch = $c & 0x01;
  500. $n = 5;
  501. }
  502. else {
  503. return $err;
  504. }
  505. $i++;
  506. $u8len--;
  507. if ($n > $u8len) {
  508. return $err;
  509. }
  510. for ($j=0; $j < $n; $j++) {
  511. $o = ord($str[$i+$j]);
  512. if (($o & 0xc0) != 0x80) {
  513. return $err;
  514. }
  515. $ch = ($ch << 6) | ($o & 0x3f);
  516. }
  517. if ($n > 1 && !($ch >> ($n * 5 + 1))) {
  518. return $err;
  519. }
  520. $i += $n;
  521. $u8len -= $n;
  522. if ($ch < 0x20 || $ch >= 0x7f) {
  523. if (!$base64) {
  524. $p .= '&';
  525. $base64 = 1;
  526. $b = 0;
  527. $k = 10;
  528. }
  529. if ($ch & ~0xffff) {
  530. $ch = 0xfffe;
  531. }
  532. $p .= $B64Chars[($b | $ch >> $k)];
  533. $k -= 6;
  534. for (; $k >= 0; $k -= 6) {
  535. $p .= $B64Chars[(($ch >> $k) & 0x3f)];
  536. }
  537. $b = ($ch << (-$k)) & 0x3f;
  538. $k += 16;
  539. }
  540. else {
  541. if ($base64) {
  542. if ($k > 10) {
  543. $p .= $B64Chars[$b];
  544. }
  545. $p .= '-';
  546. $base64 = 0;
  547. }
  548. $p .= chr($ch);
  549. if (chr($ch) == '&') {
  550. $p .= '-';
  551. }
  552. }
  553. }
  554. if ($base64) {
  555. if ($k > 10) {
  556. $p .= $B64Chars[$b];
  557. }
  558. $p .= '-';
  559. }
  560. return $p;
  561. }
  562. /**
  563. * A method to guess character set of a string.
  564. *
  565. * @param string $string String
  566. * @param string $failover Default result for failover
  567. * @param string $language User language
  568. *
  569. * @return string Charset name
  570. */
  571. public static function detect($string, $failover = null, $language = null)
  572. {
  573. if (substr($string, 0, 4) == "\0\0\xFE\xFF") return 'UTF-32BE'; // Big Endian
  574. if (substr($string, 0, 4) == "\xFF\xFE\0\0") return 'UTF-32LE'; // Little Endian
  575. if (substr($string, 0, 2) == "\xFE\xFF") return 'UTF-16BE'; // Big Endian
  576. if (substr($string, 0, 2) == "\xFF\xFE") return 'UTF-16LE'; // Little Endian
  577. if (substr($string, 0, 3) == "\xEF\xBB\xBF") return 'UTF-8';
  578. // heuristics
  579. if ($string[0] == "\0" && $string[1] == "\0" && $string[2] == "\0" && $string[3] != "\0") return 'UTF-32BE';
  580. if ($string[0] != "\0" && $string[1] == "\0" && $string[2] == "\0" && $string[3] == "\0") return 'UTF-32LE';
  581. if ($string[0] == "\0" && $string[1] != "\0" && $string[2] == "\0" && $string[3] != "\0") return 'UTF-16BE';
  582. if ($string[0] != "\0" && $string[1] == "\0" && $string[2] != "\0" && $string[3] == "\0") return 'UTF-16LE';
  583. if (empty($language)) {
  584. $rcube = rcube::get_instance();
  585. $language = $rcube->get_user_language();
  586. }
  587. // Prioritize charsets according to current language (#1485669)
  588. switch ($language) {
  589. case 'ja_JP':
  590. $prio = array('ISO-2022-JP', 'JIS', 'UTF-8', 'EUC-JP', 'eucJP-win', 'SJIS', 'SJIS-win');
  591. break;
  592. case 'zh_CN':
  593. case 'zh_TW':
  594. $prio = array('UTF-8', 'BIG-5', 'GB2312', 'EUC-TW');
  595. break;
  596. case 'ko_KR':
  597. $prio = array('UTF-8', 'EUC-KR', 'ISO-2022-KR');
  598. break;
  599. case 'ru_RU':
  600. $prio = array('UTF-8', 'WINDOWS-1251', 'KOI8-R');
  601. break;
  602. case 'tr_TR':
  603. $prio = array('UTF-8', 'ISO-8859-9', 'WINDOWS-1254');
  604. break;
  605. }
  606. // mb_detect_encoding() is not reliable for some charsets (#1490135)
  607. // use mb_check_encoding() to make charset priority lists really working
  608. if ($prio && function_exists('mb_check_encoding')) {
  609. foreach ($prio as $encoding) {
  610. if (mb_check_encoding($string, $encoding)) {
  611. return $encoding;
  612. }
  613. }
  614. }
  615. if (function_exists('mb_detect_encoding')) {
  616. if (!$prio) {
  617. $prio = array('UTF-8', 'SJIS', 'GB2312',
  618. 'ISO-8859-1', 'ISO-8859-2', 'ISO-8859-3', 'ISO-8859-4',
  619. 'ISO-8859-5', 'ISO-8859-6', 'ISO-8859-7', 'ISO-8859-8', 'ISO-8859-9',
  620. 'ISO-8859-10', 'ISO-8859-13', 'ISO-8859-14', 'ISO-8859-15', 'ISO-8859-16',
  621. 'WINDOWS-1252', 'WINDOWS-1251', 'EUC-JP', 'EUC-TW', 'KOI8-R', 'BIG-5',
  622. 'ISO-2022-KR', 'ISO-2022-JP',
  623. );
  624. }
  625. $encodings = array_unique(array_merge($prio, mb_list_encodings()));
  626. if ($encoding = mb_detect_encoding($string, $encodings)) {
  627. return $encoding;
  628. }
  629. }
  630. // No match, check for UTF-8
  631. // from http://w3.org/International/questions/qa-forms-utf-8.html
  632. if (preg_match('/\A(
  633. [\x09\x0A\x0D\x20-\x7E]
  634. | [\xC2-\xDF][\x80-\xBF]
  635. | \xE0[\xA0-\xBF][\x80-\xBF]
  636. | [\xE1-\xEC\xEE\xEF][\x80-\xBF]{2}
  637. | \xED[\x80-\x9F][\x80-\xBF]
  638. | \xF0[\x90-\xBF][\x80-\xBF]{2}
  639. | [\xF1-\xF3][\x80-\xBF]{3}
  640. | \xF4[\x80-\x8F][\x80-\xBF]{2}
  641. )*\z/xs', substr($string, 0, 2048))
  642. ) {
  643. return 'UTF-8';
  644. }
  645. return $failover;
  646. }
  647. /**
  648. * Removes non-unicode characters from input.
  649. *
  650. * @param mixed $input String or array.
  651. *
  652. * @return mixed String or array
  653. */
  654. public static function clean($input)
  655. {
  656. // handle input of type array
  657. if (is_array($input)) {
  658. foreach ($input as $idx => $val) {
  659. $input[$idx] = self::clean($val);
  660. }
  661. return $input;
  662. }
  663. if (!is_string($input) || $input == '') {
  664. return $input;
  665. }
  666. // iconv/mbstring are much faster (especially with long strings)
  667. if (function_exists('mb_convert_encoding')) {
  668. $msch = mb_substitute_character();
  669. mb_substitute_character('none');
  670. $res = mb_convert_encoding($input, 'UTF-8', 'UTF-8');
  671. mb_substitute_character($msch);
  672. if ($res !== false) {
  673. return $res;
  674. }
  675. }
  676. if (function_exists('iconv')) {
  677. if (($res = @iconv('UTF-8', 'UTF-8//IGNORE', $input)) !== false) {
  678. return $res;
  679. }
  680. }
  681. $seq = '';
  682. $out = '';
  683. $regexp = '/^('.
  684. // '[\x00-\x7F]'. // UTF8-1
  685. '|[\xC2-\xDF][\x80-\xBF]'. // UTF8-2
  686. '|\xE0[\xA0-\xBF][\x80-\xBF]'. // UTF8-3
  687. '|[\xE1-\xEC][\x80-\xBF][\x80-\xBF]'. // UTF8-3
  688. '|\xED[\x80-\x9F][\x80-\xBF]'. // UTF8-3
  689. '|[\xEE-\xEF][\x80-\xBF][\x80-\xBF]'. // UTF8-3
  690. '|\xF0[\x90-\xBF][\x80-\xBF][\x80-\xBF]'. // UTF8-4
  691. '|[\xF1-\xF3][\x80-\xBF][\x80-\xBF][\x80-\xBF]'.// UTF8-4
  692. '|\xF4[\x80-\x8F][\x80-\xBF][\x80-\xBF]'. // UTF8-4
  693. ')$/';
  694. for ($i = 0, $len = strlen($input); $i < $len; $i++) {
  695. $chr = $input[$i];
  696. $ord = ord($chr);
  697. // 1-byte character
  698. if ($ord <= 0x7F) {
  699. if ($seq !== '') {
  700. $out .= preg_match($regexp, $seq) ? $seq : '';
  701. $seq = '';
  702. }
  703. $out .= $chr;
  704. }
  705. // first byte of multibyte sequence
  706. else if ($ord >= 0xC0) {
  707. if ($seq !== '') {
  708. $out .= preg_match($regexp, $seq) ? $seq : '';
  709. $seq = '';
  710. }
  711. $seq = $chr;
  712. }
  713. // next byte of multibyte sequence
  714. else if ($seq !== '') {
  715. $seq .= $chr;
  716. }
  717. }
  718. if ($seq !== '') {
  719. $out .= preg_match($regexp, $seq) ? $seq : '';
  720. }
  721. return $out;
  722. }
  723. }