Gecko [ www.jdcpatala.org ]

Name	Size	Permission
libmbfl	[ DIR ]	drwxr-xr-x
tests	[ DIR ]	drwxr-xr-x
ucgendat	[ DIR ]	drwxr-xr-x
CREDITS	56 B	-rw-r--r--
common_codepoints.txt	96.72 KB	-rw-r--r--
config.m4	6.01 KB	-rw-r--r--
config.w32	3.17 KB	-rw-r--r--
gen_rare_cp_bitvec.php	1.46 KB	-rw-r--r--
mb_gpc.c	9.95 KB	-rw-r--r--
mb_gpc.h	1.74 KB	-rw-r--r--
mbstring.c	121.96 KB	-rw-r--r--
mbstring.h	4.78 KB	-rw-r--r--
mbstring.stub.php	6.63 KB	-rw-r--r--
mbstring_arginfo.h	18.29 KB	-rw-r--r--
php_mbregex.c	40.31 KB	-rw-r--r--
php_mbregex.h	1.69 KB	-rw-r--r--
php_onig_compat.h	256 B	-rw-r--r--
php_unicode.c	11.7 KB	-rw-r--r--
php_unicode.h	7.58 KB	-rw-r--r--
rare_cp_bitvec.h	24.5 KB	-rw-r--r--
unicode_data.h	249.2 KB	-rw-r--r--

Code Editor : gen_rare_cp_bitvec.php

#!/usr/bin/env php
<?php

if ($argc < 2) {
    echo "Usage: php gen_rare_cp_bitvec.php ./common_codepoints.txt\n";
    return;
}

$bitvec = array_fill(0, (0xFFFF / 32) + 1, 0xFFFFFFFF);

$input = file_get_contents($argv[1]);
foreach (explode("\n", $input) as $line) {
    if (false !== $hashPos = strpos($line, '#')) {
        $line = substr($line, 0, $hashPos);
    }

$line = trim($line);
    if ($line === '') {
        continue;
    }

$range = explode("\t", $line);
    $start = hexdec($range[0]);
    $end   = hexdec($range[1]);

for ($i = $start; $i <= $end; $i++) {
        $bitvec[$i >> 5] &= ~(1 << ($i & 0x1F));
    }
}

$result = <<<'HEADER'
/* Machine-generated file; do not edit! See gen_rare_cp_bitvec.php.
 *
 * The below array has one bit for each Unicode codepoint from U+0000 to U+FFFF.
 * The bit is 1 if the codepoint is considered 'rare' for the purpose of
 * guessing the text encoding of a string.
 *
 * Each 'rare' codepoint which appears in a string when it is interpreted
 * using a candidate encoding causes the candidate encoding to be treated
 * as less likely to be the correct one.
 */

static uint32_t rare_codepoint_bitvec[] = {
HEADER;

for ($i = 0; $i < 0xFFFF / 32; $i++) {
    if ($i % 8 === 0) {
        $result .= "\n";
    } else {
        $result .= " ";
    }

$result .= "0x" . str_pad(dechex($bitvec[$i]), 8, '0', STR_PAD_LEFT) . ",";
}

$result .= "\n};\n";

file_put_contents(__DIR__ . '/rare_cp_bitvec.h', $result);

echo "Done.\n";
?>

${this.title}

Code Editor : gen_rare_cp_bitvec.php