Personal
/
cops
1
0
Fork 0
Code Issues 0 Pull Requests 0 Releases 0 Wiki Activity
Browse Source

Allow normalized search (without diacritics) it should also fix the case problem with non latin languages (cyrillic). 

The last part is not tested.

disclaimer : Slow !

re #49, #48
master
Sébastien Lucas 10 years ago
parent
23fcc4d641
commit
78d42b48e2
5 changed files with 148 additions and 7 deletions
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. +95
    -6
      base.php
  2. +36
    -0
      checkconfig.php
  3. +7
    -0
      config_default.php
  4. +4
    -0
      test/baseTest.php
  5. +6
    -1
      test/pageTest.php

+ 95
- 6
base.php View File

@@ -256,6 +256,83 @@ function addURLParameter($urlParams, $paramName, $paramValue) {
return $start . http_build_query($params);
}

function useNormAndUp () {
global $config;
return extension_loaded('mbstring') &&
extension_loaded('intl') &&
class_exists("Normalizer", $autoload = false) &&
$config ['cops_normalized_search'] == "1";
}

function normalizeUtf8String( $s)
{
$original_string = $s;

// maps German (umlauts) and other European characters onto two characters before just removing diacritics
$s = preg_replace( '@\x{00c4}@u' , "AE", $s ); // umlaut Ä => AE
$s = preg_replace( '@\x{00d6}@u' , "OE", $s ); // umlaut Ö => OE
$s = preg_replace( '@\x{00dc}@u' , "UE", $s ); // umlaut Ü => UE
$s = preg_replace( '@\x{00e4}@u' , "ae", $s ); // umlaut ä => ae
$s = preg_replace( '@\x{00f6}@u' , "oe", $s ); // umlaut ö => oe
$s = preg_replace( '@\x{00fc}@u' , "ue", $s ); // umlaut ü => ue
$s = preg_replace( '@\x{00f1}@u' , "ny", $s ); // ñ => ny
$s = preg_replace( '@\x{00ff}@u' , "yu", $s ); // ÿ => yu


// maps special characters (characters with diacritics) on their base-character followed by the diacritical mark
// exmaple: Ú => U´, á => a`
$s = Normalizer::normalize( $s, Normalizer::FORM_D );


$s = preg_replace( '@\pM@u' , "", $s ); // removes diacritics


$s = preg_replace( '@\x{00df}@u' , "ss", $s ); // maps German ß onto ss
$s = preg_replace( '@\x{00c6}@u' , "AE", $s ); // Æ => AE
$s = preg_replace( '@\x{00e6}@u' , "ae", $s ); // æ => ae
$s = preg_replace( '@\x{0132}@u' , "IJ", $s ); // ? => IJ
$s = preg_replace( '@\x{0133}@u' , "ij", $s ); // ? => ij
$s = preg_replace( '@\x{0152}@u' , "OE", $s ); // Π=> OE
$s = preg_replace( '@\x{0153}@u' , "oe", $s ); // œ => oe

$s = preg_replace( '@\x{00d0}@u' , "D", $s ); // Ð => D
$s = preg_replace( '@\x{0110}@u' , "D", $s ); // Ð => D
$s = preg_replace( '@\x{00f0}@u' , "d", $s ); // ð => d
$s = preg_replace( '@\x{0111}@u' , "d", $s ); // d => d
$s = preg_replace( '@\x{0126}@u' , "H", $s ); // H => H
$s = preg_replace( '@\x{0127}@u' , "h", $s ); // h => h
$s = preg_replace( '@\x{0131}@u' , "i", $s ); // i => i
$s = preg_replace( '@\x{0138}@u' , "k", $s ); // ? => k
$s = preg_replace( '@\x{013f}@u' , "L", $s ); // ? => L
$s = preg_replace( '@\x{0141}@u' , "L", $s ); // L => L
$s = preg_replace( '@\x{0140}@u' , "l", $s ); // ? => l
$s = preg_replace( '@\x{0142}@u' , "l", $s ); // l => l
$s = preg_replace( '@\x{014a}@u' , "N", $s ); // ? => N
$s = preg_replace( '@\x{0149}@u' , "n", $s ); // ? => n
$s = preg_replace( '@\x{014b}@u' , "n", $s ); // ? => n
$s = preg_replace( '@\x{00d8}@u' , "O", $s ); // Ø => O
$s = preg_replace( '@\x{00f8}@u' , "o", $s ); // ø => o
$s = preg_replace( '@\x{017f}@u' , "s", $s ); // ? => s
$s = preg_replace( '@\x{00de}@u' , "T", $s ); // Þ => T
$s = preg_replace( '@\x{0166}@u' , "T", $s ); // T => T
$s = preg_replace( '@\x{00fe}@u' , "t", $s ); // þ => t
$s = preg_replace( '@\x{0167}@u' , "t", $s ); // t => t

// remove all non-ASCii characters
$s = preg_replace( '@[^\0-\x80]@u' , "", $s );


// possible errors in UTF8-regular-expressions
if (empty($s))
return $original_string;
else
return $s;
}

function normAndUp ($a) {
return mb_strtoupper (normalizeUtf8String($a), 'UTF-8');
}

class Link
{
const OPDS_THUMBNAIL_TYPE = "http://opds-spec.org/image/thumbnail";
@@ -792,29 +869,33 @@ class PageQueryResult extends Page
private function searchByScope ($scope, $limit = FALSE) {
$n = $this->n;
$numberPerPage = NULL;
$queryNormedAndUp = $this->query;
if (useNormAndUp ()) {
$queryNormedAndUp = normAndUp ($this->query);
}
if ($limit) {
$n = 1;
$numberPerPage = 5;
}
switch ($scope) {
case self::SCOPE_BOOK :
$array = Book::getBooksByStartingLetter ('%' . $this->query, $n, NULL, $numberPerPage);
$array = Book::getBooksByStartingLetter ('%' . $queryNormedAndUp, $n, NULL, $numberPerPage);
break;
case self::SCOPE_AUTHOR :
$array = Author::getAuthorsForSearch ('%' . $this->query);
$array = Author::getAuthorsForSearch ('%' . $queryNormedAndUp);
break;
case self::SCOPE_SERIES :
$array = Serie::getAllSeriesByQuery ($this->query);
$array = Serie::getAllSeriesByQuery ($queryNormedAndUp);
break;
case self::SCOPE_TAG :
$array = Tag::getAllTagsByQuery ($this->query, $n, NULL, $numberPerPage);
$array = Tag::getAllTagsByQuery ($queryNormedAndUp, $n, NULL, $numberPerPage);
break;
case self::SCOPE_PUBLISHER :
$array = Publisher::getAllPublishersByQuery ($this->query);
$array = Publisher::getAllPublishersByQuery ($queryNormedAndUp);
break;
default:
$array = Book::getBooksByQuery (
array ("all" => "%" . $this->query . "%"), $n);
array ("all" => "%" . $queryNormedAndUp . "%"), $n);
}

return $array;
@@ -1138,6 +1219,9 @@ abstract class Base
try {
if (is_readable (self::getDbFileName ($database))) {
self::$db = new PDO('sqlite:'. self::getDbFileName ($database));
if (useNormAndUp ()) {
self::$db->sqliteCreateFunction ('normAndUp', 'normAndUp', 1);
}
} else {
self::error ();
}
@@ -1167,6 +1251,11 @@ abstract class Base
public static function executeQuery($query, $columns, $filter, $params, $n, $database = NULL, $numberPerPage = NULL) {
$totalResult = -1;

if (useNormAndUp ()) {
$query = preg_replace("/upper/", "normAndUp", $query);
$columns = preg_replace("/upper/", "normAndUp", $columns);
}

if (is_null ($numberPerPage)) {
$numberPerPage = getCurrentOption ("max_item_per_page");
}


+ 36
- 0
checkconfig.php View File

@@ -85,6 +85,42 @@
?>
</h4>
</article>
<article class="frontpage">
<h2>Check if mbstring is properly installed and loaded</h2>
<h4>
<?php
if (extension_loaded('mbstring')) {
echo "OK";
} else {
echo "Please install the php5-mbstring extension and make sure it's enabled";
}
?>
</h4>
</article>
<article class="frontpage">
<h2>Check if intl is properly installed and loaded</h2>
<h4>
<?php
if (extension_loaded('intl')) {
echo "OK";
} else {
echo "Please install the php5-intl extension and make sure it's enabled";
}
?>
</h4>
</article>
<article class="frontpage">
<h2>Check if Normalizer class is properly installed and loaded</h2>
<h4>
<?php
if (class_exists("Normalizer", $autoload = false)) {
echo "OK";
} else {
echo "Please make sure intl is enabled in your php.ini";
}
?>
</h4>
</article>
<article class="frontpage">
<h2>Check if the rendering will be done on client side or server side</h2>
<h4>


+ 7
- 0
config_default.php View File

@@ -258,3 +258,10 @@
* BEWARE : Do not touch this if you're not using password, not using PRS-TX or not using Aldiko.
*/
$config ['cops_fetch_protect'] = "0";

/*
* Make the search better (don't care about diacritics, uppercase should work on Cyrillic) but slower.
* 1 : Yes (enable)
* 0 : No
*/
$config ['cops_normalized_search'] = "0";

+ 4
- 0
test/baseTest.php View File

@@ -125,4 +125,8 @@ class BaseTest extends PHPUnit_Framework_TestCase

$this->assertTrue (Base::checkDatabaseAvailability ());
}

public function testNormalizeUtf8String () {
$this->assertEquals ("AAAAAEACEEEEIIIIOEOOOOOEUUUUEYaaaaaeaceeeeiiiioedoooooeuuuueyyuny", normalizeUtf8String ("ÀÁÂÃÄÅÇÈÉÊËÌÍÎÏŒÒÓÔÕÖÙÚÛÜÝàáâãäåçèéêëìíîïœðòóôõöùúûüýÿñ"));
}
}

+ 6
- 1
test/pageTest.php View File

@@ -669,16 +669,21 @@ class PageTest extends PHPUnit_Framework_TestCase
$query = "curee";
$qid = NULL;
$n = "1";
$config ['cops_normalized_search'] = "1";
if (!useNormAndUp ()) {
$this->markTestIncomplete();
}

$currentPage = Page::getPage ($page, $qid, $query, $n);
$currentPage->InitializeContent ();

$this->markTestIncomplete();
$this->assertEquals ("Search result for *curee*", $currentPage->title);
$this->assertCount (1, $currentPage->entryArray);
$this->assertEquals ("Search result for *curee* in books", $currentPage->entryArray [0]->title);
$this->assertEquals ("1 book", $currentPage->entryArray [0]->content);
$this->assertFalse ($currentPage->ContainsBook ());

$config ['cops_normalized_search'] = "0";
}

public function testAuthorSearch_ByName ()


Write Preview
Loading…
Cancel
Save