Personal/drupal-civicrm
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

First commit

Browse source
This commit is contained in:
Theodotos Andreou 2018-01-14 13:10:16 +00:00
commit c6e2478c40
13918 changed files with 2303184 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

49
sites/all/modules/civicrm/CRM/Dedupe/BAO/QueryBuilder.php Normal file
View file

@ -0,0 +1,49 @@
<?php
/**
* Class CRM_Dedupe_BAO_QueryBuilder
*/
class CRM_Dedupe_BAO_QueryBuilder {
/**
* @param $rg
* @param string $strID1
* @param string $strID2
*
* @return string
*/
public static function internalFilters($rg, $strID1 = 'contact1.id', $strID2 = 'contact2.id') {
// Add a contact id filter for dedupe by group requests and add logic
// to remove duplicate results with opposing orders, i.e. 1,2 and 2,1
if (!empty($rg->contactIds)) {
$cids = implode(',', $rg->contactIds);
return "($strID1 IN ($cids) AND ( $strID2 NOT IN ($cids) OR ($strID2 IN ($cids) AND $strID1 < $strID2) ))";
}
else {
return "($strID1 < $strID2)";
}
}
/**
* If a contact list is specified then adjust the query to ensure one contact is in that list.
*
* Doing an OR join here will lead to a server-killing unindexed query. However, a union will
* perform better.
*
* @param array $contactList
* @param string $query
* @param string $strID1
* @param string $strID2
*
* @return string
*/
protected static function filterQueryByContactList(array $contactList, $query, $strID1 = 'contact1.id', $strID2 = 'contact2.id') {
if (empty($contactList)) {
return $query . " AND ($strID1 < $strID2)";
}
$contactIDs = implode(',', $contactList);
return "$query AND $strID1 IN ($contactIDs) AND $strID1 > $strID2
UNION $query AND $strID1 > $strID2 AND $strID2 IN ($contactIDs) AND $strID1 NOT IN ($contactIDs)
";
}
}

72
sites/all/modules/civicrm/CRM/Dedupe/BAO/QueryBuilder/IndividualGeneral.php Normal file
View file

@ -0,0 +1,72 @@
<?php
/**
* TODO: How to handle NULL values/records?
* Class CRM_Dedupe_BAO_QueryBuilder_IndividualGeneral
*/
class CRM_Dedupe_BAO_QueryBuilder_IndividualGeneral extends CRM_Dedupe_BAO_QueryBuilder {
/**
* @param $rg
*
* @return array
*/
public static function record($rg) {
$civicrm_contact = CRM_Utils_Array::value('civicrm_contact', $rg->params);
$civicrm_address = CRM_Utils_Array::value('civicrm_address', $rg->params);
// Since definitely have first and last name, escape them upfront.
$first_name = CRM_Core_DAO::escapeString(CRM_Utils_Array::value('first_name', $civicrm_contact, ''));
$last_name = CRM_Core_DAO::escapeString(CRM_Utils_Array::value('last_name', $civicrm_contact, ''));
$street_address = CRM_Core_DAO::escapeString(CRM_Utils_Array::value('street_address', $civicrm_address, ''));
$query = "
SELECT contact1.id id1, {$rg->threshold} as weight
FROM civicrm_contact AS contact1
JOIN civicrm_address AS address1 ON contact1.id=address1.contact_id
WHERE contact1.contact_type = 'Individual'
AND contact1.first_name = '$first_name'
AND contact1.last_name = '$last_name'
AND address1.street_address = '$street_address'
";
if ($birth_date = CRM_Core_DAO::escapeString(CRM_Utils_Array::value('birth_date', $civicrm_contact, ''))) {
$query .= " AND (contact1.birth_date IS NULL or contact1.birth_date = '$birth_date')\n";
}
if ($suffix_id = CRM_Core_DAO::escapeString(CRM_Utils_Array::value('suffix_id', $civicrm_contact, ''))) {
$query .= " AND (contact1.suffix_id IS NULL or contact1.suffix_id = $suffix_id)\n";
}
if ($middle_name = CRM_Core_DAO::escapeString(CRM_Utils_Array::value('middle_name', $civicrm_contact, ''))) {
$query .= " AND (contact1.middle_name IS NULL or contact1.middle_name = '$middle_name')\n";
}
return array("civicrm_contact.{$rg->name}.{$rg->threshold}" => $query);
}
/**
* @param $rg
*
* @return array
*/
public static function internal($rg) {
$query = "
SELECT contact1.id id1, contact2.id id2, {$rg->threshold} weight
FROM civicrm_contact AS contact1
JOIN civicrm_contact AS contact2 ON (
contact1.first_name = contact2.first_name AND
contact1.last_name = contact2.last_name AND
contact1.contact_type = contact2.contact_type)
JOIN civicrm_address AS address1 ON address1.contact_id = contact1.id
JOIN civicrm_address AS address2 ON (
address2.contact_id = contact2.id AND
address2.street_address = address1.street_address)
WHERE contact1.contact_type = 'Individual'
AND (contact1.suffix_id IS NULL OR contact2.suffix_id IS NULL OR contact1.suffix_id = contact2.suffix_id)
AND (contact1.middle_name IS NULL OR contact2.middle_name IS NULL OR contact1.middle_name = contact2.middle_name)
AND (contact1.birth_date IS NULL OR contact2.birth_date IS NULL OR contact1.birth_date = contact2.birth_date)
AND " . self::internalFilters($rg);
return array("civicrm_contact.{$rg->name}.{$rg->threshold}" => $query);
}
}

73
sites/all/modules/civicrm/CRM/Dedupe/BAO/QueryBuilder/IndividualSupervised.php Normal file
View file

@ -0,0 +1,73 @@
<?php
/**
* TODO: How to handle NULL values/records?
* Class CRM_Dedupe_BAO_QueryBuilder_IndividualSupervised
*/
class CRM_Dedupe_BAO_QueryBuilder_IndividualSupervised extends CRM_Dedupe_BAO_QueryBuilder {
/**
* Record - what do I do.
*
* @param object $rg
*
* @return array
*/
public static function record($rg) {
$civicrm_contact = CRM_Utils_Array::value('civicrm_contact', $rg->params, array());
$civicrm_email = CRM_Utils_Array::value('civicrm_email', $rg->params, array());
$params = array(
1 => array(
CRM_Utils_Array::value('first_name', $civicrm_contact, ''),
'String',
),
2 => array(
CRM_Utils_Array::value('last_name', $civicrm_contact, ''),
'String',
),
3 => array(
CRM_Utils_Array::value('email', $civicrm_email, ''),
'String',
),
);
return array(
"civicrm_contact.{$rg->name}.{$rg->threshold}" => CRM_Core_DAO::composeQuery("
SELECT contact.id as id1, {$rg->threshold} as weight
FROM civicrm_contact as contact
JOIN civicrm_email as email ON email.contact_id=contact.id
WHERE contact_type = 'Individual'
AND first_name = %1
AND last_name = %2
AND email = %3", $params, TRUE),
);
}
/**
* Internal - what do I do.
*
* @param object $rg
*
* @return array
*/
public static function internal($rg) {
$query = self::filterQueryByContactList($rg->contactIds, "
SELECT contact1.id as id1, contact2.id as id2, {$rg->threshold} as weight
FROM civicrm_contact as contact1
JOIN civicrm_email as email1 ON email1.contact_id=contact1.id
JOIN civicrm_contact as contact2 ON
contact1.first_name = contact2.first_name AND
contact1.last_name = contact2.last_name
JOIN civicrm_email as email2 ON
email2.contact_id=contact2.id AND
email1.email=email2.email
WHERE contact1.contact_type = 'Individual'");
return array(
"civicrm_contact.{$rg->name}.{$rg->threshold}" => $query,
);
}
}

115
sites/all/modules/civicrm/CRM/Dedupe/BAO/QueryBuilder/IndividualUnsupervised.php Normal file
View file

@ -0,0 +1,115 @@
<?php
/*
+--------------------------------------------------------------------+
| CiviCRM version 4.7 |
+--------------------------------------------------------------------+
| Copyright CiviCRM LLC (c) 2004-2017 |
+--------------------------------------------------------------------+
| This file is a part of CiviCRM. |
| |
| CiviCRM is free software; you can copy, modify, and distribute it |
| under the terms of the GNU Affero General Public License |
| Version 3, 19 November 2007 and the CiviCRM Licensing Exception. |
| |
| CiviCRM is distributed in the hope that it will be useful, but |
| WITHOUT ANY WARRANTY; without even the implied warranty of |
| MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. |
| See the GNU Affero General Public License for more details. |
| |
| You should have received a copy of the GNU Affero General Public |
| License and the CiviCRM Licensing Exception along |
| with this program; if not, contact CiviCRM LLC |
| at info[AT]civicrm[DOT]org. If you have questions about the |
| GNU Affero General Public License or the licensing of CiviCRM, |
| see the CiviCRM license FAQ at http://civicrm.org/licensing |
+--------------------------------------------------------------------+
*/
/**
* Class CRM_Dedupe_BAO_QueryBuilder_IndividualUnsupervised
*/
class CRM_Dedupe_BAO_QueryBuilder_IndividualUnsupervised extends CRM_Dedupe_BAO_QueryBuilder {
/**
* @param $rg
*
* @return array
*/
public static function record($rg) {
$civicrm_email = CRM_Utils_Array::value('civicrm_email', $rg->params, array());
$params = array(
1 => array(CRM_Utils_Array::value('email', $civicrm_email, ''), 'String'),
);
return array(
"civicrm_contact.{$rg->name}.{$rg->threshold}" => CRM_Core_DAO::composeQuery("
SELECT contact.id as id1, {$rg->threshold} as weight
FROM civicrm_contact as contact
JOIN civicrm_email as email ON email.contact_id=contact.id
WHERE contact_type = 'Individual'
AND email = %1", $params, TRUE),
);
}
/**
* @param $rg
*
* @return array
*/
public static function internal($rg) {
$query = "
SELECT contact1.id as id1, contact2.id as id2, {$rg->threshold} as weight
FROM civicrm_contact as contact1
JOIN civicrm_email as email1 ON email1.contact_id=contact1.id
JOIN civicrm_contact as contact2
JOIN civicrm_email as email2 ON
email2.contact_id=contact2.id AND
email1.email=email2.email
WHERE contact1.contact_type = 'Individual'
AND " . self::internalFilters($rg);
return array("civicrm_contact.{$rg->name}.{$rg->threshold}" => $query);
}
/**
* An alternative version which might perform a lot better
* than the above. Will need to do some testing
*
* @param string $rg
*
* @return array
*/
public static function internalOptimized($rg) {
$sql = "
CREATE TEMPORARY TABLE emails (
email varchar(255),
contact_id1 int,
contact_id2 int,
INDEX(contact_id1),
INDEX(contact_id2)
) ENGINE=InnoDB
";
CRM_Core_DAO::executeQuery($sql);
$sql = "
INSERT INTO emails
SELECT email1.email as email, email1.contact_id as contact_id1, email2.contact_id as contact_id2
FROM civicrm_email as email1
JOIN civicrm_email as email2 USING (email)
WHERE email1.contact_id < email2.contact_id
AND " . self::internalFilters($rg, "email1.contact_id", "email2.contact_id");
CRM_Core_DAO::executeQuery($sql);
$query = "
SELECT contact_id1 as id1, contact_id2 as id2, {$rg->threshold} as weight
FROM emails
JOIN civicrm_contact as contact1 on contact1.id=contact_id1
JOIN civicrm_contact as contact2 on contact2.id=contact_id2
WHERE contact1.contact_type='Individual'
AND contact2.contact_type='Individual'
AND " . self::internalFilters($rg);
return array("civicrm_contact.{$rg->name}.{$rg->threshold}" => $query);
}
}

243
sites/all/modules/civicrm/CRM/Dedupe/BAO/Rule.php Normal file
View file

@ -0,0 +1,243 @@
<?php
/*
+--------------------------------------------------------------------+
| CiviCRM version 4.7 |
+--------------------------------------------------------------------+
| Copyright CiviCRM LLC (c) 2004-2017 |
+--------------------------------------------------------------------+
| This file is a part of CiviCRM. |
| |
| CiviCRM is free software; you can copy, modify, and distribute it |
| under the terms of the GNU Affero General Public License |
| Version 3, 19 November 2007 and the CiviCRM Licensing Exception. |
| |
| CiviCRM is distributed in the hope that it will be useful, but |
| WITHOUT ANY WARRANTY; without even the implied warranty of |
| MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. |
| See the GNU Affero General Public License for more details. |
| |
| You should have received a copy of the GNU Affero General Public |
| License and the CiviCRM Licensing Exception along |
| with this program; if not, contact CiviCRM LLC |
| at info[AT]civicrm[DOT]org. If you have questions about the |
| GNU Affero General Public License or the licensing of CiviCRM, |
| see the CiviCRM license FAQ at http://civicrm.org/licensing |
+--------------------------------------------------------------------+
*/
/**
*
* @package CRM
* @copyright CiviCRM LLC (c) 2004-2017
* $Id$
*
*/
/**
* The CiviCRM duplicate discovery engine is based on an
* algorithm designed by David Strauss <david@fourkitchens.com>.
*/
class CRM_Dedupe_BAO_Rule extends CRM_Dedupe_DAO_Rule {
/**
* Ids of the contacts to limit the SQL queries (whole-database queries otherwise)
*/
var $contactIds = array();
/**
* Params to dedupe against (queries against the whole contact set otherwise)
*/
var $params = array();
/**
* Return the SQL query for the given rule - either for finding matching
* pairs of contacts, or for matching against the $params variable (if set).
*
* @return string
* SQL query performing the search
*/
public function sql() {
if ($this->params &&
(!array_key_exists($this->rule_table, $this->params) ||
!array_key_exists($this->rule_field, $this->params[$this->rule_table])
)
) {
// if params is present and doesn't have an entry for a field, don't construct the clause.
return NULL;
}
// we need to initialise WHERE, ON and USING here, as some table types
// extend them; $where is an array of required conditions, $on and
// $using are arrays of required field matchings (for substring and
// full matches, respectively)
$where = array();
$on = array("SUBSTR(t1.{$this->rule_field}, 1, {$this->rule_length}) = SUBSTR(t2.{$this->rule_field}, 1, {$this->rule_length})");
$using = array($this->rule_field);
switch ($this->rule_table) {
case 'civicrm_contact':
$id = 'id';
//we should restrict by contact type in the first step
$sql = "SELECT contact_type FROM civicrm_dedupe_rule_group WHERE id = {$this->dedupe_rule_group_id};";
$ct = CRM_Core_DAO::singleValueQuery($sql);
if ($this->params) {
$where[] = "t1.contact_type = '{$ct}'";
}
else {
$where[] = "t1.contact_type = '{$ct}'";
$where[] = "t2.contact_type = '{$ct}'";
}
break;
case 'civicrm_address':
$id = 'contact_id';
$on[] = 't1.location_type_id = t2.location_type_id';
$using[] = 'location_type_id';
if ($this->params['civicrm_address']['location_type_id']) {
$locTypeId = CRM_Utils_Type::escape($this->params['civicrm_address']['location_type_id'], 'Integer', FALSE);
if ($locTypeId) {
$where[] = "t1.location_type_id = $locTypeId";
}
}
break;
case 'civicrm_email':
case 'civicrm_im':
case 'civicrm_openid':
case 'civicrm_phone':
$id = 'contact_id';
break;
case 'civicrm_note':
$id = 'entity_id';
if ($this->params) {
$where[] = "t1.entity_table = 'civicrm_contact'";
}
else {
$where[] = "t1.entity_table = 'civicrm_contact'";
$where[] = "t2.entity_table = 'civicrm_contact'";
}
break;
default:
// custom data tables
if (preg_match('/^civicrm_value_/', $this->rule_table) || preg_match('/^custom_value_/', $this->rule_table)) {
$id = 'entity_id';
}
else {
CRM_Core_Error::fatal("Unsupported rule_table for civicrm_dedupe_rule.id of {$this->id}");
}
break;
}
// build SELECT based on the field names containing contact ids
// if there are params provided, id1 should be 0
if ($this->params) {
$select = "t1.$id id1, {$this->rule_weight} weight";
$subSelect = 'id1, weight';
}
else {
$select = "t1.$id id1, t2.$id id2, {$this->rule_weight} weight";
$subSelect = 'id1, id2, weight';
}
// build FROM (and WHERE, if it's a parametrised search)
// based on whether the rule is about substrings or not
if ($this->params) {
$from = "{$this->rule_table} t1";
$str = 'NULL';
if (isset($this->params[$this->rule_table][$this->rule_field])) {
$str = CRM_Utils_Type::escape($this->params[$this->rule_table][$this->rule_field], 'String');
}
if ($this->rule_length) {
$where[] = "SUBSTR(t1.{$this->rule_field}, 1, {$this->rule_length}) = SUBSTR('$str', 1, {$this->rule_length})";
$where[] = "t1.{$this->rule_field} IS NOT NULL";
$where[] = "t1.{$this->rule_field} <> ''";
}
else {
$where[] = "t1.{$this->rule_field} = '$str'";
}
}
else {
if ($this->rule_length) {
$from = "{$this->rule_table} t1 JOIN {$this->rule_table} t2 ON (" . implode(' AND ', $on) . ")";
}
else {
$from = "{$this->rule_table} t1 JOIN {$this->rule_table} t2 USING (" . implode(', ', $using) . ")";
}
}
// finish building WHERE, also limit the results if requested
if (!$this->params) {
$where[] = "t1.$id < t2.$id";
$where[] = "t1.{$this->rule_field} IS NOT NULL";
$where[] = "t1.{$this->rule_field} <> ''";
}
$query = "SELECT $select FROM $from WHERE " . implode(' AND ', $where);
if ($this->contactIds) {
$cids = array();
foreach ($this->contactIds as $cid) {
$cids[] = CRM_Utils_Type::escape($cid, 'Integer');
}
if (count($cids) == 1) {
$query .= " AND (t1.$id = {$cids[0]}) UNION $query AND t2.$id = {$cids[0]}";
}
else {
$query .= " AND t1.$id IN (" . implode(',', $cids) . ")
UNION $query AND t2.$id IN (" . implode(',', $cids) . ")";
}
// The `weight` is ambiguous in the context of the union; put the whole
// thing in a subquery.
$query = "SELECT $subSelect FROM ($query) subunion";
}
return $query;
}
/**
* find fields related to a rule group.
*
* @param array $params contains the rule group property to identify rule group
*
* @return array
* rule fields array associated to rule group
*/
public static function dedupeRuleFields($params) {
$rgBao = new CRM_Dedupe_BAO_RuleGroup();
$rgBao->used = $params['used'];
$rgBao->contact_type = $params['contact_type'];
$rgBao->find(TRUE);
$ruleBao = new CRM_Dedupe_BAO_Rule();
$ruleBao->dedupe_rule_group_id = $rgBao->id;
$ruleBao->find();
$ruleFields = array();
while ($ruleBao->fetch()) {
$ruleFields[] = $ruleBao->rule_field;
}
return $ruleFields;
}
/**
* @param int $cid
* @param int $oid
*
* @return bool
*/
public static function validateContacts($cid, $oid) {
if (!$cid || !$oid) {
return NULL;
}
$exception = new CRM_Dedupe_DAO_Exception();
$exception->contact_id1 = $cid;
$exception->contact_id2 = $oid;
//make sure contact2 > contact1.
if ($cid > $oid) {
$exception->contact_id1 = $oid;
$exception->contact_id2 = $cid;
}
return $exception->find(TRUE) ? FALSE : TRUE;
}
}

514
sites/all/modules/civicrm/CRM/Dedupe/BAO/RuleGroup.php Normal file
View file

@ -0,0 +1,514 @@
<?php
/*
+--------------------------------------------------------------------+
| CiviCRM version 4.7 |
+--------------------------------------------------------------------+
| Copyright CiviCRM LLC (c) 2004-2017 |
+--------------------------------------------------------------------+
| This file is a part of CiviCRM. |
| |
| CiviCRM is free software; you can copy, modify, and distribute it |
| under the terms of the GNU Affero General Public License |
| Version 3, 19 November 2007 and the CiviCRM Licensing Exception. |
| |
| CiviCRM is distributed in the hope that it will be useful, but |
| WITHOUT ANY WARRANTY; without even the implied warranty of |
| MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. |
| See the GNU Affero General Public License for more details. |
| |
| You should have received a copy of the GNU Affero General Public |
| License and the CiviCRM Licensing Exception along |
| with this program; if not, contact CiviCRM LLC |
| at info[AT]civicrm[DOT]org. If you have questions about the |
| GNU Affero General Public License or the licensing of CiviCRM, |
| see the CiviCRM license FAQ at http://civicrm.org/licensing |
+--------------------------------------------------------------------+
*/
/**
*
* @package CRM
* @copyright CiviCRM LLC (c) 2004-2017
* $Id$
*
*/
/**
* The CiviCRM duplicate discovery engine is based on an
* algorithm designed by David Strauss <david@fourkitchens.com>.
*/
class CRM_Dedupe_BAO_RuleGroup extends CRM_Dedupe_DAO_RuleGroup {
/**
* Ids of the contacts to limit the SQL queries (whole-database queries otherwise)
*/
var $contactIds = array();
/**
* Set the contact IDs to restrict the dedupe to.
*
* @param array $contactIds
*/
public function setContactIds($contactIds) {
$this->contactIds = $contactIds;
}
/**
* Params to dedupe against (queries against the whole contact set otherwise)
*/
var $params = array();
/**
* If there are no rules in rule group.
*/
var $noRules = FALSE;
/**
* Return a structure holding the supported tables, fields and their titles
*
* @param string $requestedType
* The requested contact type.
*
* @return array
* a table-keyed array of field-keyed arrays holding supported fields' titles
*/
public static function &supportedFields($requestedType) {
static $fields = NULL;
if (!$fields) {
// this is needed, as we're piggy-backing importableFields() below
$replacements = array(
'civicrm_country.name' => 'civicrm_address.country_id',
'civicrm_county.name' => 'civicrm_address.county_id',
'civicrm_state_province.name' => 'civicrm_address.state_province_id',
'gender.label' => 'civicrm_contact.gender_id',
'individual_prefix.label' => 'civicrm_contact.prefix_id',
'individual_suffix.label' => 'civicrm_contact.suffix_id',
'addressee.label' => 'civicrm_contact.addressee_id',
'email_greeting.label' => 'civicrm_contact.email_greeting_id',
'postal_greeting.label' => 'civicrm_contact.postal_greeting_id',
'civicrm_phone.phone' => 'civicrm_phone.phone_numeric',
);
// the table names we support in dedupe rules - a filter for importableFields()
$supportedTables = array(
'civicrm_address',
'civicrm_contact',
'civicrm_email',
'civicrm_im',
'civicrm_note',
'civicrm_openid',
'civicrm_phone',
);
foreach (array('Individual', 'Organization', 'Household') as $ctype) {
// take the table.field pairs and their titles from importableFields() if the table is supported
foreach (CRM_Contact_BAO_Contact::importableFields($ctype) as $iField) {
if (isset($iField['where'])) {
$where = $iField['where'];
if (isset($replacements[$where])) {
$where = $replacements[$where];
}
list($table, $field) = explode('.', $where);
if (!in_array($table, $supportedTables)) {
continue;
}
$fields[$ctype][$table][$field] = $iField['title'];
}
}
// add custom data fields
foreach (CRM_Core_BAO_CustomGroup::getTree($ctype, NULL, NULL, -1) as $key => $cg) {
if (!is_int($key)) {
continue;
}
foreach ($cg['fields'] as $cf) {
$fields[$ctype][$cg['table_name']][$cf['column_name']] = $cf['label'];
}
}
}
}
CRM_Utils_Hook::dupeQuery(CRM_Core_DAO::$_nullObject, 'supportedFields', $fields);
return $fields[$requestedType];
}
/**
* Return the SQL query for dropping the temporary table.
*/
public function tableDropQuery() {
return 'DROP TEMPORARY TABLE IF EXISTS dedupe';
}
/**
* Return a set of SQL queries whose cummulative weights will mark matched
* records for the RuleGroup::threasholdQuery() to retrieve.
*/
public function tableQuery() {
// make sure we've got a fetched dbrecord, not sure if this is enforced
if (!$this->name == NULL || $this->is_reserved == NULL) {
$this->find(TRUE);
}
// Reserved Rule Groups can optionally get special treatment by
// implementing an optimization class and returning a query array.
if ($this->is_reserved &&
CRM_Utils_File::isIncludable("CRM/Dedupe/BAO/QueryBuilder/{$this->name}.php")
) {
$command = empty($this->params) ? 'internal' : 'record';
$queries = call_user_func(array("CRM_Dedupe_BAO_QueryBuilder_{$this->name}", $command), $this);
}
else {
// All other rule groups have queries generated by the member dedupe
// rules defined in the administrative interface.
// Find all rules contained by this script sorted by weight so that
// their execution can be short circuited on RuleGroup::fillTable()
$bao = new CRM_Dedupe_BAO_Rule();
$bao->dedupe_rule_group_id = $this->id;
$bao->orderBy('rule_weight DESC');
$bao->find();
// Generate a SQL query for each rule in the rule group that is
// tailored to respect the param and contactId options provided.
$queries = array();
while ($bao->fetch()) {
$bao->contactIds = $this->contactIds;
$bao->params = $this->params;
// Skipping empty rules? Empty rules shouldn't exist; why check?
if ($query = $bao->sql()) {
$queries["{$bao->rule_table}.{$bao->rule_field}.{$bao->rule_weight}"] = $query;
}
}
}
// if there are no rules in this rule group
// add an empty query fulfilling the pattern
if (!$queries) {
$this->noRules = TRUE;
return array();
}
return $queries;
}
public function fillTable() {
// get the list of queries handy
$tableQueries = $this->tableQuery();
if ($this->params && !$this->noRules) {
$tempTableQuery = "CREATE TEMPORARY TABLE dedupe (id1 int, weight int, UNIQUE UI_id1 (id1)) ENGINE=InnoDB";
$insertClause = "INSERT INTO dedupe (id1, weight)";
$groupByClause = "GROUP BY id1, weight";
$dupeCopyJoin = " JOIN dedupe_copy ON dedupe_copy.id1 = t1.column WHERE ";
}
else {
$tempTableQuery = "CREATE TEMPORARY TABLE dedupe (id1 int, id2 int, weight int, UNIQUE UI_id1_id2 (id1, id2)) ENGINE=InnoDB";
$insertClause = "INSERT INTO dedupe (id1, id2, weight)";
$groupByClause = "GROUP BY id1, id2, weight";
$dupeCopyJoin = " JOIN dedupe_copy ON dedupe_copy.id1 = t1.column AND dedupe_copy.id2 = t2.column WHERE ";
}
$patternColumn = '/t1.(\w+)/';
$exclWeightSum = array();
// create temp table
$dao = new CRM_Core_DAO();
$dao->query($tempTableQuery);
CRM_Utils_Hook::dupeQuery($this, 'table', $tableQueries);
while (!empty($tableQueries)) {
list($isInclusive, $isDie) = self::isQuerySetInclusive($tableQueries, $this->threshold, $exclWeightSum);
if ($isInclusive) {
// order queries by table count
self::orderByTableCount($tableQueries);
$weightSum = array_sum($exclWeightSum);
$searchWithinDupes = !empty($exclWeightSum) ? 1 : 0;
while (!empty($tableQueries)) {
// extract the next query ( and weight ) to be executed
$fieldWeight = array_keys($tableQueries);
$fieldWeight = $fieldWeight[0];
$query = array_shift($tableQueries);
if ($searchWithinDupes) {
// get prepared to search within already found dupes if $searchWithinDupes flag is set
$dao->query("DROP TEMPORARY TABLE IF EXISTS dedupe_copy");
$dao->query("CREATE TEMPORARY TABLE dedupe_copy SELECT * FROM dedupe WHERE weight >= {$weightSum}");
$dao->free();
preg_match($patternColumn, $query, $matches);
$query = str_replace(' WHERE ', str_replace('column', $matches[1], $dupeCopyJoin), $query);
// CRM-19612: If there's a union, there will be two WHEREs, and you
// can't use the temp table twice.
if (preg_match('/dedupe_copy[\S\s]*(union)[\S\s]*dedupe_copy/i', $query, $matches, PREG_OFFSET_CAPTURE)) {
// Make a second temp table:
$dao->query("DROP TEMPORARY TABLE IF EXISTS dedupe_copy_2");
$dao->query("CREATE TEMPORARY TABLE dedupe_copy_2 SELECT * FROM dedupe WHERE weight >= {$weightSum}");
$dao->free();
// After the union, use that new temp table:
$part1 = substr($query, 0, $matches[1][1]);
$query = $part1 . str_replace('dedupe_copy', 'dedupe_copy_2', substr($query, $matches[1][1]));
}
}
$searchWithinDupes = 1;
// construct and execute the intermediate query
$query = "{$insertClause} {$query} {$groupByClause} ON DUPLICATE KEY UPDATE weight = weight + VALUES(weight)";
$dao->query($query);
// FIXME: we need to be more acurate with affected rows, especially for insert vs duplicate insert.
// And that will help optimize further.
$affectedRows = $dao->affectedRows();
$dao->free();
// In an inclusive situation, failure of any query means no further processing -
if ($affectedRows == 0) {
// reset to make sure no further execution is done.
$tableQueries = array();
break;
}
$weightSum = substr($fieldWeight, strrpos($fieldWeight, '.') + 1) + $weightSum;
}
// An exclusive situation -
}
elseif (!$isDie) {
// since queries are already sorted by weights, we can continue as is
$fieldWeight = array_keys($tableQueries);
$fieldWeight = $fieldWeight[0];
$query = array_shift($tableQueries);
$query = "{$insertClause} {$query} {$groupByClause} ON DUPLICATE KEY UPDATE weight = weight + VALUES(weight)";
$dao->query($query);
if ($dao->affectedRows() >= 1) {
$exclWeightSum[] = substr($fieldWeight, strrpos($fieldWeight, '.') + 1);
}
$dao->free();
}
else {
// its a die situation
break;
}
}
}
/**
* Function to determine if a given query set contains inclusive or exclusive set of weights.
* The function assumes that the query set is already ordered by weight in desc order.
* @param $tableQueries
* @param $threshold
* @param array $exclWeightSum
*
* @return array
*/
public static function isQuerySetInclusive($tableQueries, $threshold, $exclWeightSum = array()) {
$input = array();
foreach ($tableQueries as $key => $query) {
$input[] = substr($key, strrpos($key, '.') + 1);
}
if (!empty($exclWeightSum)) {
$input = array_merge($input, $exclWeightSum);
rsort($input);
}
if (count($input) == 1) {
return array(FALSE, $input[0] < $threshold);
}
$totalCombinations = 0;
for ($i = 0; $i < count($input); $i++) {
$combination = array($input[$i]);
if (array_sum($combination) >= $threshold) {
$totalCombinations++;
continue;
}
for ($j = $i + 1; $j < count($input); $j++) {
$combination[] = $input[$j];
if (array_sum($combination) >= $threshold) {
$totalCombinations++;
}
}
}
return array($totalCombinations == 1, $totalCombinations <= 0);
}
/**
* sort queries by number of records for the table associated with them.
* @param $tableQueries
*/
public static function orderByTableCount(&$tableQueries) {
static $tableCount = array();
$tempArray = array();
foreach ($tableQueries as $key => $query) {
$table = explode(".", $key);
$table = $table[0];
if (!array_key_exists($table, $tableCount)) {
$query = "SELECT COUNT(*) FROM {$table}";
$tableCount[$table] = CRM_Core_DAO::singleValueQuery($query);
}
$tempArray[$key] = $tableCount[$table];
}
asort($tempArray);
foreach ($tempArray as $key => $count) {
$tempArray[$key] = $tableQueries[$key];
}
$tableQueries = $tempArray;
}
/**
* Return the SQL query for getting only the interesting results out of the dedupe table.
*
* @$checkPermission boolean $params a flag to indicate if permission should be considered.
* default is to always check permissioning but public pages for example might not want
* permission to be checked for anonymous users. Refer CRM-6211. We might be beaking
* Multi-Site dedupe for public pages.
*
* @param bool $checkPermission
*
* @return string
*/
public function thresholdQuery($checkPermission = TRUE) {
$this->_aclFrom = '';
// CRM-6603: anonymous dupechecks side-step ACLs
$this->_aclWhere = ' AND is_deleted = 0 ';
if ($this->params && !$this->noRules) {
if ($checkPermission) {
list($this->_aclFrom, $this->_aclWhere) = CRM_Contact_BAO_Contact_Permission::cacheClause('civicrm_contact');
$this->_aclWhere = $this->_aclWhere ? "AND {$this->_aclWhere}" : '';
}
$query = "SELECT dedupe.id1 as id
FROM dedupe JOIN civicrm_contact ON dedupe.id1 = civicrm_contact.id {$this->_aclFrom}
WHERE contact_type = '{$this->contact_type}' {$this->_aclWhere}
AND weight >= {$this->threshold}";
}
else {
$this->_aclWhere = ' AND c1.is_deleted = 0 AND c2.is_deleted = 0';
if ($checkPermission) {
list($this->_aclFrom, $this->_aclWhere) = CRM_Contact_BAO_Contact_Permission::cacheClause(array('c1', 'c2'));
$this->_aclWhere = $this->_aclWhere ? "AND {$this->_aclWhere}" : '';
}
$query = "SELECT IF(dedupe.id1 < dedupe.id2, dedupe.id1, dedupe.id2) as id1,
IF(dedupe.id1 < dedupe.id2, dedupe.id2, dedupe.id1) as id2, dedupe.weight
FROM dedupe JOIN civicrm_contact c1 ON dedupe.id1 = c1.id
JOIN civicrm_contact c2 ON dedupe.id2 = c2.id {$this->_aclFrom}
LEFT JOIN civicrm_dedupe_exception exc ON dedupe.id1 = exc.contact_id1 AND dedupe.id2 = exc.contact_id2
WHERE c1.contact_type = '{$this->contact_type}' AND
c2.contact_type = '{$this->contact_type}' {$this->_aclWhere}
AND weight >= {$this->threshold} AND exc.contact_id1 IS NULL";
}
CRM_Utils_Hook::dupeQuery($this, 'threshold', $query);
return $query;
}
/**
* find fields related to a rule group.
*
* @param array $params
*
* @return array
* (rule field => weight) array and threshold associated to rule group
*/
public static function dedupeRuleFieldsWeight($params) {
$rgBao = new CRM_Dedupe_BAO_RuleGroup();
$rgBao->contact_type = $params['contact_type'];
if (!empty($params['id'])) {
// accept an ID if provided
$rgBao->id = $params['id'];
}
else {
$rgBao->used = $params['used'];
}
$rgBao->find(TRUE);
$ruleBao = new CRM_Dedupe_BAO_Rule();
$ruleBao->dedupe_rule_group_id = $rgBao->id;
$ruleBao->find();
$ruleFields = array();
while ($ruleBao->fetch()) {
$ruleFields[$ruleBao->rule_field] = $ruleBao->rule_weight;
}
return array($ruleFields, $rgBao->threshold);
}
/**
* Get all of the combinations of fields that would work with a rule.
*
* @param array $rgFields
* @param int $threshold
* @param array $combos
* @param array $running
*/
public static function combos($rgFields, $threshold, &$combos, $running = array()) {
foreach ($rgFields as $rgField => $weight) {
unset($rgFields[$rgField]);
$diff = $threshold - $weight;
$runningnow = $running;
$runningnow[] = $rgField;
if ($diff > 0) {
self::combos($rgFields, $diff, $combos, $runningnow);
}
else {
$combos[] = $runningnow;
}
}
}
/**
* Get an array of rule group id to rule group name
* for all th groups for that contactType. If contactType
* not specified, do it for all
*
* @param string $contactType
* Individual, Household or Organization.
*
*
* @return array
* id => "nice name" of rule group
*/
public static function getByType($contactType = NULL) {
$dao = new CRM_Dedupe_DAO_RuleGroup();
if ($contactType) {
$dao->contact_type = $contactType;
}
$dao->find();
$result = array();
while ($dao->fetch()) {
$title = !empty($dao->title) ? $dao->title : (!empty($dao->name) ? $dao->name : $dao->contact_type);
$name = "$title - {$dao->used}";
$result[$dao->id] = $name;
}
return $result;
}
/**
* Get the cached contact type for a particular rule group.
*
* @param int $rule_group_id
*
* @return string
*/
public static function getContactTypeForRuleGroup($rule_group_id) {
if (!isset(\Civi::$statics[__CLASS__]) || !isset(\Civi::$statics[__CLASS__]['rule_groups'])) {
\Civi::$statics[__CLASS__]['rule_groups'] = array();
}
if (empty(\Civi::$statics[__CLASS__]['rule_groups'][$rule_group_id])) {
\Civi::$statics[__CLASS__]['rule_groups'][$rule_group_id]['contact_type'] = CRM_Core_DAO::getFieldValue(
'CRM_Dedupe_DAO_RuleGroup',
$rule_group_id,
'contact_type'
);
}
return \Civi::$statics[__CLASS__]['rule_groups'][$rule_group_id]['contact_type'];
}
}