From ae8b18530b7e6dee31eb19a42e76dcf0bd9b5360 Mon Sep 17 00:00:00 2001
From: Guillaume Heurtier <guillaume.heurtier@maarch.org>
Date: Tue, 27 Oct 2020 12:01:16 +0100
Subject: [PATCH] FEAT #13271 TIME 6:45 optimize search
---
migration/20.10/2010.sql | 2 ++
sql/structure.sql | 2 ++
.../contact/controllers/ContactController.php | 2 +-
.../search/controllers/SearchController.php | 24 ++++++-------
.../controllers/AutoCompleteController.php | 34 +++++++++----------
5 files changed, 34 insertions(+), 30 deletions(-)
diff --git a/migration/20.10/2010.sql b/migration/20.10/2010.sql
index 8e9dd2479c1..57851cc3456 100755
--- a/migration/20.10/2010.sql
+++ b/migration/20.10/2010.sql
@@ -9,6 +9,8 @@ UPDATE parameters SET param_value_string = '20.10.1' WHERE id = 'database_versio
DROP VIEW IF EXISTS res_view_letterbox;
+CREATE EXTENSION unaccent;
+
/* SENDMAIL */
DROP TABLE IF EXISTS sendmail;
diff --git a/sql/structure.sql b/sql/structure.sql
index 3da57bbc7a3..e03bcc9a582 100755
--- a/sql/structure.sql
+++ b/sql/structure.sql
@@ -13,6 +13,8 @@ SET search_path = public, pg_catalog;
SET default_tablespace = '';
SET default_with_oids = false;
+CREATE EXTENSION unaccent;
+
CREATE TABLE actions
(
id serial NOT NULL,
diff --git a/src/app/contact/controllers/ContactController.php b/src/app/contact/controllers/ContactController.php
index ebdb9ffb7b4..2b1a25997a7 100755
--- a/src/app/contact/controllers/ContactController.php
+++ b/src/app/contact/controllers/ContactController.php
@@ -75,7 +75,7 @@ class ContactController
if (!empty($queryParams['search'])) {
$fields = ['firstname', 'lastname', 'company', 'address_number', 'address_street', 'address_additional1', 'address_additional2', 'address_postcode', 'address_town', 'address_country'];
$fieldsNumber = count($fields);
- $fields = AutoCompleteController::getUnsensitiveFieldsForRequest(['fields' => $fields]);
+ $fields = AutoCompleteController::getInsensitiveFieldsForRequest(['fields' => $fields]);
$requestData = AutoCompleteController::getDataForRequest([
'search' => $queryParams['search'],
diff --git a/src/app/search/controllers/SearchController.php b/src/app/search/controllers/SearchController.php
index 378bae2c671..e18883b9ea4 100644
--- a/src/app/search/controllers/SearchController.php
+++ b/src/app/search/controllers/SearchController.php
@@ -324,7 +324,7 @@ class SearchController
$args['searchData'] = array_merge($args['searchData'], [$quick, $quick, $quick, $quick, $quick]);
} else {
$fields = ['subject', 'alt_identifier', 'barcode'];
- $fields = AutoCompleteController::getUnsensitiveFieldsForRequest(['fields' => $fields]);
+ $fields = AutoCompleteController::getInsensitiveFieldsForRequest(['fields' => $fields]);
$requestDataDocument = AutoCompleteController::getDataForRequest([
'search' => $body['meta']['values'],
'fields' => $fields,
@@ -334,7 +334,7 @@ class SearchController
]);
$fields = ['title', 'identifier'];
- $fields = AutoCompleteController::getUnsensitiveFieldsForRequest(['fields' => $fields]);
+ $fields = AutoCompleteController::getInsensitiveFieldsForRequest(['fields' => $fields]);
$requestDataAttachment = AutoCompleteController::getDataForRequest([
'search' => $body['meta']['values'],
'fields' => $fields,
@@ -381,7 +381,7 @@ class SearchController
$args['searchData'][] = $subject;
} else {
$fields = ['subject'];
- $fields = AutoCompleteController::getUnsensitiveFieldsForRequest(['fields' => $fields]);
+ $fields = AutoCompleteController::getInsensitiveFieldsForRequest(['fields' => $fields]);
$requestData = AutoCompleteController::getDataForRequest([
'search' => $body['subject']['values'],
'fields' => $fields,
@@ -390,7 +390,7 @@ class SearchController
'fieldsNumber' => 1
]);
$subjectGlue = implode(' AND ', $requestData['where']);
- $attachmentField = AutoCompleteController::getUnsensitiveFieldsForRequest(['fields' => ['title']]);
+ $attachmentField = AutoCompleteController::getInsensitiveFieldsForRequest(['fields' => ['title']]);
$subjectGlue = "(($subjectGlue) OR res_id in (select res_id_master from res_attachments where {$attachmentField} and status in ('TRA', 'A_TRA', 'FRZ')))";
$args['searchWhere'][] = $subjectGlue;
$args['searchData'] = array_merge($args['searchData'], $requestData['data']);
@@ -589,7 +589,7 @@ class SearchController
$args['searchData'][] = $sendersMatch;
}
if (!empty($body['senders']) && !empty($body['senders']['values']) && is_array($body['senders']['values']) && is_string($body['senders']['values'][0])) {
- $fields = AutoCompleteController::getUnsensitiveFieldsForRequest(['fields' => ['company']]);
+ $fields = AutoCompleteController::getInsensitiveFieldsForRequest(['fields' => ['company']]);
$requestData = AutoCompleteController::getDataForRequest([
'search' => $body['senders']['values'][0],
@@ -645,7 +645,7 @@ class SearchController
$args['searchData'][] = $recipientsMatch;
}
if (!empty($body['recipients']) && !empty($body['recipients']['values']) && is_array($body['recipients']['values']) && is_string($body['recipients']['values'][0])) {
- $fields = AutoCompleteController::getUnsensitiveFieldsForRequest(['fields' => ['company']]);
+ $fields = AutoCompleteController::getInsensitiveFieldsForRequest(['fields' => ['company']]);
$requestData = AutoCompleteController::getDataForRequest([
'search' => $body['recipients']['values'][0],
@@ -856,7 +856,7 @@ class SearchController
if (!empty($where)) {
$where .= ' OR ';
}
- $where .= '(item_id = ? AND item_type = ?';
+ $where .= '((item_id = ? AND item_type = ?)';
$data[] = $itemValue['id'];
$data[] = $itemValue['type'] == 'user' ? 'user_id' : 'entity_id';
@@ -920,7 +920,7 @@ class SearchController
$args['searchData'][] = $subject;
} else {
$fields = ["custom_fields->>'{$customFieldId}'"];
- $fields = AutoCompleteController::getUnsensitiveFieldsForRequest(['fields' => $fields]);
+ $fields = AutoCompleteController::getInsensitiveFieldsForRequest(['fields' => $fields]);
$requestData = AutoCompleteController::getDataForRequest([
'search' => $value['values'],
'fields' => $fields,
@@ -995,7 +995,7 @@ class SearchController
}
$args['searchWhere'][] = '(' . implode(' or ', $contactSearchWhere) . ')';
} elseif (!empty($value['values']) && is_array($value['values']) && is_string($value['values'][0])) {
- $fields = AutoCompleteController::getUnsensitiveFieldsForRequest(['fields' => ['company']]);
+ $fields = AutoCompleteController::getInsensitiveFieldsForRequest(['fields' => ['company']]);
$requestData = AutoCompleteController::getDataForRequest([
'search' => $value['values'],
@@ -1120,7 +1120,7 @@ class SearchController
$args['searchData'][] = $registeredMailsMatch;
}
if (!empty($body['registeredMail_recipient']) && !empty($body['registeredMail_recipient']['values']) && is_array($body['registeredMail_recipient']['values']) && is_string($body['registeredMail_recipient']['values'][0])) {
- $fields = AutoCompleteController::getUnsensitiveFieldsForRequest(['fields' => ["recipient->>'company'"]]);
+ $fields = AutoCompleteController::getInsensitiveFieldsForRequest(['fields' => ["recipient->>'company'"]]);
$requestData = AutoCompleteController::getDataForRequest([
'search' => $body['registeredMail_recipient']['values'][0],
'fields' => $fields,
@@ -1835,7 +1835,7 @@ class SearchController
$subject = trim($body['subject']['values'], '"');
$data[] = $subject;
} else {
- $attachmentField = AutoCompleteController::getUnsensitiveFieldsForRequest(['fields' => ['title']]);
+ $attachmentField = AutoCompleteController::getInsensitiveFieldsForRequest(['fields' => ['title']]);
$wherePlus = "res_id in (select res_id_master from res_attachments where {$attachmentField} and status in ('TRA', 'A_TRA', 'FRZ'))";
$data[] = "%{$body['subject']['values']}%";
}
@@ -1858,7 +1858,7 @@ class SearchController
$data[] = $quick;
} else {
$fields = ['title', 'identifier'];
- $fields = AutoCompleteController::getUnsensitiveFieldsForRequest(['fields' => $fields]);
+ $fields = AutoCompleteController::getInsensitiveFieldsForRequest(['fields' => $fields]);
$requestDataAttachment = AutoCompleteController::getDataForRequest([
'search' => $body['meta']['values'],
'fields' => $fields,
diff --git a/src/core/controllers/AutoCompleteController.php b/src/core/controllers/AutoCompleteController.php
index c8bdc875204..9c29007e111 100755
--- a/src/core/controllers/AutoCompleteController.php
+++ b/src/core/controllers/AutoCompleteController.php
@@ -49,7 +49,7 @@ class AutoCompleteController
}
$fields = ['firstname', 'lastname'];
- $fields = AutoCompleteController::getUnsensitiveFieldsForRequest(['fields' => $fields]);
+ $fields = AutoCompleteController::getInsensitiveFieldsForRequest(['fields' => $fields]);
$requestData = AutoCompleteController::getDataForRequest([
'search' => $queryParams['search'],
@@ -174,7 +174,7 @@ class AutoCompleteController
}
$fieldsNumber = count($fields);
- $fields = AutoCompleteController::getUnsensitiveFieldsForRequest(['fields' => $fields]);
+ $fields = AutoCompleteController::getInsensitiveFieldsForRequest(['fields' => $fields]);
$requestData = AutoCompleteController::getDataForRequest([
'search' => $queryParams['search'],
@@ -214,7 +214,7 @@ class AutoCompleteController
$nbFields = count($fields);
- $fields = AutoCompleteController::getUnsensitiveFieldsForRequest(['fields' => $fields]);
+ $fields = AutoCompleteController::getInsensitiveFieldsForRequest(['fields' => $fields]);
$requestData = AutoCompleteController::getDataForRequest([
'search' => $queryParams['search'],
'fields' => $fields,
@@ -258,7 +258,7 @@ class AutoCompleteController
$nbFields = count($fields);
- $fields = AutoCompleteController::getUnsensitiveFieldsForRequest(['fields' => $fields]);
+ $fields = AutoCompleteController::getInsensitiveFieldsForRequest(['fields' => $fields]);
$requestData = AutoCompleteController::getDataForRequest([
'search' => $queryParams['search'],
'fields' => $fields,
@@ -295,7 +295,7 @@ class AutoCompleteController
$autocompleteContactsGroups = [];
if (empty($queryParams['noContactsGroups'])) {
$fields = ['label'];
- $fields = AutoCompleteController::getUnsensitiveFieldsForRequest(['fields' => $fields]);
+ $fields = AutoCompleteController::getInsensitiveFieldsForRequest(['fields' => $fields]);
$requestData = AutoCompleteController::getDataForRequest([
'search' => $queryParams['search'],
'fields' => $fields,
@@ -347,7 +347,7 @@ class AutoCompleteController
$entities = EntityModel::getAllEntitiesByUserId(['userId' => $GLOBALS['id']]);
$fields = ['users.firstname', 'users.lastname'];
- $fields = AutoCompleteController::getUnsensitiveFieldsForRequest(['fields' => $fields]);
+ $fields = AutoCompleteController::getInsensitiveFieldsForRequest(['fields' => $fields]);
$requestData = AutoCompleteController::getDataForRequest([
'search' => $data['search'],
@@ -371,7 +371,7 @@ class AutoCompleteController
if (count($users) < self::LIMIT) {
$fields = ['users.firstname', 'users.lastname'];
- $fields = AutoCompleteController::getUnsensitiveFieldsForRequest(['fields' => $fields]);
+ $fields = AutoCompleteController::getInsensitiveFieldsForRequest(['fields' => $fields]);
$requestData = AutoCompleteController::getDataForRequest([
'search' => $data['search'],
@@ -448,7 +448,7 @@ class AutoCompleteController
if (!empty($queryParams['search'])) {
$fields = ['users.firstname', 'users.lastname'];
- $fields = AutoCompleteController::getUnsensitiveFieldsForRequest(['fields' => $fields]);
+ $fields = AutoCompleteController::getInsensitiveFieldsForRequest(['fields' => $fields]);
$requestData = AutoCompleteController::getDataForRequest([
'search' => $queryParams['search'],
@@ -489,7 +489,7 @@ class AutoCompleteController
}
$fields = ['entity_label'];
- $fields = AutoCompleteController::getUnsensitiveFieldsForRequest(['fields' => $fields]);
+ $fields = AutoCompleteController::getInsensitiveFieldsForRequest(['fields' => $fields]);
$requestData = AutoCompleteController::getDataForRequest([
'search' => $queryParams['search'],
@@ -560,7 +560,7 @@ class AutoCompleteController
}
$fieldsNumber = count($fields);
- $fields = AutoCompleteController::getUnsensitiveFieldsForRequest(['fields' => $fields]);
+ $fields = AutoCompleteController::getInsensitiveFieldsForRequest(['fields' => $fields]);
$requestData = AutoCompleteController::getDataForRequest([
'search' => $data['search'],
@@ -594,7 +594,7 @@ class AutoCompleteController
return $response->withStatus(400)->withJson(['errors' => 'Query params search is empty']);
}
- $fields = AutoCompleteController::getUnsensitiveFieldsForRequest(['fields' => ['company']]);
+ $fields = AutoCompleteController::getInsensitiveFieldsForRequest(['fields' => ['company']]);
$contacts = ContactModel::get([
'select' => [
'id', 'company', 'address_number as "addressNumber"', 'address_street as "addressStreet"',
@@ -754,7 +754,7 @@ class AutoCompleteController
}
$fieldsNumber = count($fields);
- $fields = AutoCompleteController::getUnsensitiveFieldsForRequest(['fields' => $fields]);
+ $fields = AutoCompleteController::getInsensitiveFieldsForRequest(['fields' => $fields]);
$requestData = AutoCompleteController::getDataForRequest([
'search' => $queryParams['search'],
@@ -863,7 +863,7 @@ class AutoCompleteController
$arrScopedFoldersIds = array_column($scopedFolders, 'id');
- $fields = AutoCompleteController::getUnsensitiveFieldsForRequest(['fields' => ['label']]);
+ $fields = AutoCompleteController::getInsensitiveFieldsForRequest(['fields' => ['label']]);
$selectedFolders = FolderModel::get([
'where' => ["{$fields} AND id in (?)"],
@@ -893,7 +893,7 @@ class AutoCompleteController
}
$fields = ['label'];
- $fields = AutoCompleteController::getUnsensitiveFieldsForRequest(['fields' => $fields]);
+ $fields = AutoCompleteController::getInsensitiveFieldsForRequest(['fields' => $fields]);
$requestData = AutoCompleteController::getDataForRequest([
'search' => $data['search'],
@@ -953,15 +953,15 @@ class AutoCompleteController
return ['where' => $args['where'], 'data' => $args['data']];
}
- public static function getUnsensitiveFieldsForRequest(array $args)
+ public static function getInsensitiveFieldsForRequest(array $args)
{
ValidatorModel::notEmpty($args, ['fields']);
ValidatorModel::arrayType($args, ['fields']);
$fields = [];
foreach ($args['fields'] as $key => $field) {
- $fields[$key] = "translate({$field}, 'ÀÃÂÃÄÅÆÇÈÉÊËÌÃÃŽÃÃÑÒÓÔÕÖØÙÚÛÜÃÞßà áâãäåæçèéêëìÃîïðñòóôõöøùúûýýþÿŔŕ', 'aaaaaaaceeeeiiiidnoooooouuuuybsaaaaaaaceeeeiiiidnoooooouuuyybyRr')";
- $fields[$key] .= "ilike translate(?, 'ÀÃÂÃÄÅÆÇÈÉÊËÌÃÃŽÃÃÑÒÓÔÕÖØÙÚÛÜÃÞßà áâãäåæçèéêëìÃîïðñòóôõöøùúûýýþÿŔŕ', 'aaaaaaaceeeeiiiidnoooooouuuuybsaaaaaaaceeeeiiiidnoooooouuuyybyRr')";
+ $fields[$key] = "unaccent({$field}::text)";
+ $fields[$key] .= " ilike unaccent(?::text)";
}
$fields = implode(' OR ', $fields);
$fields = "({$fields})";
--
GitLab