Skip to content
Snippets Groups Projects
Verified Commit 4ea3730c authored by Florian Azizian's avatar Florian Azizian
Browse files

FEAT #12754 TIME 0:40 improve contact migration speed

parent 76f3fe06
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
migration/19.12/migrateContacts.php
+ 61
− 170
View file @ 4ea3730c
...@@ -160,51 +160,37 @@ foreach ($customs as $custom) { ...@@ -160,51 +160,37 @@ foreach ($customs as $custom) {
unset($contactInfo['society_short']); unset($contactInfo['society_short']);
$id = \Contact\models\ContactModel::create($contactInfo); $id = \Contact\models\ContactModel::create($contactInfo);
$debutAllMigration = microtime(true);
migrateCustomField(['newContactId' => $id, 'contactCustomInfo' => $contactCustomInfo, 'newCustomFields' => $newCustomFields]); migrateCustomField(['newContactId' => $id, 'contactCustomInfo' => $contactCustomInfo, 'newCustomFields' => $newCustomFields]);
$finAllMigration = microtime(true); \SrcCore\models\DatabaseModel::update([
$delaiAllMigration = $finAllMigration - $debutAllMigration; 'set' => ['contact_id' => $id],
echo "migrateCustomField : ".$delaiAllMigration." secondes.\n"; 'table' => 'acknowledgement_receipts',
'where' => ['contact_address_id = ?'],
$debutAllMigration = microtime(true); 'data' => [$oldAddressId]
migrateAcknowledgementReceipt(['oldAddressId' => $oldAddressId, 'newContactId' => $id]); ]);
$finAllMigration = microtime(true); \SrcCore\models\DatabaseModel::update([
$delaiAllMigration = $finAllMigration - $debutAllMigration; 'set' => ['contact_id' => $id],
echo "migrateAcknowledgementReceipt : ".$delaiAllMigration." secondes.\n"; 'table' => 'contacts_groups_lists',
'where' => ['contact_addresses_id = ?'],
$debutAllMigration = microtime(true); 'data' => [$oldAddressId]
migrateContactGroupsLists(['oldAddressId' => $oldAddressId, 'newContactId' => $id]); ]);
$finAllMigration = microtime(true);
$delaiAllMigration = $finAllMigration - $debutAllMigration;
echo "migrateContactGroupsLists : ".$delaiAllMigration." secondes.\n";
$debutAllMigration = microtime(true);
migrateContactRes(['oldAddressId' => $oldAddressId, 'oldContactId' => $oldContactId, 'newContactId' => $id]); migrateContactRes(['oldAddressId' => $oldAddressId, 'oldContactId' => $oldContactId, 'newContactId' => $id]);
$finAllMigration = microtime(true); \SrcCore\models\DatabaseModel::update([
$delaiAllMigration = $finAllMigration - $debutAllMigration; 'set' => ['item_id' => $id, 'type' => 'contact_v3'],
echo "migrateContactRes : ".$delaiAllMigration." secondes.\n"; 'table' => 'resource_contacts',
'where' => ['item_id = ?', 'type = ?'],
$debutAllMigration = microtime(true); 'data' => [$oldAddressId, 'contact']
migrateResourceContacts(['oldAddressId' => $oldAddressId, 'newContactId' => $id]); ]);
$finAllMigration = microtime(true); \SrcCore\models\DatabaseModel::update([
$delaiAllMigration = $finAllMigration - $debutAllMigration; 'set' => ['recipient_id' => $id, 'recipient_type' => 'contact'],
echo "migrateResourceContacts : ".$delaiAllMigration." secondes.\n"; 'table' => 'res_attachments',
'where' => ['dest_contact_id = ?', 'dest_address_id = ?'],
$debutAllMigration = microtime(true); 'data' => [$oldContactId, $oldAddressId],
migrateResAttachments(['oldAddressId' => $oldAddressId, 'oldContactId' => $oldContactId, 'newContactId' => $id]); ]);
$finAllMigration = microtime(true);
$delaiAllMigration = $finAllMigration - $debutAllMigration;
echo "migrateResAttachments : ".$delaiAllMigration." secondes.\n";
$debutAllMigration = microtime(true);
migrateResletterbox(['oldAddressId' => $oldAddressId, 'newContactId' => $id]); migrateResletterbox(['oldAddressId' => $oldAddressId, 'newContactId' => $id]);
$finAllMigration = microtime(true);
$delaiAllMigration = $finAllMigration - $debutAllMigration;
echo "migrateResletterbox : ".$delaiAllMigration." secondes.\n";
$migrated++; $migrated++;
if ($migrated % 5000 == 0) { if ($migrated % 10000 == 0) {
$finMigrateInProgress = microtime(true); $finMigrateInProgress = microtime(true);
$delaiInProgress = $finMigrateInProgress - $debutMigrateInProgress; $delaiInProgress = $finMigrateInProgress - $debutMigrateInProgress;
echo "Migration En cours : ".$delaiInProgress." secondes.\n"; echo "Migration En cours : ".$delaiInProgress." secondes.\n";
...@@ -214,9 +200,9 @@ foreach ($customs as $custom) { ...@@ -214,9 +200,9 @@ foreach ($customs as $custom) {
} }
$debutEndMigrate = microtime(true); $debutEndMigrate = microtime(true);
migrateContactRes_Users(); migrateContactRes_Users(['firstManId' => $firstMan[0]['id']]);
migrateResletterbox_Users(); migrateResletterbox_Users(['firstManId' => $firstMan[0]['id']]);
migrateResattachments_Users(); migrateResattachments_Users(['firstManId' => $firstMan[0]['id']]);
migrateContactParameters(); migrateContactParameters();
migrateContactPrivileges(); migrateContactPrivileges();
$finEndMigrate = microtime(true); $finEndMigrate = microtime(true);
...@@ -299,26 +285,6 @@ function migrateCustomField($args = []) ...@@ -299,26 +285,6 @@ function migrateCustomField($args = [])
\SrcCore\models\DatabaseModel::commitTransaction(); \SrcCore\models\DatabaseModel::commitTransaction();
} }
function migrateAcknowledgementReceipt($args = [])
{
\SrcCore\models\DatabaseModel::update([
'set' => ['contact_id' => $args['newContactId']],
'table' => 'acknowledgement_receipts',
'where' => ['contact_address_id = ?'],
'data' => [$args['oldAddressId']]
]);
}
function migrateContactGroupsLists($args = [])
{
\SrcCore\models\DatabaseModel::update([
'set' => ['contact_id' => $args['newContactId']],
'table' => 'contacts_groups_lists',
'where' => ['contact_addresses_id = ?'],
'data' => [$args['oldAddressId']]
]);
}
function migrateContactRes($args = []) function migrateContactRes($args = [])
{ {
\SrcCore\models\DatabaseModel::beginTransaction(); \SrcCore\models\DatabaseModel::beginTransaction();
...@@ -343,55 +309,26 @@ function migrateContactRes($args = []) ...@@ -343,55 +309,26 @@ function migrateContactRes($args = [])
$mode = 'recipient'; $mode = 'recipient';
} }
$aValues[] = [ $aValues[] = implode("\t", [
$value['res_id'], $value['res_id'],
$args['newContactId'], $args['newContactId'],
'contact_v3', 'contact_v3',
$mode $mode
]; ]) . "\n";
$countContactRes++;
if ($countContactRes % 50 == 0) {
\SrcCore\models\DatabaseModel::insertMultiple([
'table' => 'resource_contacts',
'columns' => ['res_id', 'item_id', 'type', 'mode'],
'values' => $aValues
]);
$aValues = [];
}
} }
if (!empty($aValues)) { if (!empty($aValues)) {
\SrcCore\models\DatabaseModel::insertMultiple([ \SrcCore\models\DatabaseModel::pgsqlCopyFromArray([
'table' => 'resource_contacts', 'table' => 'resource_contacts',
'columns' => ['res_id', 'item_id', 'type', 'mode'], 'rows' => $aValues,
'values' => $aValues 'delimiter' => "\t",
'nullAs' => "\\\\N",
'fields' => 'res_id, item_id, type, mode'
]); ]);
} }
\SrcCore\models\DatabaseModel::commitTransaction(); \SrcCore\models\DatabaseModel::commitTransaction();
} }
function migrateResourceContacts($args = [])
{
\SrcCore\models\DatabaseModel::update([
'set' => ['item_id' => $args['newContactId'], 'type' => 'contact_v3'],
'table' => 'resource_contacts',
'where' => ['item_id = ?', 'type = ?'],
'data' => [$args['oldAddressId'], 'contact']
]);
}
function migrateResAttachments($args = [])
{
\SrcCore\models\DatabaseModel::update([
'set' => ['recipient_id' => $args['newContactId'], 'recipient_type' => 'contact'],
'table' => 'res_attachments',
'where' => ['dest_contact_id = ?', 'dest_address_id = ?'],
'data' => [$args['oldContactId'], $args['oldAddressId']],
]);
}
function migrateResletterbox($args = []) function migrateResletterbox($args = [])
{ {
\SrcCore\models\DatabaseModel::beginTransaction(); \SrcCore\models\DatabaseModel::beginTransaction();
...@@ -415,19 +352,6 @@ function migrateResletterbox($args = []) ...@@ -415,19 +352,6 @@ function migrateResletterbox($args = [])
'contact_v3', 'contact_v3',
$mode $mode
]) . "\n"; ]) . "\n";
$countContactLetterbox++;
if ($countContactLetterbox % 200 == 0) {
\SrcCore\models\DatabaseModel::pgsqlCopyFromArray([
'table' => 'resource_contacts',
'rows' => $aValues,
'delimiter' => "\t",
'nullAs' => "\\\\N",
'fields' => 'res_id, item_id, type, mode'
]);
$aValues = [];
}
} }
if (!empty($aValues)) { if (!empty($aValues)) {
...@@ -442,7 +366,7 @@ function migrateResletterbox($args = []) ...@@ -442,7 +366,7 @@ function migrateResletterbox($args = [])
\SrcCore\models\DatabaseModel::commitTransaction(); \SrcCore\models\DatabaseModel::commitTransaction();
} }
function migrateContactRes_Users() function migrateContactRes_Users($args = [])
{ {
\SrcCore\models\DatabaseModel::beginTransaction(); \SrcCore\models\DatabaseModel::beginTransaction();
$userContactRes = \SrcCore\models\DatabaseModel::select([ $userContactRes = \SrcCore\models\DatabaseModel::select([
...@@ -451,7 +375,6 @@ function migrateContactRes_Users() ...@@ -451,7 +375,6 @@ function migrateContactRes_Users()
'where' => ['address_id = 0'] 'where' => ['address_id = 0']
]); ]);
$firstMan = \User\models\UserModel::get(['select' => ['id'], 'orderBy' => ['id'], 'limit' => 1, 'where' => ['status = ?'], 'data' => ['OK']]);
$aValues = []; $aValues = [];
foreach ($userContactRes as $value) { foreach ($userContactRes as $value) {
$resInfo = \SrcCore\models\DatabaseModel::select([ $resInfo = \SrcCore\models\DatabaseModel::select([
...@@ -463,7 +386,7 @@ function migrateContactRes_Users() ...@@ -463,7 +386,7 @@ function migrateContactRes_Users()
$user = \User\models\UserModel::getByLogin(['login' => $value['contact_id'], 'select' => ['id']]); $user = \User\models\UserModel::getByLogin(['login' => $value['contact_id'], 'select' => ['id']]);
if (empty($user)) { if (empty($user)) {
$user = $firstMan[0]['id']; $user = $args['firstManId'];
} else { } else {
$user = $user['id']; $user = $user['id'];
} }
...@@ -473,36 +396,27 @@ function migrateContactRes_Users() ...@@ -473,36 +396,27 @@ function migrateContactRes_Users()
$mode = 'recipient'; $mode = 'recipient';
} }
$aValues[] = [ $aValues[] = implode("\t", [
$value['res_id'], $value['res_id'],
$user, $user,
'user', 'user',
$mode $mode
]; ]) . "\n";
$countContactUser++;
if ($countContactUser % 50 == 0) {
\SrcCore\models\DatabaseModel::insertMultiple([
'table' => 'resource_contacts',
'columns' => ['res_id', 'item_id', 'type', 'mode'],
'values' => $aValues
]);
$aValues = [];
}
} }
if (!empty($aValues)) { if (!empty($aValues)) {
\SrcCore\models\DatabaseModel::insertMultiple([ \SrcCore\models\DatabaseModel::pgsqlCopyFromArray([
'table' => 'resource_contacts', 'table' => 'resource_contacts',
'columns' => ['res_id', 'item_id', 'type', 'mode'], 'rows' => $aValues,
'values' => $aValues 'delimiter' => "\t",
'nullAs' => "\\\\N",
'fields' => 'res_id, item_id, type, mode'
]); ]);
} }
\SrcCore\models\DatabaseModel::commitTransaction(); \SrcCore\models\DatabaseModel::commitTransaction();
} }
function migrateResletterbox_Users() function migrateResletterbox_Users($args = [])
{ {
\SrcCore\models\DatabaseModel::beginTransaction(); \SrcCore\models\DatabaseModel::beginTransaction();
$userContact = \SrcCore\models\DatabaseModel::select([ $userContact = \SrcCore\models\DatabaseModel::select([
...@@ -511,7 +425,6 @@ function migrateResletterbox_Users() ...@@ -511,7 +425,6 @@ function migrateResletterbox_Users()
'where' => ['(exp_user_id != \'\' and exp_user_id is not null) or (dest_user_id != \'\' and dest_user_id is not null)'] 'where' => ['(exp_user_id != \'\' and exp_user_id is not null) or (dest_user_id != \'\' and dest_user_id is not null)']
]); ]);
$firstMan = \User\models\UserModel::get(['select' => ['id'], 'orderBy' => ['id'], 'limit' => 1, 'where' => ['status = ?'], 'data' => ['OK']]);
$aValues = []; $aValues = [];
foreach ($userContact as $value) { foreach ($userContact as $value) {
if (!empty($value['exp_user_id'])) { if (!empty($value['exp_user_id'])) {
...@@ -523,41 +436,32 @@ function migrateResletterbox_Users() ...@@ -523,41 +436,32 @@ function migrateResletterbox_Users()
} }
$user = \User\models\UserModel::getByLogin(['login' => $login, 'select' => ['id']]); $user = \User\models\UserModel::getByLogin(['login' => $login, 'select' => ['id']]);
if (empty($user)) { if (empty($user)) {
$user = $firstMan[0]['id']; $user = $args['firstManId'];
} else { } else {
$user = $user['id']; $user = $user['id'];
} }
$aValues[] = [ $aValues[] = implode("\t", [
$value['res_id'], $value['res_id'],
$user, $user,
'user', 'user',
$mode $mode
]; ]) . "\n";
$countContactUser++;
if ($countContactUser % 50 == 0) {
\SrcCore\models\DatabaseModel::insertMultiple([
'table' => 'resource_contacts',
'columns' => ['res_id', 'item_id', 'type', 'mode'],
'values' => $aValues
]);
$aValues = [];
}
} }
if (!empty($aValues)) { if (!empty($aValues)) {
\SrcCore\models\DatabaseModel::insertMultiple([ \SrcCore\models\DatabaseModel::pgsqlCopyFromArray([
'table' => 'resource_contacts', 'table' => 'resource_contacts',
'columns' => ['res_id', 'item_id', 'type', 'mode'], 'rows' => $aValues,
'values' => $aValues 'delimiter' => "\t",
'nullAs' => "\\\\N",
'fields' => 'res_id, item_id, type, mode'
]); ]);
} }
\SrcCore\models\DatabaseModel::commitTransaction(); \SrcCore\models\DatabaseModel::commitTransaction();
} }
function migrateResattachments_Users() function migrateResattachments_Users($args = [])
{ {
\SrcCore\models\DatabaseModel::beginTransaction(); \SrcCore\models\DatabaseModel::beginTransaction();
$attachments = \SrcCore\models\DatabaseModel::select([ $attachments = \SrcCore\models\DatabaseModel::select([
...@@ -566,11 +470,10 @@ function migrateResattachments_Users() ...@@ -566,11 +470,10 @@ function migrateResattachments_Users()
'where' => ['dest_user != \'\' and dest_user is not null'] 'where' => ['dest_user != \'\' and dest_user is not null']
]); ]);
$firstMan = \User\models\UserModel::get(['select' => ['id'], 'orderBy' => ['id'], 'limit' => 1, 'where' => ['status = ?'], 'data' => ['OK']]);
foreach ($attachments as $value) { foreach ($attachments as $value) {
$user = \User\models\UserModel::getByLogin(['login' => $value['dest_user'], 'select' => ['id']]); $user = \User\models\UserModel::getByLogin(['login' => $value['dest_user'], 'select' => ['id']]);
if (empty($user)) { if (empty($user)) {
$user = $firstMan[0]['id']; $user = $args['firstManId'];
} else { } else {
$user = $user['id']; $user = $user['id'];
} }
...@@ -626,26 +529,14 @@ function migrateContactParameters() ...@@ -626,26 +529,14 @@ function migrateContactParameters()
$value['searchable'], $value['searchable'],
$value['displayable'] $value['displayable']
]; ];
$countParameters++;
if ($countParameters % 50 == 0) {
\SrcCore\models\DatabaseModel::insertMultiple([
'table' => 'contacts_parameters',
'columns' => ['identifier', 'mandatory', 'filling', 'searchable', 'displayable'],
'values' => $aValues
]);
$aValues = [];
}
} }
if (!empty($aValues)) { \SrcCore\models\DatabaseModel::insertMultiple([
\SrcCore\models\DatabaseModel::insertMultiple([ 'table' => 'contacts_parameters',
'table' => 'contacts_parameters', 'columns' => ['identifier', 'mandatory', 'filling', 'searchable', 'displayable'],
'columns' => ['identifier', 'mandatory', 'filling', 'searchable', 'displayable'], 'values' => $aValues
'values' => $aValues ]);
]);
}
\SrcCore\models\DatabaseModel::commitTransaction(); \SrcCore\models\DatabaseModel::commitTransaction();
} }
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment